8000 Unify Python dependencies and declare optional features by Napreth · Pull Request #832 · bkerler/edl · GitHub
[go: up one dir, main page]

Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/scripts/update_requirements.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Generate the legacy requirements file from project dependency metadata."""
import argparse
from pathlib import Path

try:
import tomllib
except ModuleNotFoundError:
import tomli as tomllib


ROOT = Path(__file__).resolve().parents[2]


def render_requirements():
with (ROOT / "pyproject.toml").open("rb") as source:
project = tomllib.load(source)["project"]
lines = [
"# Generated from pyproject.toml; do not edit this file directly.",
"# Regenerate with: python .github/scripts/update_requirements.py",
"# Includes all optional features for legacy source installations.",
"",
"# Required by the CLI commands",
*project["dependencies"],
]
for extra, requirements in project.get("optional-dependencies", {}).items():
lines.extend(["", f"# Optional feature: {extra}", *requirements])
return "\n".join(lines) + "\n"


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true", help="Fail if requirements.txt is out of date")
args = parser.parse_args()
target = ROOT / "requirements.txt"
expected = render_requirements()
if args.check:
if not target.exists() or target.read_text(encoding="utf-8") != expected:
parser.exit(1, "requirements.txt is out of date; run python .github/scripts/update_requirements.py\n")
else:
target.write_text(expected, encoding="utf-8")


if __name__ == "__main__":
main()
140 changes: 140 additions & 0 deletions .github/tests/test_dependencies.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
"""Dependency resolution checks, independent of CLI packaging fixes."""
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import venv


ROOT = Path(__file__).resolve().parents[2]


def run(*command, cwd):
env = os.environ.copy()
env.pop("PYTHONPATH", None)
env.pop("PYTHONHOME", None)
env["PYTHONIOENCODING"] = "utf-8"
result = subprocess.run(
command, cwd=cwd, env=env, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=180,
)
if result.returncode:
raise AssertionError(
f"Command failed ({result.returncode}): {command}\n"
f"{result.stdout}\n{result.stderr}"
)
return result.stdout


class DependencyTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temp = tempfile.TemporaryDirectory()
cls.addClassCleanup(cls.temp.cleanup)
cls.work = Path(cls.temp.name)
cls.wheels = []
for name, flags in (("direct", ["--wheel"]), ("sdist", [])):
output = cls.work / name
run(sys.executable, "-m", "build", "--no-isolation", *flags,
"--outdir", str(output), cwd=ROOT)
cls.wheels.append(next(output.glob("*.whl")))

@classmethod
def environment(cls, name):
root = cls.work / name
venv.EnvBuilder(with_pip=True).create(root)
scripts = root / ("Scripts" if os.name == "nt" else "bin")
return scripts / ("python.exe" if os.name == "nt" else "python")

def check_script(self, python, code):
# A successful exit alone cannot detect sys.exit(0) during an import.
output = run(str(python), "-c", code + '\nprint("dependency-check-complete")',
cwd=self.work)
self.assertEqual(output, "dependency-check-complete\n")

def test_default_dependency_install(self):
for wheel in self.wheels:
with self.subTest(wheel=wheel.parent.name):
python = self.environment(wheel.parent.name + "-installed")
run(str(python), "-m", "pip", "install", str(wheel), cwd=self.work)
run(str(python), "-m", "pip", "check", cwd=self.work)
self.check_script(python, "\n".join((
"import importlib.util, pathlib, sys",
"for name in ('capstone', 'keystone', 'qrcode'):",
" assert importlib.util.find_spec(name) is None, name",
"from edlclient.Library import cryptutils",
"from edlclient.Library.Modules import oneplus, oneplus_param",
"from edlclient.Library.utils import patchtools",
"module = pathlib.Path(cryptutils.__file__).resolve()",
"assert pathlib.Path(sys.prefix).resolve() in module.parents, module",
"assert 'capstone' not in sys.modules and 'keystone' not in sys.modules",
"tools = patchtools()",
"for method, args in ((tools.assembler, ('nop',)),",
" (tools.disasm, (bytes.fromhex('1f2003d5'), 0x1000))):",
" try:",
" method(*args)",
" except ImportError as exc:",
" assert 'edlclient[asm]' in str(exc), str(exc)",
" else:",
" raise AssertionError('Missing optional dependency was not reported')",
)))

def test_optional_dependency_install(self):
for extra, missing, check in (
("asm", "qrcode", "\n".join((
"from capstone import Cs, CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN",
"from edlclient.Library.utils import patchtools",
"code = bytes.fromhex('1f2003d5')",
"disassembler = Cs(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN)",
"instructions = list(disassembler.disasm(code, 0x1000))",
"assert len(instructions) == 1 and instructions[0].mnemonic == 'nop'",
"tools = patchtools()",
"assert tools.assembler('nop') == code.hex()",
"assert tools.disasm(code, 0x1000) == ['nop\\t']",
))),
("qrcode", "capstone", "\n".join((
"from pathlib import Path",
"import qrcode",
"target = Path('optional-qr.png')",
"qrcode.make('op_eng://dependency-test').save(str(target))",
"assert target.read_bytes().startswith(bytes.fromhex('89504e470d0a1a0a'))",
))),
):
with self.subTest(extra=extra):
python = self.environment("extra-" + extra)
run(str(python), "-m", "pip", "install",
str(self.wheels[0]) + "[" + extra + "]", cwd=self.work)
run(str(python), "-m", "pip", "check", cwd=self.work)
self.check_script(python,
f"import importlib.util; assert importlib.util.find_spec('{missing}') is None")
if extra == "asm":
output = run(str(python), "-m", "edlclient.Library.asmtools",
"--asm", "arm64,arm", "--inp", "nop", cwd=self.work)
self.assertEqual(output.splitlines(),
["CPU: arm64, MODE: arm", "1f2003d5"])
self.check_script(python, check)

def test_legacy_requirements(self):
python = self.environment("legacy-source")
run(str(python), "-m", "pip", "install", "-r",
str(ROOT / "requirements.txt"), cwd=self.work)
run(str(python), "-m", "pip", "check", cwd=self.work)
self.check_script(python, "\n".join((
"import importlib.util",
"assert importlib.util.find_spec('edlclient') is None",
"from Cryptodome.Cipher import AES",
"from Exscript.protocols.telnetlib import Telnet",
"from passlib.hash import md5_crypt",
"import requests, serial, usb.core, capstone, keystone, qrcode",
"from PIL import Image",
)))

def test_generated_requirements(self):
run(sys.executable, str(ROOT / ".github/scripts/update_requirements.py"),
"--check", cwd=self.work)


if __name__ == "__main__":
unittest.main()
29 changes: 29 additions & 0 deletions .github/workflows/python-dependencies.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Python dependencies

on:
push:
branches: ["master"]
pull_request:
branches: ["master"]

permissions:
contents: read

jobs:
dependencies:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
python-version: ["3.10", "3.14"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install test dependencies
run: python -m pip install build setuptools wheel
- name: Check generated requirements
run: python .github/scripts/update_requirements.py --check
- name: Verify default and optional dependency installation
run: python -m unittest discover -s .github/tests -v
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
include README.md
include LICENSE
include fastpwn
include requirements.txt
include .github/scripts/update_requirements.py
recursive-include Loaders *
include edlclient/Windows/*.dll
include edlclient/Config/nvitems.xml
Expand Down
23 changes: 23 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,29 @@ git submodule update --init --recursive
pip3 install -r requirements.txt
```

### Python dependencies

Python 3.10 or newer is required. Install the package with `python -m pip install .`
to get the dependencies needed by all declared CLI commands, including modem tools
and OnePlus encryption support.

Additional features are available as extras:

| Extra | Install from a checkout | Features |
| --- | --- | --- |
| `asm` | `python -m pip install ".[asm]"` | Assembly and disassembly with Capstone and Keystone |
| `qrcode` | `python -m pip install ".[qrcode]"` | OnePlus factory QR codes saved as PNG images |

Install both with `python -m pip install ".[asm,qrcode]"`.
`python -m pip install -r requirements.txt` installs the base dependencies and both
extras for users running scripts from a checkout; it does not install the package.

`pyproject.toml` is the source of truth for dependencies. `requirements.txt` is a
generated compatibility file for existing installation scripts. After changing
dependencies, run `python .github/scripts/update_requirements.py` and `uv lock`. The generator
uses `tomli` on Python 3.10 (also installed by the `build` development tool).
CI checks the generated file with `python .github/scripts/update_requirements.py --check`.

### Linux (Debian/Ubuntu/Mint/etc):
```bash
# Debian/Ubuntu/Mint/etc
Expand Down
23 changes: 14 additions & 9 deletions edlclient/Library/utils.py
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,6 @@

import colorama

try:
from capstone import *
except ImportError:
print("Capstone library is missing (optional).")
try:
from keystone import *
except ImportError:
print("Keystone library is missing (optional).")


def is_windows():
if sys.platform == 'win32' or sys.platform == 'win64' or sys.platform == 'winnt':
Expand Down Expand Up @@ -628,6 +619,13 @@ def uart_valid_sc(self, sc):
return True

def disasm(self, code, size):
try:
from capstone import Cs, CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN
except ImportError as exc:
raise ImportError(
'Disassembly requires Capstone; install it with pip install "edlclient[asm]".'
) from exc

cs = Cs(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN)
instr = []
for i in cs.disasm(code, size):
Expand All @@ -636,6 +634,13 @@ def disasm(self, code, size):
return instr

def assembler(self, code):
try:
from keystone import Ks, KsError, KS_ARCH_ARM64, KS_MODE_LITTLE_ENDIAN
except ImportError as exc:
raise ImportError(
'Assembly requires Keystone; install it with pip install "edlclient[asm]".'
) from exc

ks = Ks(KS_ARCH_ARM64, KS_MODE_LITTLE_ENDIAN)
if self.bDebug:
try:
Expand Down
11 changes: 6 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,19 @@ classifiers = [
]
dependencies = [
"colorama>=0.4.6",
"docopt",
"docopt>=0.6.2",
"pyusb>=1.3.1",
"pyserial>=3.5",
"lxml>=6.0.2",
"pylzma",
"pycryptodome>=3.23.0",
"pycryptodomex>=3.23.0",
"Exscript>=2.6.32",
"paramiko>=4.0.0,<5",
"requests>=2.34.2",
"passlib",
]

[ project.optional-dependencies ]
asm = ["capstone", "keystone-engine"]
qrcode = ["qrcode[pil]>=8.2"]

[ project.urls ]
Homepage = "https://github.com/bkerler/edl"
"Bug Tracker" = "https://github.com/bkerler/edl/issues"
Expand Down
27 changes: 16 additions & 11 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
wheel
# Generated from pyproject.toml; do not edit this file directly.
# Regenerate with: python .github/scripts/update_requirements.py
# Includes all optional features for legacy source installations.

# Required by the CLI commands
colorama>=0.4.6
docopt>=0.6.2
pyusb>=1.3.1
pyserial>=3.5
docopt>=0.6.2
pycryptodome>=3.23.0
pycryptodomex
lxml>=6.0.2
colorama
capstone
keystone-engine
qrcode
pycryptodomex>=3.23.0
Exscript>=2.6.32
requests>=2.34.2
passlib
Exscript>=2.6.32
paramiko>=4.0.0,<5

# Optional feature: asm
capstone
keystone-engine

# Optional feature: qrcode
qrcode[pil]>=8.2
Loading
0