From d8719e4d044fb68f2f9367e941622f851c8879e0 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Wed, 29 Nov 2023 17:40:31 -0600 Subject: [PATCH 01/28] Get package name from settings.toml, allow arbitrary files in packages This is more dependable, and when we know the package name we can glob inside it to get all files such as bin or ttf files. This will allow e.g., 5x8.bin & ov5640_autofocus.bin within bundles. the behavior of bundlefly and circup when encountering .bin files needs to be checked. Tested by building modified pycamera bundle and the autofocus.bin file appears in the generated zip files: ``` pycamera-py-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 pycamera-8.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 pycamera-9.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096 ``` There's at least one library in the bundle that has incorrect metadata and that leads to an error: https://github.com/adafruit/Adafruit_CircuitPython_Colorsys/pull/29 --- circuitpython_build_tools/build.py | 260 ++++++++++++++--------------- requirements.txt | 1 + 2 files changed, 122 insertions(+), 139 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 06bcfad..21f7849 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -36,8 +36,29 @@ import subprocess import tempfile +if sys.version_info >= (3, 11): + from tomllib import loads as load_toml +else: + from tomli import loads as load_toml + +def load_settings_toml(lib_path: pathlib.Path): + try: + return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) + except FileNotFoundError: + print(f"No settings.toml in {lib_path}") + return {} + +def get_nested(doc, *args, default=None): + for a in args: + if doc is None: return default + try: + doc = doc[a] + except (KeyError, IndexError) as e: + return default + return doc + IGNORE_PY = ["setup.py", "conf.py", "__init__.py"] -GLOB_PATTERNS = ["*.py", "font5x8.bin"] +GLOB_PATTERNS = ["*.py", "*.bin"] S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross" def version_string(path=None, *, valid_semver=False): @@ -131,17 +152,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) def _munge_to_temp(original_path, temp_file, library_version): - with open(original_path, "rb") as original_file: + with open(original_path, "r", encoding="utf-8") as original_file: for line in original_file: - if original_path.endswith(".bin"): - # this is solely for adafruit_framebuf/examples/font5x8.bin - temp_file.write(line) - else: - line = line.decode("utf-8").strip("\n") - if line.startswith("__version__"): - line = line.replace("0.0.0-auto.0", library_version) - line = line.replace("0.0.0+auto.0", library_version) - temp_file.write(line.encode("utf-8") + b"\r\n") + line = line.strip("\n") + if line.startswith("__version__"): + line = line.replace("0.0.0-auto.0", library_version) + line = line.replace("0.0.0+auto.0", library_version) + print(line, file=temp_file) temp_file.flush() def get_package_info(library_path, package_folder_prefix): @@ -154,61 +171,46 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - package_info["is_package"] = False - for file in glob_search: - if file.parts[parent_idx] != "examples": - if len(file.parts) > parent_idx + 1: - for prefix in package_folder_prefix: - if file.parts[parent_idx].startswith(prefix): - package_info["is_package"] = True - if package_info["is_package"]: - package_files.append(file) - else: - if file.name in IGNORE_PY: - #print("Ignoring:", file.resolve()) - continue - if file.parent == lib_path: - py_files.append(file) - - if package_files: - package_info["module_name"] = package_files[0].relative_to(library_path).parent.name - elif py_files: - package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] - else: - package_info["module_name"] = None - - try: - package_info["version"] = version_string(library_path, valid_semver=True) - except ValueError as e: - package_info["version"] = version_string(library_path) - - return package_info - -def library(library_path, output_directory, package_folder_prefix, - mpy_cross=None, example_bundle=False): - py_files = [] - package_files = [] - example_files = [] - total_size = 512 - - lib_path = pathlib.Path(library_path) - parent_idx = len(lib_path.parts) - glob_search = [] - for pattern in GLOB_PATTERNS: - glob_search.extend(list(lib_path.rglob(pattern))) - - for file in glob_search: - if file.parts[parent_idx] == "examples": - example_files.append(file) - else: - if not example_bundle: - is_package = False + settings_toml = load_settings_toml(lib_path) + py_modules = get_nested(settings_toml, "tool", "setuptools", "py-modules", default=[]) + packages = get_nested(settings_toml, "tool", "setuptools", "packages", default=[]) + + example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") + if sub_path.is_file()] + + if packages and py_modules: + raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages") + + elif packages: + if len(packages) > 1: + raise ValueError("Only a single package is supported") + package_name = packages[0] + print(f"Using package name from settings.toml: {package_name}") + package_info["is_package"] = True + package_info["module_name"] = package_name + package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") + if sub_path.is_file()] + + elif py_modules: + if len(py_modules) > 1: + raise ValueError("Only a single module is supported") + print("Using module name from settings.toml") + py_module = py_modules[0] + package_name = py_module + package_info["is_package"] = False + package_info["module_name"] = py_module + py_files = [lib_path / f"{py_module}.py"] + + if not packages and not py_modules: + print("Using legacy autodetection") + package_info["is_package"] = False + for file in glob_search: + if file.parts[parent_idx] != "examples": if len(file.parts) > parent_idx + 1: for prefix in package_folder_prefix: if file.parts[parent_idx].startswith(prefix): - is_package = True - - if is_package: + package_info["is_package"] = True + if package_info["is_package"]: package_files.append(file) else: if file.name in IGNORE_PY: @@ -217,91 +219,78 @@ def library(library_path, output_directory, package_folder_prefix, if file.parent == lib_path: py_files.append(file) + if package_files: + package_info["module_name"] = package_files[0].relative_to(library_path).parent.name + elif py_files: + package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] + else: + package_info["module_name"] = None + if len(py_files) > 1: raise ValueError("Multiple top level py files not allowed. Please put " "them in a package or combine them into a single file.") - if package_files: - module_name = package_files[0].relative_to(library_path).parent.name - elif py_files: - module_name = py_files[0].relative_to(library_path).name[:-3] - else: - module_name = None + package_info["package_files"] = package_files + package_info["py_files"] = py_files + package_info["example_files"] = example_files + + try: + package_info["version"] = version_string(library_path, valid_semver=True) + except ValueError as e: + print(library_path + " has version that doesn't follow SemVer (semver.org)") + print(e) + package_info["version"] = version_string(library_path) + + return package_info + +def library(library_path, output_directory, package_folder_prefix, + mpy_cross=None, example_bundle=False): + lib_path = pathlib.Path(library_path) + package_info = get_package_info(library_path, package_folder_prefix) + py_package_files = package_info["package_files"] + package_info["py_files"] + example_files = package_info["example_files"] + module_name = package_info["module_name"] for fn in example_files: base_dir = os.path.join(output_directory.replace("/lib", "/"), fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - for fn in package_files: + for fn in py_package_files: base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - - new_extension = ".py" - if mpy_cross: - new_extension = ".mpy" - - try: - library_version = version_string(library_path, valid_semver=True) - except ValueError as e: - print(library_path + " has version that doesn't follow SemVer (semver.org)") - print(e) - library_version = version_string(library_path) - for filename in py_files: - full_path = os.path.join(library_path, filename) - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if mpy_cross: - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - else: - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) + library_version = package_info['version'] - for filename in package_files: - full_path = os.path.join(library_path, filename) - output_file = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if not mpy_cross or os.stat(full_path).st_size == 0: - output_file = os.path.join(output_directory, - filename.relative_to(library_path)) - shutil.copyfile(temp_filename, output_file) - else: - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - os.remove(temp_filename) + if not example_bundle: + for filename in py_package_files: + full_path = os.path.join(library_path, filename) + output_file = output_directory / filename.relative_to(library_path) + if filename.suffix == ".py": + with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file: + temp_file_name = temp_file.name + try: + _munge_to_temp(full_path, temp_file, library_version) + temp_file.close() + if mpy_cross: + output_file = output_file.with_suffix(".mpy") + mpy_success = subprocess.call([ + mpy_cross, + "-o", output_file, + "-s", str(filename.relative_to(library_path)), + temp_file.name + ]) + if mpy_success != 0: + raise RuntimeError("mpy-cross failed on", full_path) + else: + shutil.copyfile(full_path, output_file) + finally: + os.remove(temp_file_name) + else: + shutil.copyfile(full_path, output_file) requirements_files = lib_path.glob("requirements.txt*") requirements_files = [f for f in requirements_files if f.stat().st_size > 0] @@ -314,11 +303,9 @@ def library(library_path, output_directory, package_folder_prefix, requirements_dir = pathlib.Path(output_directory).parent / "requirements" if not os.path.isdir(requirements_dir): os.makedirs(requirements_dir, exist_ok=True) - total_size += 512 requirements_subdir = f"{requirements_dir}/{module_name}" if not os.path.isdir(requirements_subdir): os.makedirs(requirements_subdir, exist_ok=True) - total_size += 512 for filename in requirements_files: full_path = os.path.join(library_path, filename) output_file = os.path.join(requirements_subdir, filename.name) @@ -328,9 +315,4 @@ def library(library_path, output_directory, package_folder_prefix, full_path = os.path.join(library_path, filename) output_file = os.path.join(output_directory.replace("/lib", "/"), filename.relative_to(library_path)) - temp_filename = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) + shutil.copyfile(full_path, output_file) diff --git a/requirements.txt b/requirements.txt index 861b8da..8a3514c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ Click requests semver wheel +tomli; python_version < "3.11" From 0c58704d658b2c741d47a042c0ddeb3419cf0f2e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 30 Nov 2023 09:08:41 -0600 Subject: [PATCH 02/28] Restore behavior of shipping 0-byte py files as .py, not .mpy (it's smaller on disk) --- circuitpython_build_tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 21f7849..faecfd5 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -275,7 +275,7 @@ def library(library_path, output_directory, package_folder_prefix, try: _munge_to_temp(full_path, temp_file, library_version) temp_file.close() - if mpy_cross: + if mpy_cross and os.stat(temp_file.name).st_size != 0: output_file = output_file.with_suffix(".mpy") mpy_success = subprocess.call([ mpy_cross, From ca779868d854e5aa8b0814ef6d080d8d4bdbb5bb Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 30 Nov 2023 10:34:45 -0600 Subject: [PATCH 03/28] pyproject is the name of the file --- circuitpython_build_tools/build.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index faecfd5..c80712e 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -41,11 +41,11 @@ else: from tomli import loads as load_toml -def load_settings_toml(lib_path: pathlib.Path): +def load_pyproject_toml(lib_path: pathlib.Path): try: return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) except FileNotFoundError: - print(f"No settings.toml in {lib_path}") + print(f"No pyproject.toml in {lib_path}") return {} def get_nested(doc, *args, default=None): @@ -171,9 +171,9 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - settings_toml = load_settings_toml(lib_path) - py_modules = get_nested(settings_toml, "tool", "setuptools", "py-modules", default=[]) - packages = get_nested(settings_toml, "tool", "setuptools", "packages", default=[]) + pyproject_toml = load_pyproject.toml(lib_path) + py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) + packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()] @@ -185,7 +185,7 @@ def get_package_info(library_path, package_folder_prefix): if len(packages) > 1: raise ValueError("Only a single package is supported") package_name = packages[0] - print(f"Using package name from settings.toml: {package_name}") + print(f"Using package name from pyproject.toml: {package_name}") package_info["is_package"] = True package_info["module_name"] = package_name package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") @@ -194,14 +194,14 @@ def get_package_info(library_path, package_folder_prefix): elif py_modules: if len(py_modules) > 1: raise ValueError("Only a single module is supported") - print("Using module name from settings.toml") + print("Using module name from pyproject.toml") py_module = py_modules[0] package_name = py_module package_info["is_package"] = False package_info["module_name"] = py_module py_files = [lib_path / f"{py_module}.py"] - if not packages and not py_modules: + else: print("Using legacy autodetection") package_info["is_package"] = False for file in glob_search: From 373b25644121b382b26117616b5c462407a2bda5 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 2 Dec 2023 15:22:33 -0600 Subject: [PATCH 04/28] typo --- circuitpython_build_tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index c80712e..336e50a 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -171,7 +171,7 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - pyproject_toml = load_pyproject.toml(lib_path) + pyproject_toml = load_pyproject_toml(lib_path) py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) From b51d905d9d7ad5d04ff6b63e191708ec75a02e56 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 2 Dec 2023 15:23:00 -0600 Subject: [PATCH 05/28] blacklist should let this pass CI --- circuitpython_build_tools/build.py | 31 +++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 336e50a..f884a1c 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -36,6 +36,25 @@ import subprocess import tempfile +# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! +# and should be removed when the fixed version is incorporated in its respective bundle. + +pyproject_py_modules_blacklist = set(( + # adafruit bundle + "adafruit_colorsys", + + # community bundle + "at24mac_eeprom", + "circuitpython_Candlesticks", + "CircuitPython_Color_Picker", + "CircuitPython_Equalizer", + "CircuitPython_Scales", + "circuitPython_Slider", + "circuitpython_uboxplot", + "P1AM", + "p1am_200_helpers", +)) + if sys.version_info >= (3, 11): from tomllib import loads as load_toml else: @@ -175,6 +194,12 @@ def get_package_info(library_path, package_folder_prefix): py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) + blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] + + if blacklisted: + print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml") + py_modules = packages = () + example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()] @@ -185,7 +210,7 @@ def get_package_info(library_path, package_folder_prefix): if len(packages) > 1: raise ValueError("Only a single package is supported") package_name = packages[0] - print(f"Using package name from pyproject.toml: {package_name}") + #print(f"Using package name from pyproject.toml: {package_name}") package_info["is_package"] = True package_info["module_name"] = package_name package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") @@ -194,15 +219,15 @@ def get_package_info(library_path, package_folder_prefix): elif py_modules: if len(py_modules) > 1: raise ValueError("Only a single module is supported") - print("Using module name from pyproject.toml") py_module = py_modules[0] + #print(f"Using module name from pyproject.toml: {py_module}") package_name = py_module package_info["is_package"] = False package_info["module_name"] = py_module py_files = [lib_path / f"{py_module}.py"] else: - print("Using legacy autodetection") + print(f"{lib_path}: Using legacy autodetection") package_info["is_package"] = False for file in glob_search: if file.parts[parent_idx] != "examples": From 0032a8038724d089c028a7af3dc84ed1f7c0f75b Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Tue, 5 Dec 2023 08:02:34 -0600 Subject: [PATCH 06/28] Change terminology --- circuitpython_build_tools/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index f884a1c..7c0fa25 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -39,7 +39,7 @@ # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! # and should be removed when the fixed version is incorporated in its respective bundle. -pyproject_py_modules_blacklist = set(( +pyproject_py_modules_blocklist = set(( # adafruit bundle "adafruit_colorsys", @@ -194,10 +194,10 @@ def get_package_info(library_path, package_folder_prefix): py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) - blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] + blocklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] - if blacklisted: - print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml") + if blocklisted: + print(f"{lib_path}/settings.toml:1: {blocklisted[0]} blacklisted: not using metadata from pyproject.toml") py_modules = packages = () example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") From f9a8aa3b1a4c28928c1ba7a03d1a16e5dec78f44 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Tue, 5 Dec 2023 09:55:11 -0500 Subject: [PATCH 07/28] Change remaining blacklist to blocklist --- circuitpython_build_tools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 7c0fa25..0fc240f 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -194,10 +194,10 @@ def get_package_info(library_path, package_folder_prefix): py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) - blocklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] + blocklisted = [name for name in py_modules if name in pyproject_py_modules_blocklist] if blocklisted: - print(f"{lib_path}/settings.toml:1: {blocklisted[0]} blacklisted: not using metadata from pyproject.toml") + print(f"{lib_path}/settings.toml:1: {blocklisted[0]} blocklisted: not using metadata from pyproject.toml") py_modules = packages = () example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") From 951b47c1327a68912bd12f6e06900eacdf0bd3fc Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 14 Dec 2023 10:58:14 -0600 Subject: [PATCH 08/28] require tomli at install time --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 28a6803..9b33317 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, zip_safe=False, python_requires='>=3.7', - install_requires=['Click', 'requests', 'semver'], + install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'], entry_points=''' [console_scripts] circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles From 7cd8379d01c2b43431e8f3ea66ee857298027c80 Mon Sep 17 00:00:00 2001 From: Brendan <2bndy5@gmail.com> Date: Fri, 15 Dec 2023 21:31:39 -0800 Subject: [PATCH 09/28] addresses #103 --- .../scripts/build_bundles.py | 21 ++++++++++++------- requirements.txt | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 596b7dc..ffa49ea 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -26,7 +26,6 @@ import os import os.path import re -import shlex import shutil import subprocess import sys @@ -37,7 +36,13 @@ from circuitpython_build_tools import build from circuitpython_build_tools import target_versions -import pkg_resources +import importlib.resources as importlib_resources + +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + BLINKA_LIBRARIES = [ "adafruit-blinka", @@ -244,10 +249,10 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d libs = _find_libraries(os.path.abspath(library_location), library_depth) - pkg = pkg_resources.get_distribution("circuitpython-build-tools") - build_tools_version = "devel" - if pkg: - build_tools_version = pkg.version + try: + build_tools_version = importlib_metadata.version("circuitpython-build-tools") + except importlib_metadata.PackageNotFoundError: + build_tools_version = "devel" build_tools_fn = "z-build_tools_version-{}.ignore".format( build_tools_version) @@ -269,8 +274,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d for version in target_versions.VERSIONS: # Use prebuilt mpy-cross on Travis, otherwise build our own. if "TRAVIS" in os.environ: - mpy_cross = pkg_resources.resource_filename( - target_versions.__name__, "data/mpy-cross-" + version["name"]) + mpy_cross = importlib_resources.files( + target_versions.__name__) / "data/mpy-cross-" + version["name"] else: mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) build.mpy_cross(mpy_cross, version["tag"]) diff --git a/requirements.txt b/requirements.txt index 8a3514c..988d40c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests semver wheel tomli; python_version < "3.11" +importlib_metadata; python_version < "3.8" From 4d4844451caa25bd691267c08840d3a12564cd11 Mon Sep 17 00:00:00 2001 From: Brendan <2bndy5@gmail.com> Date: Fri, 15 Dec 2023 21:02:52 -0800 Subject: [PATCH 10/28] upgrade github actions to latest major --- .github/workflows/build.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d43b404..7088297 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,14 +16,14 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - name: Set up Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.7 - name: Versions run: | python3 --version - name: Checkout Current Repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install requirements run: | sudo apt-get update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e2aa54..0e54be7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,9 +13,9 @@ jobs: upload-pypi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: '3.7' - name: Install dependencies From 1e64df5969dfb50222886b63337bd70e099851ca Mon Sep 17 00:00:00 2001 From: Brendan <2bndy5@gmail.com> Date: Sat, 16 Dec 2023 20:02:27 -0800 Subject: [PATCH 11/28] remove travis CI specific code --- circuitpython_build_tools/scripts/build_bundles.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index ffa49ea..6fa472a 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -36,8 +36,6 @@ from circuitpython_build_tools import build from circuitpython_build_tools import target_versions -import importlib.resources as importlib_resources - if sys.version_info < (3, 8): import importlib_metadata else: @@ -272,13 +270,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d if "mpy" not in ignore: os.makedirs("build_deps", exist_ok=True) for version in target_versions.VERSIONS: - # Use prebuilt mpy-cross on Travis, otherwise build our own. - if "TRAVIS" in os.environ: - mpy_cross = importlib_resources.files( - target_versions.__name__) / "data/mpy-cross-" + version["name"] - else: - mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) - build.mpy_cross(mpy_cross, version["tag"]) + mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) + build.mpy_cross(mpy_cross, version["tag"]) zip_filename = os.path.join(output_directory, filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( TAG=version["name"], From 87ba4b679b82351f60fdb34788fc738ee4354d55 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 17 Dec 2023 08:04:05 -0600 Subject: [PATCH 12/28] we now recommend github actions as the CI system of choice --- README.md | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index a8dc21d..5b85c43 100644 --- a/README.md +++ b/README.md @@ -4,40 +4,20 @@ This repo contains build scripts used to build the [Adafruit CircuitPython bundle](https://github.com/adafruit/Adafruit_CircuitPython_Bundle), [CircuitPython Community bundle](https://github.com/adafruit/CircuitPython_Community_Bundle) -and individual library release zips. Its focused on Travis CI support but will also work locally +and individual library release zips. Its focused on Github Actions support but will also work locally when a gcc compiler is present. -The pip package includes mpy-crosses that run on Travis. When building locally, the scripts will +The scripts will either fetch a pre-built mpy-cross from s3 or automatically clone the [CircuitPython repo](https://github.com/adafruit/circuitpython) and attempt -to build mpy-crosses. You'll need some version of gcc for this to work. +to build mpy-cross. You'll need some version of gcc for this to work. ## Setting up libraries -These build tools are intended for use with [Travis CI](https://travis-ci.org) -to automatically build .mpy files and zip them up for CircuitPython when a new -tagged release is created. To add support to a repo you need to: - - 1. Use the [CircuitPython cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) to generate .travis.yml. - 2. For adafruit repositories, simply give the CircuitPythonLibrarians team - write access to the repo and Adabot will do the rest. - - Otherwise, go to travis-ci.org and find the repository (it needs to be - setup to access your github account, and your github account needs access - to write to the repo). Flip the 'ON' switch on for Travis and the repo, - see the Travis docs for more details: https://docs.travis-ci.com/user/getting-started/ - 3. Get a GitHub 'personal access token' which has at least 'public_repo' or - 'repo' scope: https://help.github.com/articles/creating-an-access-token-for-command-line-use/ - Keep this token safe and secure! Anyone with the token will be able to - access and write to your GitHub repositories. Travis will use the token - to attach the .mpy files to the release. - 4. In the Travis CI settings for the repository that was enabled find the - environment variable editing page: https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings - Add an environment variable named GITHUB_TOKEN and set it to the value - of the GitHub personal access token above. Keep 'Display value in build - log' flipped off. - 5. That's it! Tag a release and Travis should go to work to add zipped .mpy files - to the release. It takes about a 2-3 minutes for a worker to spin up, - build mpy-cross, and add the binaries to the release. +These build tools automatically build .mpy files and zip them up for +CircuitPython when a new tagged release is created. To add support to a repo +you need to use the [CircuitPython +cookiecutter](https://github.com/adafruit/cookiecutter-adafruit-circuitpython) +to generate `.github/workflows/*.yml`. The bundle build will produce one zip file for every major CircuitPython release supported containing compatible mpy files and a zip with human readable py files. @@ -71,5 +51,5 @@ circuitpython-build-bundles --filename_prefix --library_loc ## Contributing Contributions are welcome! Please read our [Code of Conduct] -(https://github.com/adafruit/Adafruit_CircuitPython_adabot/blob/master/CODE_OF_CONDUCT.md) +(https://github.com/adafruit/Adafruit\_CircuitPython\_adabot/blob/master/CODE\_OF\_CONDUCT.md) before contributing to help this project stay welcoming. From e896099f0956fbec23cf135b23b655037d66dc14 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 17 Dec 2023 08:10:45 -0600 Subject: [PATCH 13/28] bump minimum python version to 3.10 .. this is what is used during the build of the adafruit bundle right now. --- .github/workflows/build.yml | 4 ++-- .github/workflows/release.yml | 2 +- requirements.txt | 1 - setup.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7088297..18d7da8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,10 +15,10 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.7 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.10 - name: Versions run: | python3 --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e54be7..5e81de8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/requirements.txt b/requirements.txt index 988d40c..8a3514c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ requests semver wheel tomli; python_version < "3.11" -importlib_metadata; python_version < "3.8" diff --git a/setup.py b/setup.py index 9b33317..0d2bbc7 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ 'circuitpython_build_tools.scripts'], package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, zip_safe=False, - python_requires='>=3.7', + python_requires='>=3.10', install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'], entry_points=''' [console_scripts] From 6a4079769f76c701500406e2f93a252a937c199e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 17 Dec 2023 08:12:48 -0600 Subject: [PATCH 14/28] string vs number --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18d7da8..3c657de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: "3.10" - name: Versions run: | python3 --version From 7dadcc70aa03271fbd8c15c9ed369981ca13e72d Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 17 Dec 2023 08:59:14 -0600 Subject: [PATCH 15/28] ensure tags are present this should fix the message during CI ``` .../setuptools_scm/git.py:163: UserWarning: "/home/runner/work/circuitpython-build-tools/circuitpython-build-tools" is shallow and may cause errors warnings.warn(f'"{wd.path}" is shallow and may cause errors') ``` This may have been introduced when upgrading action/checkout recently, or it might be pre-existing. --- .github/workflows/build.yml | 3 +++ .github/workflows/release.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3c657de..a301583 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,9 @@ jobs: python3 --version - name: Checkout Current Repo uses: actions/checkout@v4 + with: + filter: 'blob:none' + depth: 0 - name: Install requirements run: | sudo apt-get update diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e81de8..728ba8e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + filter: 'blob:none' + depth: 0 - name: Set up Python uses: actions/setup-python@v5 with: From 4f50d277dabb983edc3a5df91d3decb6adfec1af Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Tue, 19 Dec 2023 11:23:53 -0600 Subject: [PATCH 16/28] enable standard zip ("deflate") compression This saves ~50% on the main bundle and will save ~85% on the fonts bundle --- circuitpython_build_tools/scripts/build_bundles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 6fa472a..e85610e 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -205,7 +205,7 @@ def build_bundle(libs, bundle_version, output_filename, package_folder_prefix, print() print("Zipping") - with zipfile.ZipFile(output_filename, 'w') as bundle: + with zipfile.ZipFile(output_filename, 'w', compression=zipfile.ZIP_DEFLATED) as bundle: build_metadata = {"build-tools-version": build_tools_version} bundle.comment = json.dumps(build_metadata).encode("utf-8") if multiple_libs: From ff6d40dca90f3cffc14fed94add520cdb30ffee0 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Tue, 19 Dec 2023 13:15:09 -0600 Subject: [PATCH 17/28] Fix json output when a bundle does not use submodules Some bundles, such as circuitpython-fonts, don't use submodules to contain their individual libraries. Change build_bundle_json so that it works correctly if multiple libraries come from the same submodule. With this change, the generated metadata for circuitpython-fonts contains an entry for every font, not just for one font. --- .../scripts/build_bundles.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index e85610e..0962c5b 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -117,7 +117,7 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref """ Generate a JSON file of all the libraries in libs """ - packages = {} + packages = [] for library_path in libs: package = {} package_info = build.get_package_info(library_path, package_folder_prefix) @@ -130,18 +130,19 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref package["version"] = package_info["version"] package["path"] = "lib/" + package_info["module_name"] package["library_path"] = library_path - packages[module_name] = package + packages.append(package) library_submodules = {} - for id in packages: + for package in packages: library = {} - library["package"] = packages[id]["is_folder"] - library["pypi_name"] = packages[id]["pypi_name"] - library["version"] = packages[id]["version"] - library["repo"] = packages[id]["repo"] - library["path"] = packages[id]["path"] - library["dependencies"], library["external_dependencies"] = get_bundle_requirements(packages[id]["library_path"], packages) - library_submodules[packages[id]["module_name"]] = library + library["package"] = package["is_folder"] + library["pypi_name"] = package["pypi_name"] + library["version"] = package["version"] + library["repo"] = package["repo"] + library["path"] = package["path"] + library["dependencies"], library["external_dependencies"] = get_bundle_requirements(package["library_path"], packages) + library_submodules[package["module_name"]] = library + out_file = open(output_filename, "w") json.dump(library_submodules, out_file, sort_keys=True) out_file.close() From ea28feb31d02cd82991a8ef3e10a863cbda5c2c4 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 11 Jan 2024 08:58:56 -0600 Subject: [PATCH 18/28] Fix generation of dependencies[] array --- circuitpython_build_tools/scripts/build_bundles.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 0962c5b..ae3401c 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -117,7 +117,11 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref """ Generate a JSON file of all the libraries in libs """ - packages = [] + packages = {} + # TODO simplify this 2-step process + # It mostly exists so that get_bundle_requirements has a way to look up + # "pypi name to bundle name" via `package_list[pypi_name]["module_name"]` + # otherwise it's just shuffling info around for library_path in libs: package = {} package_info = build.get_package_info(library_path, package_folder_prefix) @@ -130,10 +134,10 @@ def build_bundle_json(libs, bundle_version, output_filename, package_folder_pref package["version"] = package_info["version"] package["path"] = "lib/" + package_info["module_name"] package["library_path"] = library_path - packages.append(package) + packages[module_name] = package library_submodules = {} - for package in packages: + for package in packages.values(): library = {} library["package"] = package["is_folder"] library["pypi_name"] = package["pypi_name"] From 90a2341e0454bc8dcf3272e20c4e880fa5343cd4 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Tue, 2 Jan 2024 10:25:45 -0600 Subject: [PATCH 19/28] Add an "--only" flag to quickly build just one kind of artifact Most often I find that I want to check just one kind of artifact (e.g., mpy files) and this is a quicker syntax than excluding the other 3 types with 3 --ignores. --- circuitpython_build_tools/scripts/build_bundles.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index ae3401c..3c31185 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -235,6 +235,7 @@ def _find_libraries(current_path, depth): subdirectories.extend(_find_libraries(path, depth - 1)) return subdirectories +all_modules = ["py", "mpy", "example", "json"] @click.command() @click.option('--filename_prefix', required=True, help="Filename prefix for the output zip files.") @click.option('--output_directory', default="bundles", help="Output location for the zip files.") @@ -242,8 +243,9 @@ def _find_libraries(current_path, depth): @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") @click.option('--package_folder_prefix', default="adafruit_", help="Prefix string used to determine package folders to bundle.") @click.option('--remote_name', default="origin", help="Git remote name to use during building") -@click.option('--ignore', "-i", multiple=True, type=click.Choice(["py", "mpy", "example", "json"]), help="Bundles to ignore building") -def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore): +@click.option('--ignore', "-i", multiple=True, type=click.Choice(all_modules), help="Bundles to ignore building") +@click.option('--only', "-o", multiple=True, type=click.Choice(all_modules), help="Bundles to build building") +def build_bundles(filename_prefix, output_directory, library_location, library_depth, package_folder_prefix, remote_name, ignore, only): os.makedirs(output_directory, exist_ok=True) package_folder_prefix = package_folder_prefix.split(", ") @@ -263,6 +265,11 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d with open(build_tools_fn, "w") as f: f.write(build_tools_version) + if ignore and only: + raise SystemExit("Only specify one of --ignore / --only") + if only: + ignore = set(all_modules) - set(only) + # Build raw source .py bundle if "py" not in ignore: zip_filename = os.path.join(output_directory, From 581f7c5380525a5870d146fc692d87ee6f6bc074 Mon Sep 17 00:00:00 2001 From: Justin Myers Date: Sun, 3 Mar 2024 13:55:05 -0800 Subject: [PATCH 20/28] Fix py build not using created temp file --- circuitpython_build_tools/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 0fc240f..1728247 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -311,7 +311,7 @@ def library(library_path, output_directory, package_folder_prefix, if mpy_success != 0: raise RuntimeError("mpy-cross failed on", full_path) else: - shutil.copyfile(full_path, output_file) + shutil.copyfile(temp_file_name, output_file) finally: os.remove(temp_file_name) else: From d1d1709f7f7388b2014c37bc4da902218c3a0214 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 8 Apr 2024 18:14:10 -0500 Subject: [PATCH 21/28] example subdirectories --- circuitpython_build_tools/build.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 1728247..dc35ade 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -276,12 +276,6 @@ def library(library_path, output_directory, package_folder_prefix, example_files = package_info["example_files"] module_name = package_info["module_name"] - for fn in example_files: - base_dir = os.path.join(output_directory.replace("/lib", "/"), - fn.relative_to(library_path).parent) - if not os.path.isdir(base_dir): - os.makedirs(base_dir) - for fn in py_package_files: base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent) @@ -338,6 +332,12 @@ def library(library_path, output_directory, package_folder_prefix, for filename in example_files: full_path = os.path.join(library_path, filename) + + relative_filename_parts = list(filename.relative_to(library_path).parts) + relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1]) + final_relative_filename = os.path.join(*relative_filename_parts) output_file = os.path.join(output_directory.replace("/lib", "/"), - filename.relative_to(library_path)) + final_relative_filename) + + os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True) shutil.copyfile(full_path, output_file) From c8a8513288a1716a4d3ce764ec734a9b4dcf7ebf Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 15 Jun 2024 10:46:58 -0500 Subject: [PATCH 22/28] mpy_cross: Cache it in the user cache path .. rather than in a build_deps folder, polluting the repo where it's used this also enables sharing the copy of mpy-cross across multiple repos. --- circuitpython_build_tools/scripts/build_bundles.py | 4 +++- requirements.txt | 1 + setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 3c31185..81aa815 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -32,6 +32,7 @@ import zipfile import click +import platformdirs from circuitpython_build_tools import build from circuitpython_build_tools import target_versions @@ -41,6 +42,7 @@ else: import importlib.metadata as importlib_metadata +mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) BLINKA_LIBRARIES = [ "adafruit-blinka", @@ -282,7 +284,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d if "mpy" not in ignore: os.makedirs("build_deps", exist_ok=True) for version in target_versions.VERSIONS: - mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt")) + mpy_cross = mpy_cross_path / ("mpy-cross-" + version["name"] + (".exe" * (os.name == "nt"))) build.mpy_cross(mpy_cross, version["tag"]) zip_filename = os.path.join(output_directory, filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( diff --git a/requirements.txt b/requirements.txt index 8a3514c..b11b4c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests semver wheel tomli; python_version < "3.11" +platformdirs diff --git a/setup.py b/setup.py index 0d2bbc7..9052130 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ package_data={'circuitpython_build_tools': ['data/mpy-cross-*']}, zip_safe=False, python_requires='>=3.10', - install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'], + install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'], entry_points=''' [console_scripts] circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles From 8fe3db14499e1f4a4c09b5ba6ce3ee69f2250120 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Jun 2024 11:19:19 -0500 Subject: [PATCH 23/28] Finish converting mpy-cross building to use cache directory --- circuitpython_build_tools/build.py | 68 ++++++++++++------- .../scripts/build_bundles.py | 7 +- .../scripts/build_mpy_cross.py | 15 ++-- 3 files changed, 56 insertions(+), 34 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 0fc240f..16ff279 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -24,10 +24,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +import functools +import multiprocessing import os import os.path import platform import pathlib +import re import requests import semver import shutil @@ -35,6 +38,23 @@ import sys import subprocess import tempfile +import platformdirs + +@functools.cache +def _git_version(): + version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace") + version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0) + return tuple(int(part) for part in version_str.split(".")) + +def git_filter_arg(): + clone_supports_filter = ( + False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0) + ) + + if clone_supports_filter: + return ["--filter=blob:none"] + else: + return [] # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! # and should be removed when the fixed version is incorporated in its respective bundle. @@ -60,6 +80,8 @@ else: from tomli import loads as load_toml +mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) + def load_pyproject_toml(lib_path: pathlib.Path): try: return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) @@ -106,9 +128,14 @@ def version_string(path=None, *, valid_semver=False): version = commitish return version -def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): +def mpy_cross(version, quiet=False): + circuitpython_tag = version["tag"] + name = version["name"] + ext = ".exe" * (os.name == "nt") + mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}" + if os.path.isfile(mpy_cross_filename): - return + return mpy_cross_filename # Try to pull from S3 uname = platform.uname() @@ -136,7 +163,7 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR) if not quiet: print(" FOUND") - return + return mpy_cross_filename except Exception as e: if not quiet: print(f" exception fetching from S3: {e}") @@ -149,26 +176,21 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): print(title) print("=" * len(title)) - os.makedirs("build_deps/", exist_ok=True) - if not os.path.isdir("build_deps/circuitpython"): - clone = subprocess.run("git clone https://github.com/adafruit/circuitpython.git build_deps/circuitpython", shell=True) - if clone.returncode != 0: - sys.exit(clone.returncode) - - current_dir = os.getcwd() - os.chdir("build_deps/circuitpython") - make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True) - os.chdir("tools") - make = subprocess.run("git submodule update --init .", shell=True) - os.chdir("../mpy-cross") - make = subprocess.run("make clean && make", shell=True) - os.chdir(current_dir) - - if make.returncode != 0: - print("Failed to build mpy-cross from source... bailing out") - sys.exit(make.returncode) - - shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) + build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}" + if not os.path.isdir(build_dir): + subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir]) + + subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir) + subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir) + subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross") + subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross") + + mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}" + if not os.path.exists(mpy_built): + mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}" + + shutil.copy(mpy_built, mpy_cross_filename) + return mpy_cross_filename def _munge_to_temp(original_path, temp_file, library_version): with open(original_path, "r", encoding="utf-8") as original_file: diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 81aa815..c7fad1e 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -32,7 +32,6 @@ import zipfile import click -import platformdirs from circuitpython_build_tools import build from circuitpython_build_tools import target_versions @@ -42,8 +41,6 @@ else: import importlib.metadata as importlib_metadata -mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True) - BLINKA_LIBRARIES = [ "adafruit-blinka", "adafruit-blinka-bleio", @@ -282,10 +279,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d # Build .mpy bundle(s) if "mpy" not in ignore: - os.makedirs("build_deps", exist_ok=True) for version in target_versions.VERSIONS: - mpy_cross = mpy_cross_path / ("mpy-cross-" + version["name"] + (".exe" * (os.name == "nt"))) - build.mpy_cross(mpy_cross, version["tag"]) + mpy_cross = build.mpy_cross(version) zip_filename = os.path.join(output_directory, filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format( TAG=version["name"], diff --git a/circuitpython_build_tools/scripts/build_mpy_cross.py b/circuitpython_build_tools/scripts/build_mpy_cross.py index 9abe0da..4b79aca 100644 --- a/circuitpython_build_tools/scripts/build_mpy_cross.py +++ b/circuitpython_build_tools/scripts/build_mpy_cross.py @@ -28,9 +28,14 @@ import os import sys +import click + +@click.command +@click.argument("versions") +def main(versions): + print(versions) + for version in [v for v in target_versions.VERSIONS if v['name'] in versions]: + print(f"{version['name']}: {build.mpy_cross(version)}") + if __name__ == "__main__": - output_directory = sys.argv[1] - os.makedirs(output_directory, exist_ok=True) - for version in target_versions.VERSIONS: - mpy_cross = output_directory + "/mpy-cross-" + version["name"] - build.mpy_cross(mpy_cross, version["tag"]) + main() From 9524f770b3edfc735b445fe528737538ffc67048 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Jun 2024 11:40:19 -0500 Subject: [PATCH 24/28] Add circuitpython-mpy-cross This wraps the process of building or downloading mpy-cross for a specified version, then running it with specified arguments. e.g., ``` $ circuitpython-mpy-cross --circuitpython-version 9.x -- --version CircuitPython 9.0.0-alpha.2 on 2023-10-27; mpy-cross emitting mpy v6.1 ``` --- .../scripts/circuitpython_mpy_cross.py | 21 +++++++++++++++++++ setup.py | 1 + 2 files changed, 22 insertions(+) create mode 100644 circuitpython_build_tools/scripts/circuitpython_mpy_cross.py diff --git a/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py new file mode 100644 index 0000000..d8e5cb2 --- /dev/null +++ b/circuitpython_build_tools/scripts/circuitpython_mpy_cross.py @@ -0,0 +1,21 @@ +import subprocess + +import click + +from ..target_versions import VERSIONS +from ..build import mpy_cross + +@click.command(context_settings={"ignore_unknown_options": True}) +@click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS])) +@click.option("--quiet/--no-quiet", "quiet", type=bool, default=True) +@click.argument("mpy-cross-args", nargs=-1, required=True) +def main(circuitpython_version, quiet, mpy_cross_args): + version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version] + mpy_cross_exe = str(mpy_cross(version_info, quiet)) + try: + subprocess.check_call([mpy_cross_exe, *mpy_cross_args]) + except subprocess.CalledProcessError as e: + raise SystemExit(e.returncode) + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 9052130..4358300 100644 --- a/setup.py +++ b/setup.py @@ -18,5 +18,6 @@ entry_points=''' [console_scripts] circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles + circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main ''' ) From b3542087a0ccfb29b638f1bf6a3874f2fcea862a Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 4 Jan 2025 19:42:32 -0600 Subject: [PATCH 25/28] remove entries for fixed projects --- circuitpython_build_tools/build.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 93055e9..ab28b72 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -60,18 +60,8 @@ def git_filter_arg(): # and should be removed when the fixed version is incorporated in its respective bundle. pyproject_py_modules_blocklist = set(( - # adafruit bundle - "adafruit_colorsys", - # community bundle "at24mac_eeprom", - "circuitpython_Candlesticks", - "CircuitPython_Color_Picker", - "CircuitPython_Equalizer", - "CircuitPython_Scales", - "circuitPython_Slider", - "circuitpython_uboxplot", - "P1AM", "p1am_200_helpers", )) From fd0d426d8e0f0bb887ba57b81faaa2b8aac6cf11 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sat, 8 Feb 2025 11:04:30 -0500 Subject: [PATCH 26/28] Drop building 8.x bundles --- circuitpython_build_tools/target_versions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/circuitpython_build_tools/target_versions.py b/circuitpython_build_tools/target_versions.py index e6afad0..197a653 100644 --- a/circuitpython_build_tools/target_versions.py +++ b/circuitpython_build_tools/target_versions.py @@ -25,6 +25,5 @@ # The tag specifies which version of CircuitPython to use for mpy-cross. # The name is used when constructing the zip file names. VERSIONS = [ - {"tag": "8.2.0", "name": "8.x"}, - {"tag": "9.0.0-alpha.2", "name": "9.x"}, + {"tag": "9.2.4", "name": "9.x"}, ] From 30705e57ce9a83329d51d1b9d8bf7e5e4f4b156a Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Fri, 4 Apr 2025 17:36:17 -0400 Subject: [PATCH 27/28] start building 10.x bundles --- circuitpython_build_tools/target_versions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/circuitpython_build_tools/target_versions.py b/circuitpython_build_tools/target_versions.py index 197a653..8bd2058 100644 --- a/circuitpython_build_tools/target_versions.py +++ b/circuitpython_build_tools/target_versions.py @@ -26,4 +26,5 @@ # The name is used when constructing the zip file names. VERSIONS = [ {"tag": "9.2.4", "name": "9.x"}, + {"tag": "10.0.0-alpha.2", "name": "10.x"}, ] From 553dcf78c94dc172758837b8699e3675314002e9 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 8 Apr 2025 09:19:35 -0500 Subject: [PATCH 28/28] use python 3.12 inside actions containers --- .github/workflows/build.yml | 4 ++-- .github/workflows/release.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a301583..8bb288c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,10 +15,10 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Versions run: | python3 --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 728ba8e..8605547 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy