Skip to content

Commit

Permalink
Merge pull request #101 from adafruit/include-binary-files-in-package
Browse files Browse the repository at this point in the history 
Get package name from pyproject.toml, allow arbitrary files in packages
  • Loading branch information
@dhalbert
dhalbert authored Dec 3, 2023
2 parents 34c259d + b51d905 commit 988ed98
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 138 deletions.
283 changes: 145 additions & 138 deletions circuitpython_build_tools/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,48 @@
import subprocess
import tempfile

# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
# and should be removed when the fixed version is incorporated in its respective bundle.

pyproject_py_modules_blacklist = set((
# adafruit bundle
"adafruit_colorsys",

# community bundle
"at24mac_eeprom",
"circuitpython_Candlesticks",
"CircuitPython_Color_Picker",
"CircuitPython_Equalizer",
"CircuitPython_Scales",
"circuitPython_Slider",
"circuitpython_uboxplot",
"P1AM",
"p1am_200_helpers",
))

if sys.version_info >= (3, 11):
from tomllib import loads as load_toml
else:
from tomli import loads as load_toml

def load_pyproject_toml(lib_path: pathlib.Path):
try:
return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8"))
except FileNotFoundError:
print(f"No pyproject.toml in {lib_path}")
return {}

def get_nested(doc, *args, default=None):
for a in args:
if doc is None: return default
try:
doc = doc[a]
except (KeyError, IndexError) as e:
return default
return doc

IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
GLOB_PATTERNS = ["*.py", "font5x8.bin"]
GLOB_PATTERNS = ["*.py", "*.bin"]
S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"

def version_string(path=None, *, valid_semver=False):
Expand Down Expand Up @@ -131,17 +171,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)

def _munge_to_temp(original_path, temp_file, library_version):
with open(original_path, "rb") as original_file:
with open(original_path, "r", encoding="utf-8") as original_file:
for line in original_file:
if original_path.endswith(".bin"):
# this is solely for adafruit_framebuf/examples/font5x8.bin
temp_file.write(line)
else:
line = line.decode("utf-8").strip("\n")
if line.startswith("__version__"):
line = line.replace("0.0.0-auto.0", library_version)
line = line.replace("0.0.0+auto.0", library_version)
temp_file.write(line.encode("utf-8") + b"\r\n")
line = line.strip("\n")
if line.startswith("__version__"):
line = line.replace("0.0.0-auto.0", library_version)
line = line.replace("0.0.0+auto.0", library_version)
print(line, file=temp_file)
temp_file.flush()

def get_package_info(library_path, package_folder_prefix):
Expand All @@ -154,61 +190,52 @@ def get_package_info(library_path, package_folder_prefix):
for pattern in GLOB_PATTERNS:
glob_search.extend(list(lib_path.rglob(pattern)))

package_info["is_package"] = False
for file in glob_search:
if file.parts[parent_idx] != "examples":
if len(file.parts) > parent_idx + 1:
for prefix in package_folder_prefix:
if file.parts[parent_idx].startswith(prefix):
package_info["is_package"] = True
if package_info["is_package"]:
package_files.append(file)
else:
if file.name in IGNORE_PY:
#print("Ignoring:", file.resolve())
continue
if file.parent == lib_path:
py_files.append(file)

if package_files:
package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
elif py_files:
package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
else:
package_info["module_name"] = None

try:
package_info["version"] = version_string(library_path, valid_semver=True)
except ValueError as e:
package_info["version"] = version_string(library_path)

return package_info

def library(library_path, output_directory, package_folder_prefix,
mpy_cross=None, example_bundle=False):
py_files = []
package_files = []
example_files = []
total_size = 512

lib_path = pathlib.Path(library_path)
parent_idx = len(lib_path.parts)
glob_search = []
for pattern in GLOB_PATTERNS:
glob_search.extend(list(lib_path.rglob(pattern)))
pyproject_toml = load_pyproject_toml(lib_path)
py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[])
packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[])

blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist]

if blacklisted:
print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml")
py_modules = packages = ()

example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*")
if sub_path.is_file()]

if packages and py_modules:
raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")

elif packages:
if len(packages) > 1:
raise ValueError("Only a single package is supported")
package_name = packages[0]
#print(f"Using package name from pyproject.toml: {package_name}")
package_info["is_package"] = True
package_info["module_name"] = package_name
package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*")
if sub_path.is_file()]

elif py_modules:
if len(py_modules) > 1:
raise ValueError("Only a single module is supported")
py_module = py_modules[0]
#print(f"Using module name from pyproject.toml: {py_module}")
package_name = py_module
package_info["is_package"] = False
package_info["module_name"] = py_module
py_files = [lib_path / f"{py_module}.py"]

for file in glob_search:
if file.parts[parent_idx] == "examples":
example_files.append(file)
else:
if not example_bundle:
is_package = False
else:
print(f"{lib_path}: Using legacy autodetection")
package_info["is_package"] = False
for file in glob_search:
if file.parts[parent_idx] != "examples":
if len(file.parts) > parent_idx + 1:
for prefix in package_folder_prefix:
if file.parts[parent_idx].startswith(prefix):
is_package = True

if is_package:
package_info["is_package"] = True
if package_info["is_package"]:
package_files.append(file)
else:
if file.name in IGNORE_PY:
Expand All @@ -217,91 +244,78 @@ def library(library_path, output_directory, package_folder_prefix,
if file.parent == lib_path:
py_files.append(file)

if package_files:
package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
elif py_files:
package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
else:
package_info["module_name"] = None

if len(py_files) > 1:
raise ValueError("Multiple top level py files not allowed. Please put "
"them in a package or combine them into a single file.")

if package_files:
module_name = package_files[0].relative_to(library_path).parent.name
elif py_files:
module_name = py_files[0].relative_to(library_path).name[:-3]
else:
module_name = None
package_info["package_files"] = package_files
package_info["py_files"] = py_files
package_info["example_files"] = example_files

try:
package_info["version"] = version_string(library_path, valid_semver=True)
except ValueError as e:
print(library_path + " has version that doesn't follow SemVer (semver.org)")
print(e)
package_info["version"] = version_string(library_path)

return package_info

def library(library_path, output_directory, package_folder_prefix,
mpy_cross=None, example_bundle=False):
lib_path = pathlib.Path(library_path)
package_info = get_package_info(library_path, package_folder_prefix)
py_package_files = package_info["package_files"] + package_info["py_files"]
example_files = package_info["example_files"]
module_name = package_info["module_name"]

for fn in example_files:
base_dir = os.path.join(output_directory.replace("/lib", "/"),
fn.relative_to(library_path).parent)
if not os.path.isdir(base_dir):
os.makedirs(base_dir)
total_size += 512

for fn in package_files:
for fn in py_package_files:
base_dir = os.path.join(output_directory,
fn.relative_to(library_path).parent)
if not os.path.isdir(base_dir):
os.makedirs(base_dir)
total_size += 512

new_extension = ".py"
if mpy_cross:
new_extension = ".mpy"
library_version = package_info['version']

try:
library_version = version_string(library_path, valid_semver=True)
except ValueError as e:
print(library_path + " has version that doesn't follow SemVer (semver.org)")
print(e)
library_version = version_string(library_path)

for filename in py_files:
full_path = os.path.join(library_path, filename)
output_file = os.path.join(
output_directory,
filename.relative_to(library_path).with_suffix(new_extension)
)
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
_munge_to_temp(full_path, temp_file, library_version)
temp_filename = temp_file.name
# Windows: close the temp file before it can be read or copied by name
if mpy_cross:
mpy_success = subprocess.call([
mpy_cross,
"-o", output_file,
"-s", str(filename.relative_to(library_path)),
temp_filename
])
if mpy_success != 0:
raise RuntimeError("mpy-cross failed on", full_path)
else:
shutil.copyfile(temp_filename, output_file)
os.remove(temp_filename)

for filename in package_files:
full_path = os.path.join(library_path, filename)
output_file = ""
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
_munge_to_temp(full_path, temp_file, library_version)
temp_filename = temp_file.name
# Windows: close the temp file before it can be read or copied by name
if not mpy_cross or os.stat(full_path).st_size == 0:
output_file = os.path.join(output_directory,
filename.relative_to(library_path))
shutil.copyfile(temp_filename, output_file)
else:
output_file = os.path.join(
output_directory,
filename.relative_to(library_path).with_suffix(new_extension)
)

mpy_success = subprocess.call([
mpy_cross,
"-o", output_file,
"-s", str(filename.relative_to(library_path)),
temp_filename
])
if mpy_success != 0:
raise RuntimeError("mpy-cross failed on", full_path)
os.remove(temp_filename)
if not example_bundle:
for filename in py_package_files:
full_path = os.path.join(library_path, filename)
output_file = output_directory / filename.relative_to(library_path)
if filename.suffix == ".py":
with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
temp_file_name = temp_file.name
try:
_munge_to_temp(full_path, temp_file, library_version)
temp_file.close()
if mpy_cross and os.stat(temp_file.name).st_size != 0:
output_file = output_file.with_suffix(".mpy")
mpy_success = subprocess.call([
mpy_cross,
"-o", output_file,
"-s", str(filename.relative_to(library_path)),
temp_file.name
])
if mpy_success != 0:
raise RuntimeError("mpy-cross failed on", full_path)
else:
shutil.copyfile(full_path, output_file)
finally:
os.remove(temp_file_name)
else:
shutil.copyfile(full_path, output_file)

requirements_files = lib_path.glob("requirements.txt*")
requirements_files = [f for f in requirements_files if f.stat().st_size > 0]
Expand All @@ -314,11 +328,9 @@ def library(library_path, output_directory, package_folder_prefix,
requirements_dir = pathlib.Path(output_directory).parent / "requirements"
if not os.path.isdir(requirements_dir):
os.makedirs(requirements_dir, exist_ok=True)
total_size += 512
requirements_subdir = f"{requirements_dir}/{module_name}"
if not os.path.isdir(requirements_subdir):
os.makedirs(requirements_subdir, exist_ok=True)
total_size += 512
for filename in requirements_files:
full_path = os.path.join(library_path, filename)
output_file = os.path.join(requirements_subdir, filename.name)
Expand All @@ -328,9 +340,4 @@ def library(library_path, output_directory, package_folder_prefix,
full_path = os.path.join(library_path, filename)
output_file = os.path.join(output_directory.replace("/lib", "/"),
filename.relative_to(library_path))
temp_filename = ""
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
_munge_to_temp(full_path, temp_file, library_version)
temp_filename = temp_file.name
shutil.copyfile(temp_filename, output_file)
os.remove(temp_filename)
shutil.copyfile(full_path, output_file)
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ Click
requests
semver
wheel
tomli; python_version < "3.11"

0 comments on commit 988ed98

Please sign in to comment.