From 27827271a6c6207b2b6da4a7ce54f2567a7ea709 Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 12:23:48 +0100 Subject: [PATCH 1/8] feat: Added support for poetry groups within source_path --- package.py | 52 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/package.py b/package.py index 7e96d784..6ecd704b 100644 --- a/package.py +++ b/package.py @@ -693,7 +693,7 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("pip", runtime, requirements, prefix, tmp_dir) hash(requirements) - def poetry_install_step(path, prefix=None, required=False): + def poetry_install_step(path, poetry_groups=None, prefix=None, required=False): pyproject_file = path if os.path.isdir(path): pyproject_file = os.path.join(path, "pyproject.toml") @@ -703,7 +703,7 @@ def poetry_install_step(path, prefix=None, required=False): "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, prefix) + step("poetry", runtime, path, poetry_groups, prefix) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -807,6 +807,7 @@ def commands_step(path, commands): prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") poetry_install = claim.get("poetry_install") + poetry_groups = claim.get("poetry_groups") npm_requirements = claim.get("npm_package_json") runtime = claim.get("runtime", query.runtime) @@ -828,7 +829,7 @@ def commands_step(path, commands): if poetry_install and runtime.startswith("python"): if path: - poetry_install_step(path, prefix, required=True) + poetry_install_step(path, prefix=prefix, poetry_groups=poetry_groups, required=True) if npm_requirements and runtime.startswith("nodejs"): if isinstance(npm_requirements, bool) and path: @@ -898,8 +899,9 @@ def execute(self, build_plan, zip_stream, query): # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == "poetry": - runtime, path, prefix = action[1:] - with install_poetry_dependencies(query, path) as rd: + runtime, path, poetry_groups, prefix, = action[1:] + log.info("Poetry Groups: %s", poetry_groups) + with install_poetry_dependencies(query, path, poetry_groups) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) @@ -956,7 +958,7 @@ def execute(self, build_plan, zip_stream, query): @staticmethod def _zip_write_with_filter( - zip_stream, path_filter, source_path, prefix, timestamp=None + zip_stream, path_filter, source_path, prefix, timestamp=None ): for path in path_filter.filter(source_path, prefix): if os.path.isdir(source_path): @@ -1094,7 +1096,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_poetry_dependencies(query, path): +def install_poetry_dependencies(query, path, poetry_groups): # TODO: # 1. Emit files instead of temp_dir @@ -1183,6 +1185,32 @@ def copy_file_to_target(file, temp_dir): # NOTE: poetry must be available in the build environment, which is the case with lambci/lambda:build-python* docker images but not public.ecr.aws/sam/build-python* docker images # FIXME: poetry install does not currently allow to specify the target directory so we export the # requirements then install them with "pip --no-deps" to avoid using pip dependency resolver + if poetry_groups is not None: + group_args = [] + for group in poetry_groups: + group_args.append("--with") + group_args.append(group) + + poetry_export = [ + poetry_exec, + "export", + "--format", + "requirements.txt", + "--output", + "requirements.txt", + "--with-credentials", + ] + group_args + else: + poetry_export = [ + poetry_exec, + "export", + "--format", + "requirements.txt", + "--output", + "requirements.txt", + "--with-credentials", + ] + poetry_commands = [ [ poetry_exec, @@ -1198,15 +1226,7 @@ def copy_file_to_target(file, temp_dir): "virtualenvs.in-project", "true", ], - [ - poetry_exec, - "export", - "--format", - "requirements.txt", - "--output", - "requirements.txt", - "--with-credentials", - ], + poetry_export, [ python_exec, "-m", From 41c7cf6627fae170f70d1e7de424ad411a1a0853 Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 12:25:59 +0100 Subject: [PATCH 2/8] Added documentation for poetry_groups --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d28a8b57..298aeb8a 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,7 @@ source_path = [ - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. - `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed. +- `poetry_groups` - A list of poetry dependency groups to install can be left empty to default to default dependencies - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. From 09ba34e930403df51994716a55f7a59f26551f3e Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 13:42:11 +0100 Subject: [PATCH 3/8] fixed lint issue --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 298aeb8a..0ad67b76 100644 --- a/README.md +++ b/README.md @@ -459,7 +459,7 @@ source_path = [ - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. - `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed. -- `poetry_groups` - A list of poetry dependency groups to install can be left empty to default to default dependencies +- `poetry_groups` - A list of poetry dependency groups to install can be left empty to default to default dependencies - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. From c6dd1f9e4946c7260760cf33853405be0c1731b1 Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 13:58:52 +0100 Subject: [PATCH 4/8] fixed lint issue --- package.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/package.py b/package.py index 6ecd704b..028c970a 100644 --- a/package.py +++ b/package.py @@ -829,7 +829,12 @@ def commands_step(path, commands): if poetry_install and runtime.startswith("python"): if path: - poetry_install_step(path, prefix=prefix, poetry_groups=poetry_groups, required=True) + poetry_install_step( + path, + prefix=prefix, + poetry_groups=poetry_groups, + required=True, + ) if npm_requirements and runtime.startswith("nodejs"): if isinstance(npm_requirements, bool) and path: @@ -899,7 +904,12 @@ def execute(self, build_plan, zip_stream, query): # XXX: timestamp=0 - what actually do with it? zs.write_dirs(rd, prefix=prefix, timestamp=0) elif cmd == "poetry": - runtime, path, poetry_groups, prefix, = action[1:] + ( + runtime, + path, + poetry_groups, + prefix, + ) = action[1:] log.info("Poetry Groups: %s", poetry_groups) with install_poetry_dependencies(query, path, poetry_groups) as rd: if rd: @@ -958,7 +968,7 @@ def execute(self, build_plan, zip_stream, query): @staticmethod def _zip_write_with_filter( - zip_stream, path_filter, source_path, prefix, timestamp=None + zip_stream, path_filter, source_path, prefix, timestamp=None ): for path in path_filter.filter(source_path, prefix): if os.path.isdir(source_path): From 3fad7aee4503793a0645d075f58dd6533be6fd21 Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 14:49:26 +0100 Subject: [PATCH 5/8] Updated argument to accept additional poetry arguments --- package.py | 49 +++++++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/package.py b/package.py index 028c970a..10537030 100644 --- a/package.py +++ b/package.py @@ -693,7 +693,7 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("pip", runtime, requirements, prefix, tmp_dir) hash(requirements) - def poetry_install_step(path, poetry_groups=None, prefix=None, required=False): + def poetry_install_step(path, poetry_args=[], prefix=None, required=False): pyproject_file = path if os.path.isdir(path): pyproject_file = os.path.join(path, "pyproject.toml") @@ -703,7 +703,7 @@ def poetry_install_step(path, poetry_groups=None, prefix=None, required=False): "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, poetry_groups, prefix) + step("poetry", runtime, path, poetry_args, prefix) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -807,7 +807,7 @@ def commands_step(path, commands): prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") poetry_install = claim.get("poetry_install") - poetry_groups = claim.get("poetry_groups") + additional_poetry_arg = claim.get("additional_poetry_args", []) npm_requirements = claim.get("npm_package_json") runtime = claim.get("runtime", query.runtime) @@ -832,7 +832,7 @@ def commands_step(path, commands): poetry_install_step( path, prefix=prefix, - poetry_groups=poetry_groups, + poetry_args=additional_poetry_arg, required=True, ) @@ -907,11 +907,11 @@ def execute(self, build_plan, zip_stream, query): ( runtime, path, - poetry_groups, + poetry_args, prefix, ) = action[1:] - log.info("Poetry Groups: %s", poetry_groups) - with install_poetry_dependencies(query, path, poetry_groups) as rd: + log.info("Poetry arguments: %s", poetry_args) + with install_poetry_dependencies(query, path, poetry_args) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) @@ -1106,7 +1106,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_poetry_dependencies(query, path, poetry_groups): +def install_poetry_dependencies(query, path, poetry_args): # TODO: # 1. Emit files instead of temp_dir @@ -1195,31 +1195,16 @@ def copy_file_to_target(file, temp_dir): # NOTE: poetry must be available in the build environment, which is the case with lambci/lambda:build-python* docker images but not public.ecr.aws/sam/build-python* docker images # FIXME: poetry install does not currently allow to specify the target directory so we export the # requirements then install them with "pip --no-deps" to avoid using pip dependency resolver - if poetry_groups is not None: - group_args = [] - for group in poetry_groups: - group_args.append("--with") - group_args.append(group) - poetry_export = [ - poetry_exec, - "export", - "--format", - "requirements.txt", - "--output", - "requirements.txt", - "--with-credentials", - ] + group_args - else: - poetry_export = [ - poetry_exec, - "export", - "--format", - "requirements.txt", - "--output", - "requirements.txt", - "--with-credentials", - ] + poetry_export = [ + poetry_exec, + "export", + "--format", + "requirements.txt", + "--output", + "requirements.txt", + "--with-credentials", + ] + poetry_args poetry_commands = [ [ From c32a87c5da03380d6cf97b089a2d7c247e6a46b4 Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 16:38:59 +0100 Subject: [PATCH 6/8] Updated naming conversion for poetry additional arguments and updated docs (readme) --- README.md | 2 +- package.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0ad67b76..3414617e 100644 --- a/README.md +++ b/README.md @@ -459,7 +459,7 @@ source_path = [ - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. - `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed. -- `poetry_groups` - A list of poetry dependency groups to install can be left empty to default to default dependencies +- `poetry_extra_args` - A list of additional poetry arguments to add to the poetry export command - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. diff --git a/package.py b/package.py index 10537030..3562fa38 100644 --- a/package.py +++ b/package.py @@ -693,7 +693,7 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("pip", runtime, requirements, prefix, tmp_dir) hash(requirements) - def poetry_install_step(path, poetry_args=[], prefix=None, required=False): + def poetry_install_step(path, poetry_extra_args=[], prefix=None, required=False): pyproject_file = path if os.path.isdir(path): pyproject_file = os.path.join(path, "pyproject.toml") @@ -703,7 +703,7 @@ def poetry_install_step(path, poetry_args=[], prefix=None, required=False): "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, poetry_args, prefix) + step("poetry", runtime, path, poetry_extra_args, prefix) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -807,7 +807,7 @@ def commands_step(path, commands): prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") poetry_install = claim.get("poetry_install") - additional_poetry_arg = claim.get("additional_poetry_args", []) + poetry_extra_args = claim.get("poetry_extra_args", []) npm_requirements = claim.get("npm_package_json") runtime = claim.get("runtime", query.runtime) @@ -832,7 +832,7 @@ def commands_step(path, commands): poetry_install_step( path, prefix=prefix, - poetry_args=additional_poetry_arg, + poetry_extra_args=poetry_extra_args, required=True, ) @@ -907,11 +907,11 @@ def execute(self, build_plan, zip_stream, query): ( runtime, path, - poetry_args, + poetry_extra_args, prefix, ) = action[1:] - log.info("Poetry arguments: %s", poetry_args) - with install_poetry_dependencies(query, path, poetry_args) as rd: + log.info("poetry_extra_args: %s", poetry_extra_args) + with install_poetry_dependencies(query, path, poetry_extra_args) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) @@ -1106,7 +1106,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_poetry_dependencies(query, path, poetry_args): +def install_poetry_dependencies(query, path, poetry_extra_args): # TODO: # 1. Emit files instead of temp_dir @@ -1204,7 +1204,7 @@ def copy_file_to_target(file, temp_dir): "--output", "requirements.txt", "--with-credentials", - ] + poetry_args + ] + poetry_extra_args poetry_commands = [ [ From b5ac9f4802ace80e82bb242f170ce7a85cd2e49e Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 16:41:02 +0100 Subject: [PATCH 7/8] fixed linting issue --- package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.py b/package.py index 3562fa38..b01bf645 100644 --- a/package.py +++ b/package.py @@ -693,7 +693,9 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("pip", runtime, requirements, prefix, tmp_dir) hash(requirements) - def poetry_install_step(path, poetry_extra_args=[], prefix=None, required=False): + def poetry_install_step( + path, poetry_extra_args=[], prefix=None, required=False + ): pyproject_file = path if os.path.isdir(path): pyproject_file = os.path.join(path, "pyproject.toml") From 78f8ab13e1e913922af790d774e55a349e6f2eff Mon Sep 17 00:00:00 2001 From: Morgan Chorlton Date: Tue, 11 Jun 2024 16:56:16 +0100 Subject: [PATCH 8/8] renamed poetry_extra_args to poetry_export_extra_args --- README.md | 2 +- package.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3414617e..7fdbe539 100644 --- a/README.md +++ b/README.md @@ -459,7 +459,7 @@ source_path = [ - `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed. - `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds. - `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed. -- `poetry_extra_args` - A list of additional poetry arguments to add to the poetry export command +- `poetry_export_extra_args` - A list of additional poetry arguments to add to the poetry export command - `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead. - `npm_tmp_dir` - Set the base directory to make the temporary directory for npm installs. Can be useful for Docker in Docker builds. - `prefix_in_zip` - If specified, will be used as a prefix inside zip-archive. By default, everything installs into the root of zip-archive. diff --git a/package.py b/package.py index b01bf645..1552e8e2 100644 --- a/package.py +++ b/package.py @@ -694,7 +694,7 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): hash(requirements) def poetry_install_step( - path, poetry_extra_args=[], prefix=None, required=False + path, poetry_export_extra_args=[], prefix=None, required=False ): pyproject_file = path if os.path.isdir(path): @@ -705,7 +705,7 @@ def poetry_install_step( "poetry configuration not found: {}".format(pyproject_file) ) else: - step("poetry", runtime, path, poetry_extra_args, prefix) + step("poetry", runtime, path, poetry_export_extra_args, prefix) hash(pyproject_file) pyproject_path = os.path.dirname(pyproject_file) poetry_lock_file = os.path.join(pyproject_path, "poetry.lock") @@ -809,7 +809,7 @@ def commands_step(path, commands): prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") poetry_install = claim.get("poetry_install") - poetry_extra_args = claim.get("poetry_extra_args", []) + poetry_export_extra_args = claim.get("poetry_export_extra_args", []) npm_requirements = claim.get("npm_package_json") runtime = claim.get("runtime", query.runtime) @@ -834,7 +834,7 @@ def commands_step(path, commands): poetry_install_step( path, prefix=prefix, - poetry_extra_args=poetry_extra_args, + poetry_export_extra_args=poetry_export_extra_args, required=True, ) @@ -909,11 +909,13 @@ def execute(self, build_plan, zip_stream, query): ( runtime, path, - poetry_extra_args, + poetry_export_extra_args, prefix, ) = action[1:] - log.info("poetry_extra_args: %s", poetry_extra_args) - with install_poetry_dependencies(query, path, poetry_extra_args) as rd: + log.info("poetry_export_extra_args: %s", poetry_export_extra_args) + with install_poetry_dependencies( + query, path, poetry_export_extra_args + ) as rd: if rd: if pf: self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) @@ -1108,7 +1110,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir): @contextmanager -def install_poetry_dependencies(query, path, poetry_extra_args): +def install_poetry_dependencies(query, path, poetry_export_extra_args): # TODO: # 1. Emit files instead of temp_dir @@ -1206,7 +1208,7 @@ def copy_file_to_target(file, temp_dir): "--output", "requirements.txt", "--with-credentials", - ] + poetry_extra_args + ] + poetry_export_extra_args poetry_commands = [ [