From e64592dcae01f2e1aa02995ba50a96c2a1358e4e Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Fri, 3 Feb 2023 16:35:12 -0700 Subject: [PATCH 1/7] 8.6.0 --- lib/bibliothecary/parsers/pypi.rb | 24 +++++++++++++++++++++--- lib/bibliothecary/version.rb | 2 +- spec/parsers/pypi_spec.rb | 31 +++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/lib/bibliothecary/parsers/pypi.rb b/lib/bibliothecary/parsers/pypi.rb index f8c2a5af..6702fc11 100644 --- a/lib/bibliothecary/parsers/pypi.rb +++ b/lib/bibliothecary/parsers/pypi.rb @@ -56,7 +56,7 @@ def self.mapping }, match_filename("pyproject.toml") => { kind: 'manifest', - parser: :parse_poetry + parser: :parse_pyproject }, match_filename("poetry.lock") => { kind: 'lockfile', @@ -90,9 +90,27 @@ def self.parse_pipfile(file_contents, options: {}) map_dependencies(manifest['packages'], 'runtime') + map_dependencies(manifest['dev-packages'], 'develop') end + def self.parse_pyproject(file_contents, options: {}) + deps = [] + + file_contents = Tomlrb.parse(file_contents) + + # Parse poetry [tool.poetry] deps + poetry_manifest = file_contents.fetch('tool', {}).fetch('poetry', {}) + deps += map_dependencies(poetry_manifest['dependencies'], 'runtime') + deps += map_dependencies(poetry_manifest['dev-dependencies'], 'develop') + + # Parse PEP621 [project] deps + pep621_manifest = file_contents.fetch('project', {}) + deps += map_dependencies(pep621_manifest['dependencies'], 'runtime') + + deps + end + + # TODO: this was deprecated in 8.5.2. Remove this in any major version bump >= 9.* def self.parse_poetry(file_contents, options: {}) - manifest = Tomlrb.parse(file_contents).fetch('tool', {}).fetch('poetry', {}) - map_dependencies(manifest['dependencies'], 'runtime') + map_dependencies(manifest['dev-dependencies'], 'develop') + puts "Warning: parse_poetry() is deprecated, use parse_pyproject() instead." + parse_pyproject(file_contents, options) end def self.parse_conda(file_contents, options: {}) diff --git a/lib/bibliothecary/version.rb b/lib/bibliothecary/version.rb index 9a7f8dd5..be55d630 100644 --- a/lib/bibliothecary/version.rb +++ b/lib/bibliothecary/version.rb @@ -1,3 +1,3 @@ module Bibliothecary - VERSION = "8.5.1" + VERSION = "8.6.0" end diff --git a/spec/parsers/pypi_spec.rb b/spec/parsers/pypi_spec.rb index c7e122eb..cfef6f05 100644 --- a/spec/parsers/pypi_spec.rb +++ b/spec/parsers/pypi_spec.rb @@ -309,6 +309,37 @@ }) end + # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata + it 'handles pyproject.toml with pep621-style deps' do + source = <<~FILE +[project] +name = "a_pep621_project" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "black", + "isort", + "click == 8.1.3", + "pytest == 7.2.1", + "python-gitlab == 3.12.0", +] + FILE + + expect(described_class.analyse_contents('pyproject.toml', source)).to eq({ + platform: "pypi", + path: "pyproject.toml", + dependencies: [ + {name: "black", requirement: "*", type: "runtime"}, + {name: "isort", requirement: "*", type: "runtime"}, + {name: "click == 8.1.3", requirement: "*", type: "runtime"}, + {name: "pytest == 7.2.1", requirement: "*", type: "runtime"}, + {name: "python-gitlab == 3.12.0", requirement: "*", type: "runtime"} + ], + kind: 'manifest', + success: true + }) + end + it 'parses dependencies from Poetry.lock' do expect(described_class.analyse_contents('poetry.lock', load_fixture('poetry.lock'))).to eq({ platform: "pypi", From c70de856d02b16c617cf2a5c6e2d12916ee6f975 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Fri, 3 Feb 2023 17:04:34 -0700 Subject: [PATCH 2/7] Parse names correctly, as libraries.io does --- lib/bibliothecary/parsers/pypi.rb | 15 ++++++++++++++- spec/parsers/pypi_spec.rb | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/bibliothecary/parsers/pypi.rb b/lib/bibliothecary/parsers/pypi.rb index 6702fc11..46ce9f8a 100644 --- a/lib/bibliothecary/parsers/pypi.rb +++ b/lib/bibliothecary/parsers/pypi.rb @@ -14,6 +14,9 @@ class Pypi MANIFEST_REGEXP = /.*require[^\/]*(\/)?[^\/]*\.(txt|pip|in)$/ PIP_COMPILE_REGEXP = /.*require.*$/ + # Adapted from https://peps.python.org/pep-0508/#names + PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])/i + def self.mapping { match_filenames('requirements-dev.txt', 'requirements/dev.txt', @@ -102,7 +105,8 @@ def self.parse_pyproject(file_contents, options: {}) # Parse PEP621 [project] deps pep621_manifest = file_contents.fetch('project', {}) - deps += map_dependencies(pep621_manifest['dependencies'], 'runtime') + pep621_deps = pep621_manifest.fetch('dependencies', []).map { |d| parse_pep_508_dep_spec(d) } + deps += map_dependencies(pep621_deps, 'runtime') deps end @@ -270,6 +274,15 @@ def self.pip_compile?(file_contents) # parsing after we match. false end + + # Simply parses out the name of a PEP 508 Dependency specification: https://peps.python.org/pep-0508/ + # Leaves the rest as-is with any leading semicolons or spaces stripped + def self.parse_pep_508_dep_spec(dep) + name, requirement = dep.split(PEP_508_NAME_REGEX, 2).last(2).map(&:strip) + requirement = requirement.sub(/^[\s;]*/, "") + requirement = "*" if requirement == "" + return name, requirement + end end end end diff --git a/spec/parsers/pypi_spec.rb b/spec/parsers/pypi_spec.rb index cfef6f05..f82fdeba 100644 --- a/spec/parsers/pypi_spec.rb +++ b/spec/parsers/pypi_spec.rb @@ -331,9 +331,9 @@ dependencies: [ {name: "black", requirement: "*", type: "runtime"}, {name: "isort", requirement: "*", type: "runtime"}, - {name: "click == 8.1.3", requirement: "*", type: "runtime"}, - {name: "pytest == 7.2.1", requirement: "*", type: "runtime"}, - {name: "python-gitlab == 3.12.0", requirement: "*", type: "runtime"} + {name: "click", requirement: "== 8.1.3", type: "runtime"}, + {name: "pytest", requirement: "== 7.2.1", type: "runtime"}, + {name: "python-gitlab", requirement: "== 3.12.0", type: "runtime"} ], kind: 'manifest', success: true From b0f97afa8d5a1ad7cc4acad36129e0c5bf084c18 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Fri, 3 Feb 2023 17:23:02 -0700 Subject: [PATCH 3/7] Add some more dep requirements examples --- spec/parsers/pypi_spec.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/parsers/pypi_spec.rb b/spec/parsers/pypi_spec.rb index f82fdeba..5dec1253 100644 --- a/spec/parsers/pypi_spec.rb +++ b/spec/parsers/pypi_spec.rb @@ -319,10 +319,11 @@ dependencies = [ "black", "isort", - "click == 8.1.3", "pytest == 7.2.1", "python-gitlab == 3.12.0", -] + "Click~=8.1.0", + "marshmallow-dataclass[union]~=8.5.6", +] FILE expect(described_class.analyse_contents('pyproject.toml', source)).to eq({ @@ -331,9 +332,10 @@ dependencies: [ {name: "black", requirement: "*", type: "runtime"}, {name: "isort", requirement: "*", type: "runtime"}, - {name: "click", requirement: "== 8.1.3", type: "runtime"}, {name: "pytest", requirement: "== 7.2.1", type: "runtime"}, - {name: "python-gitlab", requirement: "== 3.12.0", type: "runtime"} + {name: "python-gitlab", requirement: "== 3.12.0", type: "runtime"}, + {name: "Click", requirement: "~=8.1.0", type: "runtime"}, + {name: "marshmallow-dataclass", requirement: "[union]~=8.5.6", type: "runtime"} ], kind: 'manifest', success: true From 827329ed02797cce3ff3fb14d7673f566082f5c0 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Fri, 3 Feb 2023 17:24:55 -0700 Subject: [PATCH 4/7] Cleanup --- spec/parsers/pypi_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/parsers/pypi_spec.rb b/spec/parsers/pypi_spec.rb index 5dec1253..17544ff6 100644 --- a/spec/parsers/pypi_spec.rb +++ b/spec/parsers/pypi_spec.rb @@ -315,7 +315,6 @@ [project] name = "a_pep621_project" version = "0.1.0" -requires-python = ">=3.10" dependencies = [ "black", "isort", From 315fce263f7b2180e6b78f9097d3cb6a6fa462f3 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 6 Feb 2023 09:44:28 -0700 Subject: [PATCH 5/7] Update comment --- lib/bibliothecary/parsers/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bibliothecary/parsers/pypi.rb b/lib/bibliothecary/parsers/pypi.rb index 46ce9f8a..1b7b250b 100644 --- a/lib/bibliothecary/parsers/pypi.rb +++ b/lib/bibliothecary/parsers/pypi.rb @@ -111,7 +111,7 @@ def self.parse_pyproject(file_contents, options: {}) deps end - # TODO: this was deprecated in 8.5.2. Remove this in any major version bump >= 9.* + # TODO: this was deprecated in 8.6.0. Remove this in any major version bump >= 9.* def self.parse_poetry(file_contents, options: {}) puts "Warning: parse_poetry() is deprecated, use parse_pyproject() instead." parse_pyproject(file_contents, options) From 91b517a1790997a9b6d1e0f2f9148501040a9879 Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 6 Feb 2023 10:18:00 -0700 Subject: [PATCH 6/7] Add a note about combining them --- lib/bibliothecary/parsers/pypi.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bibliothecary/parsers/pypi.rb b/lib/bibliothecary/parsers/pypi.rb index 1b7b250b..89939c94 100644 --- a/lib/bibliothecary/parsers/pypi.rb +++ b/lib/bibliothecary/parsers/pypi.rb @@ -108,6 +108,8 @@ def self.parse_pyproject(file_contents, options: {}) pep621_deps = pep621_manifest.fetch('dependencies', []).map { |d| parse_pep_508_dep_spec(d) } deps += map_dependencies(pep621_deps, 'runtime') + # We're combining both poetry+PEP621 deps instead of making them mutually exclusive, until we + # find a reason not to ingest them both. deps end From bcaffbc51b62a1fc4b55a749df110524b48f8b0a Mon Sep 17 00:00:00 2001 From: Tieg Zaharia Date: Mon, 6 Feb 2023 10:26:13 -0700 Subject: [PATCH 7/7] Ensure we uniq in pypi parser --- lib/bibliothecary/parsers/pypi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bibliothecary/parsers/pypi.rb b/lib/bibliothecary/parsers/pypi.rb index 89939c94..e7a61d60 100644 --- a/lib/bibliothecary/parsers/pypi.rb +++ b/lib/bibliothecary/parsers/pypi.rb @@ -110,7 +110,7 @@ def self.parse_pyproject(file_contents, options: {}) # We're combining both poetry+PEP621 deps instead of making them mutually exclusive, until we # find a reason not to ingest them both. - deps + deps.uniq end # TODO: this was deprecated in 8.6.0. Remove this in any major version bump >= 9.*