pyproject.toml: support PEP621 projects too (#568)

* Parse PEP508 names correctly, as libraries.io does

* Add some more dep requirements examples

* Ensure we uniq in pypi parser

lib/bibliothecary/parsers/pypi.rb

@@ -14,6 +14,9 @@ class Pypi
       MANIFEST_REGEXP = /.*require[^\/]*(\/)?[^\/]*\.(txt|pip|in)$/
       PIP_COMPILE_REGEXP = /.*require.*$/
 
+      # Adapted from https://peps.python.org/pep-0508/#names
+      PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])/i
+
       def self.mapping
         {
           match_filenames('requirements-dev.txt', 'requirements/dev.txt',
@@ -56,7 +59,7 @@ def self.mapping
           },
           match_filename("pyproject.toml") => {
             kind: 'manifest',
-            parser: :parse_poetry
+            parser: :parse_pyproject
           },
           match_filename("poetry.lock") => {
             kind: 'lockfile',
@@ -90,9 +93,30 @@ def self.parse_pipfile(file_contents, options: {})
         map_dependencies(manifest['packages'], 'runtime') + map_dependencies(manifest['dev-packages'], 'develop')
       end
 
+      def self.parse_pyproject(file_contents, options: {})
+        deps = []  
+
+        file_contents = Tomlrb.parse(file_contents)
+
+        # Parse poetry [tool.poetry] deps
+        poetry_manifest = file_contents.fetch('tool', {}).fetch('poetry', {})
+        deps += map_dependencies(poetry_manifest['dependencies'], 'runtime')
+        deps += map_dependencies(poetry_manifest['dev-dependencies'], 'develop')
+
+        # Parse PEP621 [project] deps
+        pep621_manifest = file_contents.fetch('project', {})
+        pep621_deps = pep621_manifest.fetch('dependencies', []).map { |d| parse_pep_508_dep_spec(d) }
+        deps += map_dependencies(pep621_deps, 'runtime')
+
+        # We're combining both poetry+PEP621 deps instead of making them mutually exclusive, until we
+        # find a reason not to ingest them both.
+        deps.uniq
+      end
+
+      # TODO: this was deprecated in 8.6.0. Remove this in any major version bump >= 9.*
       def self.parse_poetry(file_contents, options: {})
-        manifest = Tomlrb.parse(file_contents).fetch('tool', {}).fetch('poetry', {})
-        map_dependencies(manifest['dependencies'], 'runtime') + map_dependencies(manifest['dev-dependencies'], 'develop')
+        puts "Warning: parse_poetry() is deprecated, use parse_pyproject() instead."
+        parse_pyproject(file_contents, options)
       end
 
       def self.parse_conda(file_contents, options: {})
@@ -252,6 +276,15 @@ def self.pip_compile?(file_contents)
         # parsing after we match.
         false
       end
+
+      # Simply parses out the name of a PEP 508 Dependency specification: https://peps.python.org/pep-0508/
+      # Leaves the rest as-is with any leading semicolons or spaces stripped
+      def self.parse_pep_508_dep_spec(dep)
+        name, requirement = dep.split(PEP_508_NAME_REGEX, 2).last(2).map(&:strip)
+        requirement = requirement.sub(/^[\s;]*/, "")
+        requirement = "*" if requirement == ""
+        return name, requirement
+      end
     end
   end
 end

lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.5.1"
+  VERSION = "8.6.0"
 end

spec/parsers/pypi_spec.rb

@@ -309,6 +309,38 @@
     })
   end
 
+  # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
+  it 'handles pyproject.toml with pep621-style deps' do
+    source = <<~FILE
+[project]
+name = "a_pep621_project"
+version = "0.1.0"
+dependencies = [
+    "black",
+    "isort",
+    "pytest == 7.2.1",
+    "python-gitlab == 3.12.0",
+    "Click~=8.1.0",
+    "marshmallow-dataclass[union]~=8.5.6",
+]
+    FILE
+
+    expect(described_class.analyse_contents('pyproject.toml', source)).to eq({
+      platform: "pypi",
+      path: "pyproject.toml",
+      dependencies: [
+        {name: "black", requirement: "*", type: "runtime"}, 
+        {name: "isort", requirement: "*", type: "runtime"}, 
+        {name: "pytest", requirement: "== 7.2.1", type: "runtime"}, 
+        {name: "python-gitlab", requirement: "== 3.12.0", type: "runtime"},
+        {name: "Click", requirement: "~=8.1.0", type: "runtime"},
+        {name: "marshmallow-dataclass", requirement: "[union]~=8.5.6", type: "runtime"}
+      ],
+      kind: 'manifest',
+      success: true
+    })
+  end
+
   it 'parses dependencies from Poetry.lock' do
     expect(described_class.analyse_contents('poetry.lock', load_fixture('poetry.lock'))).to eq({
       platform: "pypi",