From f49eff87ae9a4226584096498836de7d5015f6a2 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 28 May 2024 13:32:27 -0700 Subject: [PATCH] Add purlcli RTD details, incorporate latest feedback, fix failing tests #365 #249 Reference: https://github.com/nexB/purldb/issues/365 Reference: https://github.com/nexB/purldb/issues/249 Signed-off-by: John M. Horan --- purldb-toolkit/README.rst | 513 +++++++++++++++++- purldb-toolkit/src/purldb_toolkit/purlcli.py | 108 +++- .../data/purlcli/expected_urls_output.json | 232 ++------ .../purlcli/expected_urls_output_head.json | 108 +--- .../expected_urls_output_head_mock.json | 9 +- purldb-toolkit/tests/test_purlcli.py | 21 +- purldb-toolkit/tests/test_purlcli_live.py | 67 +-- 7 files changed, 685 insertions(+), 373 deletions(-) diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index 4ac94a98..3662cae3 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -1,12 +1,17 @@ purldb-toolkit ============== +.. contents:: :local: + :depth: 7 + + + purldb-toolkit is command line utility and library to use the PurlDB, its API and various related libraries. The ``purlcli`` command acts as a client to the PurlDB REST API end point(s) to expose PURL services. It serves both as a tool, as a library and as an example on how to use the services programmatically. - + Installation ------------ @@ -20,84 +25,541 @@ Use this command to get basic help:: $ purlcli --help Usage: purlcli [OPTIONS] COMMAND [ARGS]... - + Return information from a PURL. - + Options: --help Show this message and exit. - + Commands: metadata Given one or more PURLs, for each PURL, return a mapping of... urls Given one or more PURLs, for each PURL, return a list of all... - validate Check the syntax of one or more PURLs. + validate Check the syntax and upstream repo status of one or more PURLs. versions Given one or more PURLs, return a list of all known versions... And the following subcommands: -- Validate a PURL:: +'validate' -- validate a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: none $ purlcli validate --help Usage: purlcli validate [OPTIONS] - - Check the syntax of one or more PURLs. - + + Check the syntax and upstream repo status of one or more PURLs. + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write validation output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli validate --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "validate", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:nginx/nginx@0.8.9'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "valid": true, + "exists": true, + "message": "The provided Package URL is valid, and the package exists in the upstream repo." + }, + { + "purl": "pkg:nginx/nginx@0.8.9", + "valid": true, + "exists": null, + "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type." + } + ] + } + + +**Submit multiple PURLs using a .txt file:** + +.. code-block:: none + + purlcli validate --file --output + +*Sample input.txt:* + +.. code-block:: console + + pkg:npm/canonical-path@1.0.0 + pkg:nginx/nginx@0.8.9 + + +Notes +####### + +``validate`` calls the ``public.purldb.io/api/validate/`` endpoint. + + +---- + + +'versions' -- collect package versions for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: none -- Collect package versions for a PURL:: - $ purlcli versions --help Usage: purlcli versions [OPTIONS] - + Given one or more PURLs, return a list of all known versions for each PURL. - - Version information is not needed in submitted PURLs and if included will be - removed before processing. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write versions output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli versions --purl pkg:npm/canonical-path --purl pkg:nginx/nginx --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "versions", + "--purl": [ + "pkg:npm/canonical-path", + "pkg:nginx/nginx" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx' not supported with `versions` command" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@0.0.1", + "version": "0.0.1", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@0.0.2", + "version": "0.0.2", + "release_date": "2013-12-19" + }, + { + "purl": "pkg:npm/canonical-path@1.0.0", + "version": "1.0.0", + "release_date": "2018-10-24" + } + ] + } + -- Collect package metadata for a PURL:: +Notes +####### + +``versions`` calls ``versions()`` from `fetchcode/package_versions.py `__. + +Version information is not needed in submitted PURLs and, if included, will be removed before processing. + + +---- + + +'metadata' -- collect package metadata for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console $ purlcli metadata --help Usage: purlcli metadata [OPTIONS] - + Given one or more PURLs, for each PURL, return a mapping of metadata fetched from the fetchcode package.py info() function. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write meta output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. --help Show this message and exit. - -- Collect package URLs for a PURL:: +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli metadata --purl pkg:openssl/openssl@3.0.6 --purl pkg:nginx/nginx@0.8.9 --purl pkg:gnu/glibc@2.38 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": [ + "pkg:openssl/openssl@3.0.6", + "pkg:nginx/nginx@0.8.9", + "pkg:gnu/glibc@2.38" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'check_existence' is not supported for 'pkg:openssl/openssl@3.0.6'", + "'pkg:nginx/nginx@0.8.9' not supported with `metadata` command", + "'check_existence' is not supported for 'pkg:gnu/glibc@2.38'" + ] + } + ], + "packages": [ + { + "purl": "pkg:openssl/openssl@3.0.6", + "type": "openssl", + "namespace": null, + "name": "openssl", + "version": "3.0.6", + "qualifiers": {}, + "subpath": null, + "primary_language": "C", + "description": null, + "release_date": "2022-10-11T12:39:09", + "parties": [], + "keywords": [], + "homepage_url": "https://www.openssl.org", + "download_url": "https://github.com/openssl/openssl/archive/refs/tags/openssl-3.0.6.tar.gz", + "api_url": "https://api.github.com/repos/openssl/openssl", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/openssl/openssl/issues", + "code_view_url": "https://github.com/openssl/openssl", + "vcs_url": "git://github.com/openssl/openssl.git", + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:gnu/glibc@2.38", + "type": "gnu", + "namespace": null, + "name": "glibc", + "version": "2.38", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": "2023-07-31T17:34:00", + "parties": [], + "keywords": [], + "homepage_url": "https://ftp.gnu.org/pub/gnu/glibc/", + "download_url": "https://ftp.gnu.org/pub/gnu/glibc/glibc-2.38.tar.gz", + "api_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ] + } + + +Notes +####### + +``metadata`` calls ``info()`` from `fetchcode/package.py `__. + +The intended output for each PURL type supported by the ``metadata`` command is + +- an input PURL with a version: output the metadata for the input version +- an input PURL with no version: output a list of the metadata for all versions + +The output of the various PURL types currently supported in `fetchcode/package.py `__ varies from type to type at the moment -- the underlying functions will be updated as needed so that all produce the intended output for input PURLs with and without a version. + + +---- + + +'urls' -- collect package URLs for a PURL +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: console $ purlcli urls --help Usage: purlcli urls [OPTIONS] - + Given one or more PURLs, for each PURL, return a list of all known URLs fetched from the packageurl-python purl2url.py code. - + Options: --purl TEXT PackageURL or PURL. --output FILENAME Write urls output as JSON to FILE. [required] --file FILENAME Read a list of PURLs from a FILE, one per line. - --unique Return data only for unique PURLs. --head Validate each URL's existence with a head request. --help Show this message and exit. +Examples +######## + +**Submit multiple PURLs using the command line:** + +.. code-block:: none + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "inferred_urls": [ + "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz" + ], + "repository_download_url": null, + "repository_homepage_url": "https://www.npmjs.com/package/canonical-path/v/1.0.0" + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "inferred_urls": [ + "https://rubygems.org/gems/rails/versions/7.0.0", + "https://rubygems.org/downloads/rails-7.0.0.gem" + ], + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/rails/versions/7.0.0" + } + ] + } + + +**Include head and get requests:** + +``--head`` + +.. code-block:: none + + purlcli urls --purl pkg:npm/canonical-path@1.0.0 --purl pkg:nginx/nginx@0.8.9 --purl pkg:rubygems/rails@7.0.0 --output --head + +*Sample output:* + +.. code-block:: console + + { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:npm/canonical-path@1.0.0", + "pkg:nginx/nginx@0.8.9", + "pkg:rubygems/rails@7.0.0" + ], + "--file": null, + "--head": true, + "--output": "" + }, + "errors": [], + "warnings": [ + "'pkg:nginx/nginx@0.8.9' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/rails@7.0.0'" + ] + } + ], + "packages": [ + { + "purl": "pkg:npm/canonical-path@1.0.0", + "download_url": { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + }, + "inferred_urls": [ + { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "http://registry.npmjs.org/canonical-path/-/canonical-path-1.0.0.tgz", + "get_request_status_code": 200, + "head_request_status_code": 301 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://www.npmjs.com/package/canonical-path/v/1.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + }, + { + "purl": "pkg:rubygems/rails@7.0.0", + "download_url": { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + "inferred_urls": [ + { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + }, + { + "url": "https://rubygems.org/downloads/rails-7.0.0.gem", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + ], + "repository_download_url": { + "url": null, + "get_request_status_code": "N/A", + "head_request_status_code": "N/A" + }, + "repository_homepage_url": { + "url": "https://rubygems.org/gems/rails/versions/7.0.0", + "get_request_status_code": 200, + "head_request_status_code": 200 + } + } + ] + } + + +Notes +####### + +- None atm. + + +Testing +------- + +Run all purldb tests: + +.. code-block:: none + + make test + +Run all purlcli non-live tests (i.e., no live network calls): + +.. code-block:: none + + DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli.py + +Run all purlcli live tests (i.e., check actual API endpoints etc.) + +.. code-block:: none + + DJANGO_SETTINGS_MODULE=purldb_project.settings pytest -vvs purldb-toolkit/tests/test_purlcli_live.py --run_live_fetch + Funding ------- @@ -132,4 +594,3 @@ See https://creativecommons.org/licenses/by-sa/4.0/legalcode for the license tex See https://github.com/nexB/purldb for support or download. See https://aboutcode.org for more information about nexB OSS projects. - diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 917c1008..5997e2a5 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -101,7 +101,7 @@ def get_metadata_details(purls, output, file, command_name): metadata_collection = collect_metadata(purl) metadata_details["packages"].extend(metadata_collection) - print(f"\nmetadata_warnings = {metadata_warnings}") + # print(f"\nmetadata_warnings = {metadata_warnings}") metadata_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, @@ -349,26 +349,64 @@ def get_urls_details(purls, output, file, head, command_name): url_purl = PackageURL.from_string(purl) - url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} + url_detail["download_url"] = purl2url.get_download_url(purl) + if head: + url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} + url_detail["inferred_urls"] = [ - {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + inferred for inferred in purl2url.get_inferred_urls(purl) ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} - url_detail["repo_download_url_by_package_type"] = {"url": None} - if url_purl.version: - url_detail["repo_download_url_by_package_type"] = { - "url": purl2url.get_repo_download_url_by_package_type( - url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - ) + if head: + url_detail["inferred_urls"] = [ + {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + ] + + # url_detail["repo_download_url"] = purl2url.get_repo_download_url(purl) + url_detail["repository_download_url"] = purl2url.get_repo_download_url(purl) + if head: + # url_detail["repo_download_url"] = { + url_detail["repository_download_url"] = { + "url": purl2url.get_repo_download_url(purl) } - url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} + + # TODO delete this -- seems always same as the renamed repository_download_url + # package_type_url = None + # # url_detail["repo_download_url_by_package_type"] = {"url": None} + # url_detail["repo_download_url_by_package_type"] = package_type_url + # if head: + # url_detail["repo_download_url_by_package_type"] = {"url": package_type_url} + + # if url_purl.version: + # package_type_url = purl2url.get_repo_download_url_by_package_type( + # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + # ) + # url_detail["repo_download_url_by_package_type"] = package_type_url + # # url_detail["repo_download_url_by_package_type"] = { + # # "url": purl2url.get_repo_download_url_by_package_type( + # # url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + # # ) + # # } + # if head: + # url_detail["repo_download_url_by_package_type"] = { + # "url": package_type_url + # } + + # url_detail["repo_url"] = purl2url.get_repo_url(purl) + # if head: + # url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} + url_detail["repository_homepage_url"] = purl2url.get_repo_url(purl) + if head: + url_detail["repository_homepage_url"] = {"url": purl2url.get_repo_url(purl)} url_list = [ "download_url", # "inferred_urls" has to be handled separately because it has a nested list - "repo_download_url", - "repo_download_url_by_package_type", - "repo_url", + # "repo_download_url", + "repository_download_url", + # TODO delete this -- seems always same as the renamed repository_download_url + # "repo_download_url_by_package_type", + # "repo_url", + "repository_homepage_url", ] if head: for purlcli_url in url_list: @@ -543,7 +581,33 @@ def get_validate_details(purls, output, file, command_name): ]: validate_warnings[purl] = validated_purl_status if validated_purl_status: - validate_details["packages"].append(validate_purl(purl)) + # TODO: move the `purl` key to the top. xxx + original_validate_purl = validate_purl(purl) + print(f"\noriginal_validate_purl = {original_validate_purl}") + + # === + # # print(f"\nresponse = {response}") + # # Create a new dict with `purl` at the top. + # # response = {"purl": response.pop("purl"), **response} + + reordered_validate_purl = { + "purl": original_validate_purl.pop("purl"), + **original_validate_purl, + } + print(f"\nreordered_validate_purl = {reordered_validate_purl}") + + # # print(f"\nupdated response = {response}") + # # or + # ordered_response = OrderedDict(response) + # purl_value = ordered_response.pop("purl") + # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) + # response = dict(ordered_response) + # # print(f"updated response = {response}") + # === + + # validate_details["packages"].append(validate_purl(purl)) + # validate_details["packages"].append(validate_purl(purl)) + validate_details["packages"].append(reordered_validate_purl) validate_details["headers"] = construct_headers( deduplicated_purls=deduplicated_purls, @@ -607,6 +671,20 @@ def validate_purl(purl): try: response = requests.get(api_query, params=request_body).json() + # ZAP 2024-05-27 Monday 19:41:16. Getting a test error: AssertionError: assert None == {'errors': {'purl': ['This field is required.']}} + # ZAP so maybe do the move-purl-key process after the return below? + # # TODO: move the `purl` key to the top of the dict. + # # print(f"\nresponse = {response}") + # # Create a new dict with `purl` at the top. + # # response = {"purl": response.pop("purl"), **response} + # # print(f"\nupdated response = {response}") + # # or + # ordered_response = OrderedDict(response) + # purl_value = ordered_response.pop("purl") + # ordered_response = OrderedDict({"purl": purl_value}, **ordered_response) + # response = dict(ordered_response) + # # print(f"updated response = {response}") + except json.decoder.JSONDecodeError as e: logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") except Exception as e: diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index dac9392b..c2bf50d7 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -54,246 +54,112 @@ "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } + "https://pypi.org/project/fetchcode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/" }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "https://pypi.org/project/fetchcode/0.3.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/" }, { "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } + "https://pypi.org/project/dejacode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/" }, { "purl": "pkg:pypi/dejacode@5.0.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0?os=windows", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } + "https://crates.io/crates/banquo" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/banquo" }, { "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } + "https://crates.io/crates/socksprox" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/socksprox" }, { "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:nuget/auth0-aspnet@1.1.0", - "download_url": { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - }, + "download_url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0", "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - } + "https://www.nuget.org/packages/auth0-aspnet/1.1.0", + "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - } + "repository_download_url": null, + "repository_homepage_url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index b256c973..2e8de8c1 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -67,17 +67,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -97,17 +92,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/0.3.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -127,17 +117,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -157,17 +142,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -187,17 +167,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -217,17 +192,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -247,17 +217,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -277,17 +242,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://crates.io/crates/banquo", "get_request_status_code": 404, "head_request_status_code": 404 @@ -307,17 +267,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://crates.io/crates/socksprox", "get_request_status_code": 404, "head_request_status_code": 404 @@ -337,17 +292,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -367,17 +317,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -402,17 +347,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { + "repository_homepage_url": { "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", "get_request_status_code": 200, "head_request_status_code": 404 diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 16578130..7de378fd 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -33,17 +33,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 7a8c90de..43def355 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -663,8 +663,6 @@ def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): {"head_request": "N/A"}, {"get_request": "N/A"}, {"head_request": "N/A"}, - {"get_request": "N/A"}, - {"head_request": "N/A"}, {"get_request": 200}, {"head_request": 200}, {"get_request": 200}, @@ -794,23 +792,12 @@ def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, + "download_url": None, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } + "https://pypi.org/project/fetchcode/", ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/", }, ], } diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index a783b89e..372b7bab 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -175,9 +175,6 @@ def test_metadata_details(self): ("version", None), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -200,6 +197,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -211,9 +211,6 @@ def test_metadata_details(self): ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -236,6 +233,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -247,9 +247,6 @@ def test_metadata_details(self): ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -272,6 +269,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), OrderedDict( @@ -283,9 +283,6 @@ def test_metadata_details(self): ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -308,6 +305,9 @@ def test_metadata_details(self): ("dependencies", []), ("contains_source_code", None), ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ] ), ], @@ -741,47 +741,32 @@ def test_urls_details(self, mock_read_log_file): "packages": [ { "purl": "pkg:pypi/fetchcode@0.3.0", - "download_url": {"url": None}, + "download_url": None, "inferred_urls": [ - {"url": "https://pypi.org/project/fetchcode/0.3.0/"} + "https://pypi.org/project/fetchcode/0.3.0/", ], - "repo_download_url": {"url": None}, - "repo_download_url_by_package_type": {"url": None}, - "repo_url": {"url": "https://pypi.org/project/fetchcode/0.3.0/"}, + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/", }, { "purl": "pkg:gem/bundler@2.3.23", - "download_url": { - "url": "https://rubygems.org/downloads/bundler-2.3.23.gem" - }, + "download_url": "https://rubygems.org/downloads/bundler-2.3.23.gem", "inferred_urls": [ - {"url": "https://rubygems.org/gems/bundler/versions/2.3.23"}, - {"url": "https://rubygems.org/downloads/bundler-2.3.23.gem"}, + "https://rubygems.org/gems/bundler/versions/2.3.23", + "https://rubygems.org/downloads/bundler-2.3.23.gem", ], - "repo_download_url": {"url": None}, - "repo_download_url_by_package_type": {"url": None}, - "repo_url": { - "url": "https://rubygems.org/gems/bundler/versions/2.3.23" - }, + "repository_download_url": None, + "repository_homepage_url": "https://rubygems.org/gems/bundler/versions/2.3.23", }, { "purl": "pkg:github/istio/istio@1.20.2", - "download_url": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, + "download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", "inferred_urls": [ - {"url": "https://github.com/istio/istio/tree/1.20.2"}, - { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, + "https://github.com/istio/istio/tree/1.20.2", + "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", ], - "repo_download_url": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, - "repo_download_url_by_package_type": { - "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" - }, - "repo_url": {"url": "https://github.com/istio/istio/tree/1.20.2"}, + "repository_download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", + "repository_homepage_url": "https://github.com/istio/istio/tree/1.20.2", }, ], }