From 4f01784899f7b68d56b6d4039e30b504e100e9b6 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 23 May 2024 12:56:28 -0700 Subject: [PATCH] Update data structure and logging, add cocoapods support #365 - Also replaced PURL normalization option with default deduplication. Reference: https://github.com/nexB/purldb/issues/365 Signed-off-by: John M. Horan --- purldb-toolkit/src/purldb_toolkit/purlcli.py | 467 ++--- .../purlcli/expected_metadata_output.json | 1310 ++++++------ .../expected_metadata_output_superseded.json | 588 ------ .../expected_metadata_output_unique.json | 353 ---- .../data/purlcli/expected_urls_output.json | 59 +- .../purlcli/expected_urls_output_head.json | 83 +- .../expected_urls_output_head_mock.json | 8 - .../purlcli/expected_urls_output_unique.json | 240 --- .../purlcli/expected_validate_output.json | 11 - .../expected_validate_output_unique.json | 77 - .../purlcli/expected_versions_output.json | 231 +- .../expected_versions_output_unique.json | 140 -- purldb-toolkit/tests/test_purlcli.py | 735 +++---- purldb-toolkit/tests/test_purlcli_live.py | 1864 ++++------------- 14 files changed, 1667 insertions(+), 4499 deletions(-) delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json delete mode 100644 purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 0269ce97..917c1008 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -17,11 +17,13 @@ import click import requests from fetchcode.package import info -from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS, versions +from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS +from fetchcode.package_versions import versions from packageurl import PackageURL from packageurl.contrib import purl2url LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") +logger = logging.getLogger(__name__) @click.group() @@ -52,162 +54,153 @@ def purlcli(): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_metadata(purls, output, file, unique): +def get_metadata(purls, output, file): """ Given one or more PURLs, for each PURL, return a mapping of metadata fetched from the fetchcode package.py info() function. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - metadata_info = get_metadata_details(purls, output, file, unique, command_name) + metadata_info = get_metadata_details(purls, output, file, command_name) json.dump(metadata_info, output, indent=4) -def get_metadata_details(purls, output, file, unique, command_name): +def get_metadata_details(purls, output, file, command_name): """ Return a dictionary containing metadata for each PURL in the `purls` input - list. `check_metadata_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ metadata_details = {} metadata_details["headers"] = [] metadata_details["packages"] = [] - metadata_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - - purl_data = {} - purl_data["purl"] = purl - - metadata_purl = check_metadata_purl(purl) - - if command_name == "metadata" and metadata_purl: - metadata_warnings[purl] = metadata_purl + metadata_purl_status = check_metadata_purl(purl) + if command_name == "metadata" and metadata_purl_status in [ + "validation_error", + "not_valid", + "valid_but_not_supported", + "not_in_upstream_repo", + ]: + metadata_warnings[purl] = metadata_purl_status continue - + if command_name == "metadata" and metadata_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + metadata_warnings[purl] = metadata_purl_status metadata_collection = collect_metadata(purl) - purl_data["metadata"] = metadata_collection - metadata_details["packages"].append(purl_data) + metadata_details["packages"].extend(metadata_collection) + + print(f"\nmetadata_warnings = {metadata_warnings}") metadata_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=metadata_warnings, ) - return metadata_details def collect_metadata(purl): """ - Return a list of release-based metadata collections from fetchcode/package.py. + Return a list of release-based metadata collections (a fetchcode Package) + from fetchcode/package.py. """ collected_metadata = [] for release in list(info(purl)): + if release is None: + continue release_detail = release.to_dict() release_detail.move_to_end("purl", last=False) collected_metadata.append(release_detail) - return collected_metadata def check_metadata_purl(purl): """ - Return a variable identifying the message for printing to the console by - get_metadata_details() if (1) the input PURL is invalid, (2) its type is not - supported by `metadata` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `metadata` command. + Return a variable identifying the warning if (1) the input PURL is invalid, + (2) its type is not supported by `metadata` or (3) its existence was not + validated (e.g., "does not exist in the upstream repo"). This will be + reported by construct_headers() in the `warnings` field of the `header` + section of the JSON object returned by the `metadata` command. """ check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" # This is manually constructed from a visual inspection of fetchcode/package.py. metadata_supported_ecosystems = [ "bitbucket", "cargo", + "generic", "github", + "gnu", "npm", + "openssl", "pypi", "rubygems", + # NOTE: cocoapods support added subject to fetchcode/package.py PR approval and new release. + "cocoapods", ] metadata_purl = PackageURL.from_string(purl) if metadata_purl.type not in metadata_supported_ecosystems: return "valid_but_not_supported" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" -def normalize_purls(purls, unique): - """ - If the command includes the `--unique` flag, take the list of input PURLs, - remove the portion of the PURL that starts with a PURL separator (`@`, `?` - or `#`), and return a deduplicated list of the resulting PURLs (in - `input_purls`) and a list of tuples of each pair of the original input PURL - and the normalized PURL (in `normalized_purls`). - """ - input_purls = [] - normalized_purls = [] - if unique: - for purl in purls: - input_purl = purl - purl = purl.strip() - purl = re.split("[@,?,#,]+", purl)[0] - normalized_purl = purl - normalized_purls.append((input_purl, normalized_purl)) - if normalized_purl not in input_purls: - input_purls.append(normalized_purl) - else: - input_purls = purls +def deduplicate_purls(purls): + """ + Deduplicate all input PURLs. PURLs with different versions or no version + are treated as unique for purposes of deduplication. + """ + reviewed = set() + deduplicated_purls = [] + duplicate_purls = [] + for purl in purls: + purl = purl.strip() + if purl not in reviewed: + reviewed.add(purl) + deduplicated_purls.append(purl) + else: + duplicate_purls.append(purl) + return deduplicated_purls, duplicate_purls - return input_purls, normalized_purls + +def read_log_file(log_file_path): + log_file = log_file_path + if log_file.is_file(): + with open(log_file_path, "r") as log_file: + return log_file.readlines() + else: + return [] def construct_headers( - purls=None, + deduplicated_purls=None, + duplicate_purls=None, output=None, file=None, command_name=None, head=None, - normalized_purls=None, - unique=None, purl_warnings=None, ): """ @@ -219,7 +212,7 @@ def construct_headers( errors = [] warnings = [] - context_purls = [p for p in purls] + context_purls = [p for p in deduplicated_purls] context_file = file context_file_name = None if context_file: @@ -235,28 +228,20 @@ def construct_headers( if head: options["--head"] = True - if unique: - options["--unique"] = True - if isinstance(output, str): options["--output"] = output else: options["--output"] = output.name headers_content["options"] = options - headers_content["purls"] = purls - - if (command_name in ["metadata", "urls", "validate", "versions"]) and unique: - for input_purl, normalized_purl in normalized_purls: - if input_purl != normalized_purl: - warnings.append( - f"input PURL: '{input_purl}' normalized to '{normalized_purl}'" - ) + if command_name in ["metadata", "urls", "validate", "versions"]: + if duplicate_purls: + for duplicate in duplicate_purls: + logger.warning(f"Duplicate input PURL removed: {duplicate}") - for purl in purls: + for purl in deduplicated_purls: if not purl: continue - warning_text = { "error_fetching_purl": f"'error fetching {purl}'", "validation_error": f"'{purl}' encountered a validation error", @@ -266,25 +251,25 @@ def construct_headers( "not_in_upstream_repo": f"'{purl}' does not exist in the upstream repo", "check_existence_not_supported": f"'check_existence' is not supported for '{purl}'", } - if command_name in ["metadata", "urls", "validate", "versions"]: purl_warning = purl_warnings.get(purl, None) - if purl_warning: warning = warning_text[purl_warning] - warnings.append(warning) + logger.warning(warning) continue log_file = Path(LOG_FILE_LOCATION) - if log_file.is_file(): - with open(log_file, "r") as f: - for line in f: - errors.append(line) + log_file_contents = read_log_file(log_file) + if log_file_contents: + for line in log_file_contents: + if line.startswith("ERROR"): + errors.append(line[8:-1]) + elif line.startswith("WARNING"): + warnings.append(line[10:-1]) headers_content["errors"] = errors headers_content["warnings"] = warnings headers.append(headers_content) - return headers @@ -309,102 +294,81 @@ def construct_headers( required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) @click.option( "--head", is_flag=True, required=False, help="Validate each URL's existence with a head request.", ) -def get_urls(purls, output, file, unique, head): +def get_urls(purls, output, file, head): """ Given one or more PURLs, for each PURL, return a list of all known URLs fetched from the packageurl-python purl2url.py code. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - urls_info = get_urls_details(purls, output, file, unique, head, command_name) + urls_info = get_urls_details(purls, output, file, head, command_name) json.dump(urls_info, output, indent=4) -def get_urls_details(purls, output, file, unique, head, command_name): +def get_urls_details(purls, output, file, head, command_name): """ Return a dictionary containing URLs for each PURL in the `purls` input - list. `check_urls_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ urls_details = {} urls_details["headers"] = [] urls_details["packages"] = [] - urls_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: url_detail = {} url_detail["purl"] = purl - purl = purl.strip() if not purl: continue - - purl_status = check_urls_purl(purl) - - if command_name == "urls" and purl_status in [ + urls_purl_status = check_urls_purl(purl) + if command_name == "urls" and urls_purl_status in [ "validation_error", "not_valid", "valid_but_not_supported", "not_in_upstream_repo", ]: - urls_warnings[purl] = purl_status + urls_warnings[purl] = urls_purl_status continue + if command_name == "urls" and urls_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + urls_warnings[purl] = urls_purl_status - if command_name == "urls" and purl_status in ["valid_but_not_fully_supported"]: - urls_warnings[purl] = purl_status - - # Add the URLs. url_purl = PackageURL.from_string(purl) url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} - url_detail["inferred_urls"] = [ {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} - - url_detail["repo_download_url_by_package_type"] = { - "url": purl2url.get_repo_download_url_by_package_type( - url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - ) - } - + url_detail["repo_download_url_by_package_type"] = {"url": None} + if url_purl.version: + url_detail["repo_download_url_by_package_type"] = { + "url": purl2url.get_repo_download_url_by_package_type( + url_purl.type, url_purl.namespace, url_purl.name, url_purl.version + ) + } url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} - url_detail["url"] = {"url": purl2url.get_url(purl)} - - # Add the http status code data. url_list = [ "download_url", # "inferred_urls" has to be handled separately because it has a nested list "repo_download_url", "repo_download_url_by_package_type", "repo_url", - "url", ] if head: for purlcli_url in url_list: @@ -414,7 +378,6 @@ def get_urls_details(purls, output, file, unique, head, command_name): url_detail[purlcli_url]["head_request_status_code"] = make_head_request( url_detail[purlcli_url]["url"] ).get("head_request") - for inferred_url in url_detail["inferred_urls"]: inferred_url["get_request_status_code"] = make_head_request( inferred_url["url"] @@ -422,44 +385,34 @@ def get_urls_details(purls, output, file, unique, head, command_name): inferred_url["head_request_status_code"] = make_head_request( inferred_url["url"] ).get("head_request") - urls_details["packages"].append(url_detail) urls_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, head=head, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=urls_warnings, ) - return urls_details def make_head_request(url_detail): """ - Make a head request (and as noted below, a get request as well, at least - for now) and return a dictionary containing status code data for the - incoming PURL URL. - - For now, this returns both get and head request status code data so the - user can evaluate -- requests.get() and requests.head() sometimes return - different status codes and sometimes return inaccurate codes, e.g., a - 404 when the URL actually exists. + Make a head request and get request and return a dictionary containing + status code data for the incoming PURL URL. This returns both get and + head request status code data so the user can evaluate -- requests.get() + and requests.head() sometimes return different status codes and sometimes + return inaccurate codes, e.g., a 404 when the URL actually exists. """ if url_detail is None: return {"get_request": "N/A", "head_request": "N/A"} - get_response = requests.get(url_detail) get_request_status_code = get_response.status_code - head_response = requests.head(url_detail) head_request_status_code = head_response.status_code - - # Return a dictionary for readability. return { "get_request": get_request_status_code, "head_request": head_request_status_code, @@ -476,17 +429,17 @@ def check_urls_purl(purl): if check_validation is None: return "validation_error" results = check_validation - if results["valid"] == False: return "not_valid" # Both of these lists are manually constructed from a visual inspection of # packageurl-python/src/packageurl/contrib/purl2url.py. - - # This list applies to the purl2url.py `repo_url` section: + # This list applies to the purl2url.py `repo_url` section: urls_supported_ecosystems_repo_url = [ "bitbucket", "cargo", + # NOTE: Temp for cocoapods dev work in purl2url. Keep in the list uncommented -- I still need to respond to Tushar's comments but the real code work is done, now supported for repo_url. + "cocoapods", "gem", "github", "gitlab", @@ -497,7 +450,6 @@ def check_urls_purl(purl): "pypi", "rubygems", ] - # This list applies to the purl2url.py `download_url` section: urls_supported_ecosystems_download_url = [ "bitbucket", @@ -510,7 +462,6 @@ def check_urls_purl(purl): "nuget", "rubygems", ] - urls_purl = PackageURL.from_string(purl) if ( @@ -518,10 +469,8 @@ def check_urls_purl(purl): and urls_purl.type not in urls_supported_ecosystems_download_url ): return "valid_but_not_supported" - if results["exists"] == False: return "not_in_upstream_repo" - if ( urls_purl.type in urls_supported_ecosystems_repo_url and urls_purl.type not in urls_supported_ecosystems_download_url @@ -529,8 +478,9 @@ def check_urls_purl(purl): urls_purl.type not in urls_supported_ecosystems_repo_url and urls_purl.type in urls_supported_ecosystems_download_url ): - return "valid_but_not_fully_supported" + if results["exists"] == None: + return "check_existence_not_supported" @purlcli.command(name="validate") @@ -554,73 +504,55 @@ def check_urls_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def validate(purls, output, file, unique): +def validate(purls, output, file): """ Check the syntax and upstream repo status of one or more PURLs. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - validated_purls = get_validate_details(purls, output, file, unique, command_name) + validated_purls = get_validate_details(purls, output, file, command_name) json.dump(validated_purls, output, indent=4) -def get_validate_details(purls, output, file, unique, command_name): +def get_validate_details(purls, output, file, command_name): """ Return a dictionary containing validation data for each PURL in the `purls` input list. """ validate_details = {} validate_details["headers"] = [] - validate_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) validate_details["packages"] = [] - clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - - validated_purl = check_validate_purl(purl) - - if command_name == "validate" and validated_purl in [ + validated_purl_status = check_validate_purl(purl) + if command_name == "validate" and validated_purl_status in [ "validation_error", "not_valid", "valid_but_not_supported", "not_in_upstream_repo", "check_existence_not_supported", ]: - validate_warnings[purl] = validated_purl - - if validated_purl: + validate_warnings[purl] = validated_purl_status + if validated_purl_status: validate_details["packages"].append(validate_purl(purl)) validate_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=validate_warnings, ) - return validate_details @@ -632,18 +564,13 @@ def check_validate_purl(purl): check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == True: + elif check_validation["exists"] == True: return check_validation - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" @@ -668,33 +595,27 @@ def validate_purl(purl): "nuget", "pypi", """ - logger = logging.getLogger(__name__) + logging.basicConfig( + level=logging.WARN, + format="%(levelname)s - %(message)s", + filename=LOG_FILE_LOCATION, + filemode="w", + ) api_query = "https://public.purldb.io/api/validate/" request_body = {"purl": purl, "check_existence": True} try: response = requests.get(api_query, params=request_body).json() - except json.decoder.JSONDecodeError as e: - - print(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - - logging.basicConfig( - filename=LOG_FILE_LOCATION, - level=logging.ERROR, - format="%(levelname)s - %(message)s", - filemode="w", - ) - logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - except Exception as e: - print(f"'validate' endpoint error for '{purl}': {e}") - + logger.error(f"'validate' endpoint error for '{purl}': {e}") else: if response is None: - print(f"'{purl}' -- response.status_code for None = {response.status_code}") + logger.error( + f"'{purl}' -- response.status_code for None = {response.status_code}" + ) return response @@ -719,99 +640,89 @@ def validate_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_versions(purls, output, file, unique): +def get_versions(purls, output, file): """ Given one or more PURLs, return a list of all known versions for each PURL. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - purl_versions = get_versions_details(purls, output, file, unique, command_name) + purl_versions = get_versions_details(purls, output, file, command_name) json.dump(purl_versions, output, indent=4) -def get_versions_details(purls, output, file, unique, command_name): +def get_versions_details(purls, output, file, command_name): """ Return a list of dictionaries containing version-related data for each PURL - in the `purls` input list. `check_versions_purl()` will print an error - message to the console (also displayed in the JSON output) when necessary. + in the `purls` input list. """ versions_details = {} versions_details["headers"] = [] versions_details["packages"] = [] - versions_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - + deduplicated_purls, duplicate_purls = deduplicate_purls(purls) clear_log_file() - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - purl_data = {} purl_data["purl"] = purl - - versions_purl = check_versions_purl(purl) - - if command_name == "versions" and versions_purl: - versions_warnings[purl] = versions_purl + versions_purl_status = check_versions_purl(purl) + if command_name == "versions" and versions_purl_status in [ + "validation_error", + "not_valid", + "valid_but_not_supported", + "not_in_upstream_repo", + ]: + versions_warnings[purl] = versions_purl_status continue - + if command_name == "versions" and versions_purl_status in [ + "valid_but_not_fully_supported", + "check_existence_not_supported", + ]: + versions_warnings[purl] = versions_purl_status version_collection = collect_versions(purl) - - purl_data["versions"] = version_collection - versions_details["packages"].append(purl_data) + versions_details["packages"].extend(version_collection) versions_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, + duplicate_purls=duplicate_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, purl_warnings=versions_warnings, ) - return versions_details def collect_versions(purl): """ Return a list of version objects collected from fetchcode/package_versions.py. + + We use `versions()` from fetchcode/package_versions.py, which keeps the + version (if any) of the input PURL in its output, so + "pkg:pypi/fetchcode@0.3.0" is returned as "pkg:pypi/fetchcode@0.3.0@0.1.0", + "pkg:pypi/fetchcode@0.3.0@0.2.0" etc. Thus, we remove any string starting + with `@` first. """ collected_versions = [] for package_version in list(versions(purl)): purl_version_data = {} purl_version = package_version.value - # We use `versions()` from fetchcode/package_versions.py, which - # keeps the version (if any) of the input PURL in its output, so - # "pkg:pypi/fetchcode@0.3.0" is returned as - # "pkg:pypi/fetchcode@0.3.0@0.1.0", "pkg:pypi/fetchcode@0.3.0@0.2.0" - # etc. Thus, we remove any string starting with `@` first. - raw_purl = purl = re.split("[@,]+", purl)[0] + raw_purl = re.split("[@,]+", purl)[0] nested_purl = raw_purl + "@" + f"{purl_version}" + pkg_ver_release_date = package_version.release_date + pkg_ver_release_date_no_time = pkg_ver_release_date.date() purl_version_data["purl"] = nested_purl purl_version_data["version"] = f"{purl_version}" - purl_version_data["release_date"] = f"{package_version.release_date}" - + purl_version_data["release_date"] = f"{pkg_ver_release_date_no_time}" collected_versions.append(purl_version_data) - return collected_versions @@ -820,11 +731,9 @@ def check_versions_purl(purl): Return a variable identifying the message for printing to the console by get_versions_details() if (1) the input PURL is invalid, (2) its type is not supported by `versions` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `versions` command. + "does not exist in the upstream repo"). This message will also be reported + by construct_headers() in the `warnings` field of the `header` section of + the JSON object returned by the `versions` command. Note for dev purposes: SUPPORTED_ECOSYSTEMS (imported from fetchcode.package_versions) comprises the following types: @@ -846,43 +755,35 @@ def check_versions_purl(purl): check_validation = validate_purl(purl) if check_validation is None: return "validation_error" - results = check_validation - - if results["valid"] == False: + elif check_validation["valid"] == False: return "not_valid" supported = SUPPORTED_ECOSYSTEMS versions_purl = PackageURL.from_string(purl) - if versions_purl.type not in supported: return "valid_but_not_supported" - - if results["exists"] == False: + elif check_validation["exists"] == False: return "not_in_upstream_repo" - - if results["exists"] == None: + elif check_validation["exists"] == None: return "check_existence_not_supported" - # This handles the conflict between the `validate`` endpoint (treats # both "pkg:deb/debian/2ping" and "pkg:deb/2ping" as valid) and # fetchcode.package_versions versions() (returns None for "pkg:deb/2ping"). - if versions(purl) is None: + elif versions(purl) is None: return "valid_but_not_supported" def check_for_duplicate_input_sources(purls, file): if purls and file: raise click.UsageError("Use either purls or file but not both.") - - if not (purls or file): + elif not (purls or file): raise click.UsageError("Use either purls or file.") def clear_log_file(): log_file = Path(LOG_FILE_LOCATION) - - if log_file.is_file(): - os.remove(log_file) + with open(log_file, "w"): + pass if __name__ == "__main__": diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 1f5cb267..32df4492 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -19,17 +19,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", @@ -43,680 +32,655 @@ "packages": [ { "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": { + "os": "windows" + }, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "cargo", + "namespace": null, + "name": "banquo", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "api_url": "https://crates.io/api/v1/crates/banquo", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:cargo/banquo@0.1.0", + "type": "cargo", + "namespace": null, + "name": "banquo", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", + "api_url": "https://crates.io/api/v1/crates/banquo", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "BSD-3-Clause", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "rubygems", + "namespace": null, + "name": "rails", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://rubyonrails.org", + "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", + "api_url": "https://rubygems.org/api/v1/gems/rails.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/rails/rails/issues", + "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": [ + "MIT" + ], + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json deleted file mode 100644 index a237dae1..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json +++ /dev/null @@ -1,588 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json deleted file mode 100644 index 5ede4644..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json +++ /dev/null @@ -1,353 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index dc1d1e4a..dac9392b 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -30,28 +30,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command", @@ -66,6 +44,7 @@ "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", "'pkg:nginx/nginx' not supported with `urls` command", "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", "'pkg:pypi/matchcode' does not exist in the upstream repo", "'abcdefg' not valid", "'pkg/abc' not valid" @@ -91,9 +70,6 @@ }, "repo_url": { "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" } }, { @@ -114,9 +90,6 @@ }, "repo_url": { "url": "https://pypi.org/project/fetchcode/0.3.0/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" } }, { @@ -137,9 +110,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" } }, { @@ -160,9 +130,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -183,9 +150,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -206,9 +170,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -229,9 +190,6 @@ }, "repo_url": { "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" } }, { @@ -252,9 +210,6 @@ }, "repo_url": { "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" } }, { @@ -275,9 +230,6 @@ }, "repo_url": { "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" } }, { @@ -298,9 +250,6 @@ }, "repo_url": { "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" } }, { @@ -321,9 +270,6 @@ }, "repo_url": { "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" } }, { @@ -347,9 +293,6 @@ }, "repo_url": { "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index d5b256e4..b256c973 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -31,28 +31,6 @@ "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command", @@ -67,6 +45,7 @@ "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", "'pkg:nginx/nginx' not supported with `urls` command", "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", + "'check_existence' is not supported for 'pkg:rubygems/bundler-sass'", "'pkg:pypi/matchcode' does not exist in the upstream repo", "'abcdefg' not valid", "'pkg/abc' not valid" @@ -102,11 +81,6 @@ "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -137,11 +111,6 @@ "url": "https://pypi.org/project/fetchcode/0.3.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -172,11 +141,6 @@ "url": "https://pypi.org/project/dejacode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -207,11 +171,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -242,11 +201,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -277,11 +231,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -312,11 +261,6 @@ "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -347,11 +291,6 @@ "url": "https://crates.io/crates/banquo", "get_request_status_code": 404, "head_request_status_code": 404 - }, - "url": { - "url": "https://crates.io/crates/banquo", - "get_request_status_code": 404, - "head_request_status_code": 404 } }, { @@ -382,11 +321,6 @@ "url": "https://crates.io/crates/socksprox", "get_request_status_code": 404, "head_request_status_code": 404 - }, - "url": { - "url": "https://crates.io/crates/socksprox", - "get_request_status_code": 404, - "head_request_status_code": 404 } }, { @@ -417,11 +351,6 @@ "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -452,11 +381,6 @@ "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 } }, { @@ -492,11 +416,6 @@ "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", "get_request_status_code": 200, "head_request_status_code": 404 - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", - "get_request_status_code": 200, - "head_request_status_code": 404 } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 5f6c2e21..16578130 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -12,9 +12,6 @@ "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command" @@ -50,11 +47,6 @@ "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 } } ] diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json deleted file mode 100644 index 193b8096..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "urls", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0#how/are/you' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@10.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:nginx/nginx@0.8.9?os=windows' normalized to 'pkg:nginx/nginx'", - "input PURL: 'pkg:nuget/auth0-aspnet@1.1.0' normalized to 'pkg:nuget/auth0-aspnet'", - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" - } - }, - { - "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" - } - }, - { - "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" - } - }, - { - "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" - } - }, - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:nuget/auth0-aspnet", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json index ac85f7c7..d1f74eba 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json @@ -19,17 +19,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json deleted file mode 100644 index 280388f2..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "validate", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'check_existence' is not supported for 'pkg:nginx/nginx'", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode" - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:cargo/banquo" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - "valid": true - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/rails" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/rails", - "valid": true - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json index 93feeb96..e9c6abf8 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json @@ -18,16 +18,6 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], "errors": [], "warnings": [ "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", @@ -38,139 +28,114 @@ ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "version": "0.3.0", + "release_date": "2023-12-18" }, { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" }, { - "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18" + }, + { + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25" + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14" + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18" + }, + { + "purl": "pkg:cargo/banquo@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07" + }, + { + "purl": "pkg:hex/coherence@0.8.0", + "version": "0.8.0", + "release_date": "2023-09-22" + }, + { + "purl": "pkg:hex/coherence@0.5.2", + "version": "0.5.2", + "release_date": "2018-09-03" + }, + { + "purl": "pkg:hex/coherence@0.5.1", + "version": "0.5.1", + "release_date": "2018-08-28" + }, + { + "purl": "pkg:hex/coherence@0.5.0", + "version": "0.5.0", + "release_date": "2017-08-02" + }, + { + "purl": "pkg:hex/coherence@0.4.0", + "version": "0.4.0", + "release_date": "2017-07-03" + }, + { + "purl": "pkg:hex/coherence@0.3.1", + "version": "0.3.1", + "release_date": "2016-11-27" + }, + { + "purl": "pkg:hex/coherence@0.3.0", + "version": "0.3.0", + "release_date": "2016-08-28" + }, + { + "purl": "pkg:hex/coherence@0.2.0", + "version": "0.2.0", + "release_date": "2016-07-30" + }, + { + "purl": "pkg:hex/coherence@0.1.3", + "version": "0.1.3", + "release_date": "2016-07-19" + }, + { + "purl": "pkg:hex/coherence@0.1.2", + "version": "0.1.2", + "release_date": "2016-07-12" + }, + { + "purl": "pkg:hex/coherence@0.1.1", + "version": "0.1.1", + "release_date": "2016-07-11" }, { "purl": "pkg:hex/coherence@0.1.0", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] + "version": "0.1.0", + "release_date": "2016-07-11" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json deleted file mode 100644 index 1991c21f..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:hex/coherence@0.1.0' normalized to 'pkg:hex/coherence'", - "'pkg:nginx/nginx' not supported with `versions` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] - }, - { - "purl": "pkg:hex/coherence", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index f9cc12d5..7a8c90de 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -10,15 +10,14 @@ import json import os from collections import OrderedDict -from contextlib import redirect_stdout -from io import StringIO from unittest import mock import pytest import requests from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -236,193 +235,184 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", None), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("download_url", None), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", ), - ], - } + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ] + ), ], } input_purls = ["pkg:pypi/fetchcode"] - output = "" file = "" command_name = "metadata" - unique = False purl_metadata_data = purlcli.get_metadata_details( input_purls, output, file, - unique, command_name, ) @@ -513,113 +503,40 @@ def test_check_metadata_purl_multiple(self, mock_validate_purl): purl_metadata = purlcli.check_metadata_purl(input_purl) assert purl_metadata == expected_state - @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", - [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), - ( - [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls + def test_deduplicate_purls(self): + input_purls = [ + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + "pkg:pypi/fetchcode@0.2.0", + ] + actual_output = purlcli.deduplicate_purls(input_purls) + expected_output = ( + ["pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0"], + [ + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + ], + ) + assert actual_output == expected_output @pytest.mark.parametrize( "test_input,expected", [ ( [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode@0.2.0", ], [ @@ -629,40 +546,51 @@ def test_normalize_purls( "--file": None, "--output": "", "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode@0.2.0", ], "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0", + "Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0", ], } ], ), ], ) - def test_construct_headers(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_deduplicate_purls_construct_headers( + self, mock_read_log_file, test_input, expected + ): + mock_read_log_file.return_value = [ + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.1.0\n", + "WARNING - Duplicate input PURL removed: pkg:pypi/fetchcode@0.2.0\n", + ] + metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, + purl_warnings={}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -690,43 +618,33 @@ def test_construct_headers(self, test_input, expected): "pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0", ], - "--unique": True, "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", + "'pkg:gem/bundler-sass' not supported with `metadata` command" ], } ], ), ], ) - def test_construct_headers_unique(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_construct_headers(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", + ] + metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), - ], - unique=True, purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -734,8 +652,9 @@ def test_construct_headers_unique(self, test_input, expected): class TestPURLCLI_urls(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.make_head_request") - def test_urls_cli_head(self, mock_make_head_request): + def test_urls_cli_head(self, mock_make_head_request, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ @@ -753,6 +672,11 @@ def test_urls_cli_head(self, mock_make_head_request): {"get_request": 200}, {"head_request": 200}, ] + + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head_mock.json" ) @@ -781,7 +705,6 @@ def test_urls_cli_head(self, mock_make_head_request): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -842,10 +765,15 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.check_urls_purl") - def test_urls_details(self, mock_check_urls_purl): + def test_urls_details(self, mock_check_urls_purl, mock_read_log_file): mock_check_urls_purl.return_value = "valid_but_not_fully_supported" + mock_read_log_file.return_value = [ + "WARNING = 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + ] + expected_data = { "headers": [ { @@ -857,7 +785,6 @@ def test_urls_details(self, mock_check_urls_purl): "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [ "'pkg:pypi/fetchcode' not fully supported with `urls` command" @@ -884,9 +811,6 @@ def test_urls_details(self, mock_check_urls_purl): "repo_url": { "url": "https://pypi.org/project/fetchcode/", }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, }, ], } @@ -899,7 +823,6 @@ def test_urls_details(self, mock_check_urls_purl): file="", command_name="urls", head=False, - unique=False, ) cli_test_utils.streamline_headers(expected_data["headers"]) cli_test_utils.streamline_headers(purl_urls["headers"]) @@ -961,59 +884,13 @@ def mock_requests_get_return_func(): results = purlcli.validate_purl(input_purl) assert mock_request_response == results - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_jsondecodeerror(self, mock_requests_get): - def json_decode_failure_exception(**kwargs): - raise json.decoder.JSONDecodeError("test", "[{}]", 0) - - mock_requests_get.return_value.json = json_decode_failure_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", json_decode_failure_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert ( - "json.decoder.JSONDecodeError for 'pkg:pypi/fetchcode': test: line 1 column 1 (char 0)" - in results - ) - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_exception(self, mock_requests_get): - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'pkg:pypi/fetchcode': \n" in results - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_none(self, mock_requests_get): - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = None - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'None': \n" in results - class TestPURLCLI_versions(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") @mock.patch("purldb_toolkit.purlcli.collect_versions") @mock.patch("purldb_toolkit.purlcli.check_versions_purl") def test_versions_details_multiple( - self, - mock_check_versions_purl, - mock_collect_versions, + self, mock_check_versions_purl, mock_collect_versions, mock_read_log_file ): mock_check_versions_purl.side_effect = [ @@ -1030,40 +907,51 @@ def test_versions_details_multiple( { "purl": "pkg:pypi/fetchcode@0.1.0", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { "purl": "pkg:pypi/fetchcode@0.2.0", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { "purl": "pkg:pypi/fetchcode@0.3.0", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ], [ { "purl": "pkg:gem/bundler-sass@0.1.2", "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", + "release_date": "2013-12-11", } ], [ { "purl": "pkg:cargo/socksprox@0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", + "release_date": "2024-02-07", "version": "0.1.1", }, { "purl": "pkg:cargo/socksprox@0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", + "release_date": "2024-02-07", "version": "0.1.0", }, ], ] + mock_read_log_file.side_effect = [ + [], + [], + [ + "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", + ], + ["WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n"], + [], + ["WARNING - 'pkg:pypi/?fetchcode' not valid\n"], + ] + input_purls_and_expected_purl_data = [ [ ["pkg:pypi/fetchcode"], @@ -1078,32 +966,26 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], }, ], @@ -1120,21 +1002,15 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:gem/bundler-sass"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], + "purl": "pkg:gem/bundler-sass@0.1.2", + "version": "0.1.2", + "release_date": "2013-12-11", } ], }, @@ -1152,7 +1028,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:rubygems/bundler-sass"], "errors": [], "warnings": [ "'pkg:rubygems/bundler-sass' not supported with `versions` command" @@ -1175,7 +1050,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:nginx/nginx"], "errors": [], "warnings": [ "'pkg:nginx/nginx' not supported with `versions` command" @@ -1198,27 +1072,21 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:cargo/socksprox"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:cargo/socksprox", - "versions": [ - { - "purl": "pkg:cargo/socksprox@0.1.1", - "version": "0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", - }, - { - "purl": "pkg:cargo/socksprox@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", - }, - ], - } + "purl": "pkg:cargo/socksprox@0.1.1", + "version": "0.1.1", + "release_date": "2024-02-07", + }, + { + "purl": "pkg:cargo/socksprox@0.1.0", + "version": "0.1.0", + "release_date": "2024-02-07", + }, ], }, ], @@ -1235,7 +1103,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/?fetchcode"], "errors": [], "warnings": ["'pkg:pypi/?fetchcode' not valid"], } @@ -1248,11 +1115,13 @@ def test_versions_details_multiple( output = "" file = "" command_name = "versions" - unique = False for input_purl, expected_data in input_purls_and_expected_purl_data: purl_versions_data = purlcli.get_versions_details( - input_purl, output, file, unique, command_name + input_purl, + output, + file, + command_name, ) assert purl_versions_data == expected_data @@ -1269,17 +1138,17 @@ def test_versions_details( { "purl": "pkg:pypi/fetchcode@0.1.0", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { "purl": "pkg:pypi/fetchcode@0.2.0", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { "purl": "pkg:pypi/fetchcode@0.3.0", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ] @@ -1296,32 +1165,26 @@ def test_versions_details( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], } @@ -1330,13 +1193,11 @@ def test_versions_details( output = "" file = "" command_name = "versions" - unique = False purl_versions_data = purlcli.get_versions_details( input_purls, output, file, - unique, command_name, ) assert purl_versions_data == expected_data diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 929bc1af..a783b89e 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -10,11 +10,13 @@ import json import os from collections import OrderedDict +from unittest import mock import pytest from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -23,7 +25,8 @@ class TestPURLCLI_metadata(object): - def test_metadata_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_metadata_cli(self, mock_read_log_file): """ Test the `metadata` command with actual and expected JSON output files. @@ -31,6 +34,14 @@ def test_metadata_cli(self): because the `--output` values (paths) differ due to the use of temporary files, and therefore we test a list of relevant key-value pairs. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `metadata` command\n", + "WARNING - 'pkg:gem/rails' not supported with `metadata` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_metadata_output.json" ) @@ -76,7 +87,6 @@ def test_metadata_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -102,94 +112,12 @@ def test_metadata_cli(self): for output, expected in result_objects: assert output == expected - # NOTE: To avoid errors from the addition of new versions, we exclude - # "packages" from the result_objects list above and handle here. - # All other live-fetch tests are handled in a similar manner. - - compare_packages(expected_data, output_data, "metadata") - - def test_metadata_cli_unique(self): """ - Test the `metadata` command with actual and expected JSON output files - with the `--unique` flag included in the command. + NOTE: To avoid errors from the addition of new versions, we exclude + "packages" from the result_objects list above and handle here. All + other live-fetch tests are handled in a similar manner. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_metadata_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_metadata_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", - "--output", - actual_result_file, - "--unique", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - streamline_metadata_packages(output_data["packages"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - streamline_metadata_packages(expected_data["packages"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), - ] - - for output, expected in result_objects: - assert output == expected - - compare_packages(expected_data, output_data, "metadata") + compare_packages(expected_data, output_data) def test_metadata_cli_duplicate_input_sources(self): """ @@ -221,430 +149,190 @@ def test_metadata_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ("download_url", None), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - ], - } - ], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "metadata", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "options": { - "command": "metadata", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'check_existence' is not supported for " - "'pkg:rubygems/bundler-sass'", - ], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:rubygems/bundler-sass"), - ("type", "rubygems"), - ("namespace", None), - ("name", "bundler-sass"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "http://github.com/vogelbek/bundler-sass", - ), - ( - "download_url", - "https://rubygems.org/gems/bundler-sass-0.1.2.gem", - ), - ( - "api_url", - "https://rubygems.org/api/v1/gems/bundler-sass.json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", ["MIT"]), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ) - ], - } - ], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "metadata", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "metadata", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "metadata", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], + def test_metadata_details(self): + expected_data = { + "headers": [ { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "metadata", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } - ], - "packages": [], - }, - ), - ], - ) - def test_metadata_details(self, test_input, expected): - """ - Test the `metadata` nested function, `get_metadata_details()`. - """ - purl_metadata = purlcli.get_metadata_details( - test_input, - output="", - file="", - command_name="metadata", - unique=False, + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": ["pkg:pypi/fetchcode"], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [], + } + ], + "packages": [ + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", None), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + ], + } + + input_purls = ["pkg:pypi/fetchcode"] + output = "" + file = "" + command_name = "metadata" + + purl_metadata_data = purlcli.get_metadata_details( + input_purls, + output, + file, + command_name, ) - cli_test_utils.streamline_headers(purl_metadata["headers"]) - streamline_metadata_packages(purl_metadata["packages"]) - cli_test_utils.streamline_headers(expected["headers"]) - streamline_metadata_packages(expected["packages"]) + cli_test_utils.streamline_headers(purl_metadata_data["headers"]) + streamline_metadata_packages(purl_metadata_data["packages"]) - assert purl_metadata["headers"] == expected["headers"] + cli_test_utils.streamline_headers(expected_data["headers"]) + streamline_metadata_packages(expected_data["packages"]) - compare_packages(expected, purl_metadata, "metadata") + assert purl_metadata_data["headers"] == expected_data["headers"] + compare_packages(expected_data, purl_metadata_data) @pytest.mark.parametrize( "test_input,expected", @@ -684,113 +372,14 @@ def test_check_metadata_purl(self, test_input, expected): assert purl_metadata == expected @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", + "test_input,expected", [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), ( [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", + "pkg:gem/bundler-sass", + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", ], [ { @@ -806,12 +395,6 @@ def test_normalize_purls( ], "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.2.0", "warnings": [ @@ -822,79 +405,18 @@ def test_normalize_purls( ), ], ) - def test_construct_headers(self, test_input, expected): - metadata_headers = purlcli.construct_headers( - test_input, - output="", - file="", - command_name="metadata", - head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, - ) - cli_test_utils.streamline_headers(expected) - cli_test_utils.streamline_headers(metadata_headers) - - assert metadata_headers == expected + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_construct_headers(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:gem/bundler-sass' not supported with `metadata` command\n", + ] - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "--unique": True, - "command": "metadata", - }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", - ], - } - ], - ), - ], - ) - def test_construct_headers_unique(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), - ], - unique=True, purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) cli_test_utils.streamline_headers(expected) @@ -904,10 +426,30 @@ def test_construct_headers_unique(self, test_input, expected): class TestPURLCLI_urls(object): - def test_urls_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_cli(self, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", + "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", + "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", + "WARNING - 'abcdefg' not valid\n", + "WARNING - 'pkg/abc' not valid\n", + ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output.json" ) @@ -973,7 +515,6 @@ def test_urls_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1000,107 +541,30 @@ def test_urls_cli(self): for output, expected in result_objects: assert output == expected - def test_urls_cli_unique(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_cli_head(self, mock_read_log_file): """ Test the `urls` command with actual and expected JSON output files. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_urls_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_urls_output_unique.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:pypi/dejacode", - "--purl", - "pkg:pypi/dejacode@5.0.0", - "--purl", - "pkg:pypi/dejacode@5.0.0?os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:cargo/socksprox", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", - "pkg:gem/bundler-sass", - "--purl", - "pkg:rubygems/bundler-sass", - "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", - "pkg:nuget/auth0-aspnet@1.1.0", - "--output", - actual_result_file, - "--unique", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - (output_data["packages"], expected_data["packages"]), + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command\n", + "WARNING - 'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `urls` command\n", + "WARNING - 'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/bundler-sass'\n", + "WARNING - 'pkg:pypi/matchcode' does not exist in the upstream repo\n", + "WARNING - 'abcdefg' not valid\n", + "WARNING - 'pkg/abc' not valid\n", ] - for output, expected in result_objects: - assert output == expected - - def test_urls_cli_head(self): - """ - Test the `urls` command with actual and expected JSON output files. - """ expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head.json" ) @@ -1167,7 +631,6 @@ def test_urls_cli_head(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1228,275 +691,102 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_urls_details(self, mock_read_log_file): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command\n", + "WARNING - 'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'\n", + ] + + input_purls = [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ] + output = "" + file = "" + command_name = "urls" + head = False + + purl_urls_data = purlcli.get_urls_details( + input_purls, + output, + file, + head, + command_name, + ) + + expected_data = { + "headers": [ { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, - }, + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [ + "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", + "'check_existence' is not supported for 'pkg:github/istio/istio@1.20.2'", ], - }, - ), - ( - ["pkg:pypi/fetchcode@10.0.0"], + } + ], + "packages": [ { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode@10.0.0"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode@10.0.0"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@10.0.0' does not exist in the upstream repo", - ], - } + "purl": "pkg:pypi/fetchcode@0.3.0", + "download_url": {"url": None}, + "inferred_urls": [ + {"url": "https://pypi.org/project/fetchcode/0.3.0/"} ], - "packages": [], + "repo_download_url": {"url": None}, + "repo_download_url_by_package_type": {"url": None}, + "repo_url": {"url": "https://pypi.org/project/fetchcode/0.3.0/"}, }, - ), - ( - ["pkg:gem/bundler-sass"], { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - }, + "purl": "pkg:gem/bundler@2.3.23", + "download_url": { + "url": "https://rubygems.org/downloads/bundler-2.3.23.gem" + }, + "inferred_urls": [ + {"url": "https://rubygems.org/gems/bundler/versions/2.3.23"}, + {"url": "https://rubygems.org/downloads/bundler-2.3.23.gem"}, ], + "repo_download_url": {"url": None}, + "repo_download_url_by_package_type": {"url": None}, + "repo_url": { + "url": "https://rubygems.org/gems/bundler/versions/2.3.23" + }, }, - ), - ( - ["pkg:rubygems/bundler-sass"], { - "headers": [ + "purl": "pkg:github/istio/istio@1.20.2", + "download_url": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "inferred_urls": [ + {"url": "https://github.com/istio/istio/tree/1.20.2"}, { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:rubygems/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" }, ], + "repo_download_url": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "repo_download_url_by_package_type": { + "url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz" + }, + "repo_url": {"url": "https://github.com/istio/istio/tree/1.20.2"}, }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "urls", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `urls` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "urls", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "urls", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "urls", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } - ], - "packages": [], - }, - ), - ], - ) - def test_urls_details(self, test_input, expected): - """ - Test the `urls` nested function, `get_urls_details()`. - """ - purl_urls = purlcli.get_urls_details( - test_input, - output="", - file="", - command_name="urls", - head=False, - unique=False, - ) - cli_test_utils.streamline_headers(expected["headers"]) - cli_test_utils.streamline_headers(purl_urls["headers"]) + ], + } - assert purl_urls == expected + assert purl_urls_data == expected_data @pytest.mark.parametrize( "test_input,expected", @@ -1511,141 +801,76 @@ def test_urls_details(self, test_input, expected): ), ( ["pkg:rubygems/bundler-sass"], - None, - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_urls_purl(self, test_input, expected): - purl_urls = purlcli.check_urls_purl(test_input[0]) - assert purl_urls == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["https://pypi.org/project/fetchcode/"], - {"get_request": 200, "head_request": 200}, - ), - ( - [None], - {"get_request": "N/A", "head_request": "N/A"}, - ), - ( - ["https://crates.io/crates/banquo"], - {"get_request": 404, "head_request": 404}, - ), - ( - ["https://crates.io/crates/socksprox"], - {"get_request": 404, "head_request": 404}, + "check_existence_not_supported", ), ( - ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], - {"get_request": 200, "head_request": 404}, - ), - ], - ) - def test_make_head_request(self, test_input, expected): - purl_status_code = purlcli.make_head_request(test_input[0]) - - assert purl_status_code == expected - - -class TestPURLCLI_validate(object): - def test_validate_cli(self): - """ - Test the `validate` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_validate_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.validate, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - - result_objects = [ + ["pkg:nginx/nginx"], + "valid_but_not_supported", + ), ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], + ["pkg:pypi/zzzzz"], + "not_in_upstream_repo", ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], + ["pkg:pypi/?fetchcode"], + "not_valid", ), ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], + ["zzzzz"], + "not_valid", ), + ], + ) + def test_check_urls_purl(self, test_input, expected): + purl_urls = purlcli.check_urls_purl(test_input[0]) + assert purl_urls == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], + ["https://pypi.org/project/fetchcode/"], + {"get_request": 200, "head_request": 200}, ), ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], + [None], + {"get_request": "N/A", "head_request": "N/A"}, ), ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], + ["https://crates.io/crates/banquo"], + {"get_request": 404, "head_request": 404}, ), - (output_data["packages"], expected_data["packages"]), - ] + ( + ["https://crates.io/crates/socksprox"], + {"get_request": 404, "head_request": 404}, + ), + ( + ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], + {"get_request": 200, "head_request": 404}, + ), + ], + ) + def test_make_head_request(self, test_input, expected): + purl_status_code = purlcli.make_head_request(test_input[0]) + + assert purl_status_code == expected - for output, expected in result_objects: - assert output == expected - def test_validate_cli_unique(self): +class TestPURLCLI_validate(object): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_validate_cli(self, mock_read_log_file): """ - Test the `validate` command with actual and expected JSON output files - with the `--unique` flag included in the command. + Test the `validate` command with actual and expected JSON output files. """ + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'check_existence' is not supported for 'pkg:nginx/nginx'\n", + "WARNING - 'check_existence' is not supported for 'pkg:rubygems/rails'\n", + ] + expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output_unique.json" + "purlcli/expected_validate_output.json" ) actual_result_file = test_env.get_temp_file("actual_validate_output.json") options = [ @@ -1669,7 +894,6 @@ def test_validate_cli_unique(self): "pkg:rubygems/rails", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.validate, options, catch_exceptions=False) @@ -1686,7 +910,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1707,10 +930,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), (output_data["packages"], expected_data["packages"]), ] @@ -1869,86 +1088,19 @@ def test_validate_purl_strip(self, test_input, expected): class TestPURLCLI_versions(object): - def test_versions_cli(self): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_versions_cli(self, mock_read_log_file): """ Test the `versions` command with actual and expected JSON output files. """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_versions_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_versions_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:hex/coherence@0.1.0", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), + mock_read_log_file.return_value = [ + "WARNING - 'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo\n", + "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", ] - for output, expected in result_objects: - assert output == expected - - compare_packages(expected_data, output_data, "versions") - - def test_versions_cli_unique(self): - """ - Test the `versions` command with actual and expected JSON output files - with the `--unique` flag included in the command. - """ expected_result_file = test_env.get_test_loc( - "purlcli/expected_versions_output_unique.json" + "purlcli/expected_versions_output.json" ) actual_result_file = test_env.get_temp_file("actual_versions_output.json") options = [ @@ -1970,7 +1122,6 @@ def test_versions_cli_unique(self): "pkg:hex/coherence@0.1.0", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) @@ -1989,7 +1140,6 @@ def test_versions_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -2010,22 +1160,26 @@ def test_versions_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), ] for output, expected in result_objects: assert output == expected - compare_packages(expected_data, output_data, "versions") + compare_packages(expected_data, output_data) @pytest.mark.parametrize( "test_input,expected", [ ( - ["pkg:pypi/fetchcode"], + [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:rubygems/bundler-sass", + "pkg:nginx/nginx", + "pkg:pypi/zzzzz", + "pkg:pypi/?fetchcode", + "zzzzz", + ], { "headers": [ { @@ -2033,197 +1187,78 @@ def test_versions_cli_unique(self): "tool_version": "0.2.0", "options": { "command": "versions", - "--purl": ["pkg:pypi/fetchcode"], + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:rubygems/bundler-sass", + "pkg:nginx/nginx", + "pkg:pypi/zzzzz", + "pkg:pypi/?fetchcode", + "zzzzz", + ], "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "release_date": "2024-03-12 07:01:29.239299+00:00", - "version": "0.4.0", - }, + "warnings": [ + "'pkg:rubygems/bundler-sass' not supported with `versions` command", + "'pkg:nginx/nginx' not supported with `versions` command", + "'pkg:pypi/zzzzz' does not exist in the upstream repo", + "'pkg:pypi/?fetchcode' not valid", + "'zzzzz' not valid", ], } ], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:gem/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:gem/bundler-sass"], - "errors": [], - "warnings": [], - } - ], "packages": [ { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], - } - ], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.1.0", + "version": "0.1.0", + "release_date": "2021-08-25", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:nginx/nginx"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:nginx/nginx"], - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.2.0", + "version": "0.2.0", + "release_date": "2022-09-14", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/zzzzz"], - "errors": [], - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.3.0", + "version": "0.3.0", + "release_date": "2023-12-18", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/?fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/?fetchcode"], - "errors": [], - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode@0.4.0", + "release_date": "2024-03-12", + "version": "0.4.0", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["zzzzz"], - "errors": [], - "warnings": ["'zzzzz' not valid"], - } + "purl": "pkg:gem/bundler-sass@0.1.2", + "version": "0.1.2", + "release_date": "2013-12-11", + }, ], - "packages": [], }, ), ], ) - def test_versions_details(self, test_input, expected): + @mock.patch("purldb_toolkit.purlcli.read_log_file") + def test_versions_details(self, mock_read_log_file, test_input, expected): + mock_read_log_file.return_value = [ + "WARNING - 'pkg:rubygems/bundler-sass' not supported with `versions` command\n", + "WARNING - 'pkg:nginx/nginx' not supported with `versions` command\n", + "WARNING - 'pkg:pypi/zzzzz' does not exist in the upstream repo\n", + "WARNING - 'pkg:pypi/?fetchcode' not valid\n", + "WARNING - 'zzzzz' not valid\n", + ] + output = "" file = "" command_name = "versions" - unique = False purl_versions = purlcli.get_versions_details( - test_input, output, file, unique, command_name + test_input, + output, + file, + command_name, ) cli_test_utils.streamline_headers(purl_versions["headers"]) @@ -2231,7 +1266,7 @@ def test_versions_details(self, test_input, expected): assert purl_versions["headers"] == expected["headers"] - compare_packages(expected, purl_versions, "versions") + compare_packages(expected, purl_versions) @pytest.mark.parametrize( "test_input,expected", @@ -2284,7 +1319,7 @@ def streamline_metadata_packages(packages): hle.pop("download_url", None) -def compare_packages(expected_data, actual_data, nested_key): +def compare_packages(expected_data, actual_data): """ Compare the expected and actual data nested inside the `packages` field returned from a live-fetch query and assert that expected data from prior @@ -2301,7 +1336,4 @@ def compare_packages(expected_data, actual_data, nested_key): actual.append(actual_pkg) for expected_entry in expected: - for actual_entry in actual: - if expected_entry["purl"] == actual_entry["purl"]: - for expected_sibling in expected_entry[nested_key]: - assert expected_sibling in actual_entry[nested_key] + assert expected_entry in actual