From 34de9ca848cce92bf3bdbe4a1d9e05ff9e794de3 Mon Sep 17 00:00:00 2001 From: J08nY Date: Fri, 19 Jul 2024 14:49:58 +0200 Subject: [PATCH 1/6] Temporarily disable broken CC USA scheme test. --- tests/cc/test_cc_schemes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py index a9e2105c..5ae5a180 100644 --- a/tests/cc/test_cc_schemes.py +++ b/tests/cc/test_cc_schemes.py @@ -178,6 +178,7 @@ def test_turkey(): @pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException) def test_usa(): + pytest.skip() certified = CCSchemes.get_usa_certified() assert len(certified) != 0 assert absolute_urls(certified) From e42f5ca573a971af80c332453341e7ff84c6b09b Mon Sep 17 00:00:00 2001 From: J08nY Date: Fri, 19 Jul 2024 16:00:03 +0200 Subject: [PATCH 2/6] Move to new dgst algorithm for CC. --- src/sec_certs/dataset/cc.py | 23 +++++-- src/sec_certs/sample/cc.py | 58 +++++++++++------- src/sec_certs/utils/sanitization.py | 9 +++ tests/cc/test_cc_analysis.py | 8 +-- tests/cc/test_cc_dataset.py | 44 ++++++------- tests/cc/test_cc_maintenance_updates.py | 4 +- ...276cca70fd723.pdf => ed91ff3e658457fd.pdf} | Bin ...276cca70fd723.txt => ed91ff3e658457fd.txt} | 0 ...276cca70fd723.pdf => ed91ff3e658457fd.pdf} | Bin ...276cca70fd723.txt => ed91ff3e658457fd.txt} | 0 tests/data/cc/analysis/reference_dataset.json | 6 +- .../transitive_vulnerability_dataset.json | 6 +- .../data/cc/analysis/vulnerable_dataset.json | 4 +- tests/data/cc/certificate/fictional_cert.json | 2 +- .../maintenances/maintenance_updates.json | 4 +- ...2dcf17.txt => report_e3dcf91ef38ddbf0.txt} | 0 ...2dcf17.txt => target_e3dcf91ef38ddbf0.txt} | 0 tests/data/cc/dataset/toy_dataset.json | 6 +- 18 files changed, 106 insertions(+), 68 deletions(-) rename tests/data/cc/analysis/certs/reports/pdf/{ebd276cca70fd723.pdf => ed91ff3e658457fd.pdf} (100%) rename tests/data/cc/analysis/certs/reports/txt/{ebd276cca70fd723.txt => ed91ff3e658457fd.txt} (100%) rename tests/data/cc/analysis/certs/targets/pdf/{ebd276cca70fd723.pdf => ed91ff3e658457fd.pdf} (100%) rename tests/data/cc/analysis/certs/targets/txt/{ebd276cca70fd723.txt => ed91ff3e658457fd.txt} (100%) rename tests/data/cc/dataset/{report_309ac2fd7f2dcf17.txt => report_e3dcf91ef38ddbf0.txt} (100%) rename tests/data/cc/dataset/{target_309ac2fd7f2dcf17.txt => target_e3dcf91ef38ddbf0.txt} (100%) diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py index 8e6ecebb..ebffd912 100644 --- a/src/sec_certs/dataset/cc.py +++ b/src/sec_certs/dataset/cc.py @@ -34,7 +34,7 @@ from sec_certs.sample.cc_scheme import EntryType from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.serialization.json import ComplexSerializableType, serialize -from sec_certs.utils import helpers +from sec_certs.utils import helpers, sanitization from sec_certs.utils import parallel_processing as cert_processing from sec_certs.utils.profiling import staged @@ -368,7 +368,14 @@ def map_ip_to_hostname(url: str) -> str: return CCDataset.BASE_URL + relative_path def _get_primary_key_str(row: Tag): - return row["category"] + row["cert_name"] + row["report_link"] + return "|".join( + [ + row["category"], + row["cert_name"], + sanitization.sanitize_link_fname(row["report_link"]) or "None", + sanitization.sanitize_link_fname(row["st_link"]) or "None", + ] + ) cert_status = "active" if "active" in str(file) else "archived" @@ -408,11 +415,15 @@ def _get_primary_key_str(row: Tag): df_base = df.loc[~df.is_maintenance].copy() df_main = df.loc[df.is_maintenance].copy() - df_base.report_link = df_base.report_link.map(map_ip_to_hostname) - df_base.st_link = df_base.st_link.map(map_ip_to_hostname) + df_base.report_link = df_base.report_link.map(map_ip_to_hostname).map(sanitization.sanitize_link) + df_base.st_link = df_base.st_link.map(map_ip_to_hostname).map(sanitization.sanitize_link) - df_main.maintenance_report_link = df_main.maintenance_report_link.map(map_ip_to_hostname) - df_main.maintenance_st_link = df_main.maintenance_st_link.map(map_ip_to_hostname) + df_main.maintenance_report_link = df_main.maintenance_report_link.map(map_ip_to_hostname).map( + sanitization.sanitize_link + ) + df_main.maintenance_st_link = df_main.maintenance_st_link.map(map_ip_to_hostname).map( + sanitization.sanitize_link + ) n_all = len(df_base) n_deduplicated = len(df_base.drop_duplicates(subset=["dgst"])) diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py index 9f8ba2bd..5354ab3b 100644 --- a/src/sec_certs/sample/cc.py +++ b/src/sec_certs/sample/cc.py @@ -15,7 +15,6 @@ import sec_certs.utils.extract import sec_certs.utils.pdf -import sec_certs.utils.sanitization from sec_certs import constants from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan from sec_certs.configuration import config @@ -27,7 +26,7 @@ from sec_certs.sample.sar import SAR from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.utils import helpers +from sec_certs.utils import helpers, sanitization from sec_certs.utils.extract import normalize_match_string, scheme_frontpage_functions @@ -57,16 +56,10 @@ class MaintenanceReport(ComplexSerializableType): maintenance_st_link: str | None def __post_init__(self): - super().__setattr__( - "maintenance_report_link", sec_certs.utils.sanitization.sanitize_cc_link(self.maintenance_report_link) - ) - super().__setattr__( - "maintenance_st_link", sec_certs.utils.sanitization.sanitize_cc_link(self.maintenance_st_link) - ) - super().__setattr__( - "maintenance_title", sec_certs.utils.sanitization.sanitize_string(self.maintenance_title) - ) - super().__setattr__("maintenance_date", sec_certs.utils.sanitization.sanitize_date(self.maintenance_date)) + super().__setattr__("maintenance_report_link", sanitization.sanitize_link(self.maintenance_report_link)) + super().__setattr__("maintenance_st_link", sanitization.sanitize_link(self.maintenance_st_link)) + super().__setattr__("maintenance_title", sanitization.sanitize_string(self.maintenance_title)) + super().__setattr__("maintenance_date", sanitization.sanitize_date(self.maintenance_date)) @classmethod def from_dict(cls, dct: dict) -> CCCertificate.MaintenanceReport: @@ -420,20 +413,20 @@ def __init__( self.status = status self.category = category - self.name = sec_certs.utils.sanitization.sanitize_string(name) + self.name = sanitization.sanitize_string(name) self.manufacturer = None if manufacturer: - self.manufacturer = sec_certs.utils.sanitization.sanitize_string(manufacturer) + self.manufacturer = sanitization.sanitize_string(manufacturer) self.scheme = scheme - self.security_level = sec_certs.utils.sanitization.sanitize_security_levels(security_level) - self.not_valid_before = sec_certs.utils.sanitization.sanitize_date(not_valid_before) - self.not_valid_after = sec_certs.utils.sanitization.sanitize_date(not_valid_after) - self.report_link = sec_certs.utils.sanitization.sanitize_cc_link(report_link) - self.st_link = sec_certs.utils.sanitization.sanitize_cc_link(st_link) - self.cert_link = sec_certs.utils.sanitization.sanitize_cc_link(cert_link) - self.manufacturer_web = sec_certs.utils.sanitization.sanitize_link(manufacturer_web) + self.security_level = sanitization.sanitize_security_levels(security_level) + self.not_valid_before = sanitization.sanitize_date(not_valid_before) + self.not_valid_after = sanitization.sanitize_date(not_valid_after) + self.report_link = sanitization.sanitize_link(report_link) + self.st_link = sanitization.sanitize_link(st_link) + self.cert_link = sanitization.sanitize_link(cert_link) + self.manufacturer_web = sanitization.sanitize_link(manufacturer_web) self.protection_profiles = protection_profiles self.maintenance_updates = maintenance_updates self.state = state if state else self.InternalState() @@ -445,6 +438,29 @@ def dgst(self) -> str: """ Computes the primary key of the sample using first 16 bytes of SHA-256 digest """ + if not (self.name is not None and self.category is not None): + raise RuntimeError("Certificate digest can't be computed, because information is missing.") + return helpers.get_first_16_bytes_sha256( + "|".join( + [ + self.category, + self.name, + sanitization.sanitize_link_fname(self.report_link) or "None", + sanitization.sanitize_link_fname(self.st_link) or "None", + ] + ) + ) + + @property + def old_dgst(self) -> str: + if not (self.name is not None and self.report_link is not None and self.category is not None): + raise RuntimeError("Certificate digest can't be computed, because information is missing.") + return helpers.get_first_16_bytes_sha256( + self.category + self.name + sanitization.sanitize_cc_link(self.report_link) # type: ignore + ) + + @property + def older_dgst(self) -> str: if not (self.name is not None and self.report_link is not None and self.category is not None): raise RuntimeError("Certificate digest can't be computed, because information is missing.") return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link) diff --git a/src/sec_certs/utils/sanitization.py b/src/sec_certs/utils/sanitization.py index fd477746..4a7326eb 100644 --- a/src/sec_certs/utils/sanitization.py +++ b/src/sec_certs/utils/sanitization.py @@ -3,6 +3,8 @@ import html import logging from datetime import date +from pathlib import Path +from urllib.parse import urlparse import numpy as np import pandas as pd @@ -23,6 +25,13 @@ def sanitize_link(record: str | None) -> str | None: return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://") +def sanitize_link_fname(record: str | None) -> str | None: + if not record: + return None + parsed = urlparse(record) + return Path(parsed.path).name + + def sanitize_cc_link(record: str | None) -> str | None: record = sanitize_link(record) if not record: diff --git a/tests/cc/test_cc_analysis.py b/tests/cc/test_cc_analysis.py index 3984a72e..ce83d705 100644 --- a/tests/cc/test_cc_analysis.py +++ b/tests/cc/test_cc_analysis.py @@ -53,7 +53,7 @@ def transitive_vulnerability_dataset(analysis_data_dir) -> CCDataset: @pytest.fixture def random_certificate(processed_cc_dset: CCDataset) -> CCCertificate: - return processed_cc_dset["ebd276cca70fd723"] + return processed_cc_dset["ed91ff3e658457fd"] def test_match_cpe(random_certificate: CCCertificate): @@ -162,7 +162,7 @@ def test_single_record_references_heuristics(random_certificate: CCCertificate): def test_reference_dataset(reference_dataset: CCDataset): reference_dataset._compute_references() - test_cert = reference_dataset["692e91451741ef49"] + test_cert = reference_dataset["d1b238729b25d745"] assert test_cert.heuristics.report_references.directly_referenced_by == {"BSI-DSZ-CC-0370-2006"} assert test_cert.heuristics.report_references.indirectly_referenced_by == { @@ -175,12 +175,12 @@ def test_reference_dataset(reference_dataset: CCDataset): def test_direct_transitive_vulnerability_dataset(transitive_vulnerability_dataset: CCDataset): transitive_vulnerability_dataset._compute_transitive_vulnerabilities() - assert transitive_vulnerability_dataset["d0705c9e6fbaeba3"].heuristics.direct_transitive_cves == {"CVE-2013-5385"} + assert transitive_vulnerability_dataset["11f77cb31b931a57"].heuristics.direct_transitive_cves == {"CVE-2013-5385"} def test_indirect_transitive_vulnerability_dataset(transitive_vulnerability_dataset: CCDataset): transitive_vulnerability_dataset._compute_transitive_vulnerabilities() - assert transitive_vulnerability_dataset["d0705c9e6fbaeba3"].heuristics.indirect_transitive_cves == {"CVE-2013-5385"} + assert transitive_vulnerability_dataset["11f77cb31b931a57"].heuristics.indirect_transitive_cves == {"CVE-2013-5385"} def test_sar_object(): diff --git a/tests/cc/test_cc_dataset.py b/tests/cc/test_cc_dataset.py index 9d1e4022..bc1433aa 100644 --- a/tests/cc/test_cc_dataset.py +++ b/tests/cc/test_cc_dataset.py @@ -11,22 +11,24 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path): + for cert in toy_dataset: + print(cert.dgst, cert.old_dgst, cert.older_dgst) template_report_pdf_hashes = { - "309ac2fd7f2dcf17": "774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52", - "8cf86948f02f047d": "533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a", - "8a5e6bcda602920c": "e277151e4b279085cd3041ce914ffb3942b43e5ace911c557ad6b8ed764a4ece", + "e3dcf91ef38ddbf0": "774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52", + "ed7611868f0f9d97": "533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a", + "8f08cacb49a742fb": "e277151e4b279085cd3041ce914ffb3942b43e5ace911c557ad6b8ed764a4ece", } template_st_pdf_hashes = { - "309ac2fd7f2dcf17": "b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9", - "8cf86948f02f047d": "3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be", - "8a5e6bcda602920c": "fcee91f09bb72a6526a1f94d0ab754a6db3fbe3ba5773cd372df19788bb25292", + "e3dcf91ef38ddbf0": "b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9", + "ed7611868f0f9d97": "3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be", + "8f08cacb49a742fb": "fcee91f09bb72a6526a1f94d0ab754a6db3fbe3ba5773cd372df19788bb25292", } template_cert_pdf_hashes = { - "309ac2fd7f2dcf17": "9d38bca310c4d349cc39471e0b75d939cc275db9a75b07b8a365d719cfbedcc5", - "8cf86948f02f047d": None, - "8a5e6bcda602920c": "4ba78f26f505819183256ca5a6b404fa90c750fe160c41791e4c400f64e2f6d5", + "e3dcf91ef38ddbf0": "9d38bca310c4d349cc39471e0b75d939cc275db9a75b07b8a365d719cfbedcc5", + "ed7611868f0f9d97": None, + "8f08cacb49a742fb": "4ba78f26f505819183256ca5a6b404fa90c750fe160c41791e4c400f64e2f6d5", } with TemporaryDirectory() as td: @@ -34,14 +36,14 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path): toy_dataset.download_all_artifacts() if not ( - toy_dataset["309ac2fd7f2dcf17"].state.report.download_ok - or toy_dataset["309ac2fd7f2dcf17"].state.st.download_ok - or toy_dataset["309ac2fd7f2dcf17"].state.cert.download_ok - or toy_dataset["8cf86948f02f047d"].state.report.download_ok - or toy_dataset["8cf86948f02f047d"].state.st.download_ok - or toy_dataset["8a5e6bcda602920c"].state.report.download_ok - or toy_dataset["8a5e6bcda602920c"].state.st.download_ok - or toy_dataset["8a5e6bcda602920c"].state.cert.download_ok + toy_dataset["e3dcf91ef38ddbf0"].state.report.download_ok + or toy_dataset["e3dcf91ef38ddbf0"].state.st.download_ok + or toy_dataset["e3dcf91ef38ddbf0"].state.cert.download_ok + or toy_dataset["ed7611868f0f9d97"].state.report.download_ok + or toy_dataset["ed7611868f0f9d97"].state.st.download_ok + or toy_dataset["8f08cacb49a742fb"].state.report.download_ok + or toy_dataset["8f08cacb49a742fb"].state.st.download_ok + or toy_dataset["8f08cacb49a742fb"].state.cert.download_ok ): pytest.xfail(reason="Fail due to error during download") @@ -60,15 +62,15 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path): if cert.cert_link: assert cert.state.cert.txt_path.exists() - template_report_txt_path = data_dir / "report_309ac2fd7f2dcf17.txt" - template_st_txt_path = data_dir / "target_309ac2fd7f2dcf17.txt" + template_report_txt_path = data_dir / "report_e3dcf91ef38ddbf0.txt" + template_st_txt_path = data_dir / "target_e3dcf91ef38ddbf0.txt" assert ( - abs(toy_dataset["309ac2fd7f2dcf17"].state.st.txt_path.stat().st_size - template_st_txt_path.stat().st_size) + abs(toy_dataset["e3dcf91ef38ddbf0"].state.st.txt_path.stat().st_size - template_st_txt_path.stat().st_size) < 1000 ) assert ( abs( - toy_dataset["309ac2fd7f2dcf17"].state.report.txt_path.stat().st_size + toy_dataset["e3dcf91ef38ddbf0"].state.report.txt_path.stat().st_size - template_report_txt_path.stat().st_size ) < 1000 diff --git a/tests/cc/test_cc_maintenance_updates.py b/tests/cc/test_cc_maintenance_updates.py index c5054002..9c89c748 100644 --- a/tests/cc/test_cc_maintenance_updates.py +++ b/tests/cc/test_cc_maintenance_updates.py @@ -41,7 +41,7 @@ def test_methods_not_meant_to_be_implemented(): def test_download_artifacts(mu_dset: CCDatasetMaintenanceUpdates): # Conversion and extraction is identical to CC, will not test. mu_dset.download_all_artifacts() - mu = mu_dset["cert_8a5e6bcda602920c_update_559ed93dd80320b5"] + mu = mu_dset["cert_8f08cacb49a742fb_update_559ed93dd80320b5"] if not (mu.state.report.download_ok or mu.state.st.download_ok): pytest.xfail(reason="Fail due to error on CC server.") @@ -82,4 +82,4 @@ def test_from_web(): dset = CCDatasetMaintenanceUpdates.from_web_latest() assert dset is not None assert len(dset) >= 492 # Contents as of November 2022, maintenances should not disappear - assert "cert_8a5e6bcda602920c_update_559ed93dd80320b5" in dset # random cert verified to be present + assert "cert_8f08cacb49a742fb_update_559ed93dd80320b5" in dset # random cert verified to be present diff --git a/tests/data/cc/analysis/certs/reports/pdf/ebd276cca70fd723.pdf b/tests/data/cc/analysis/certs/reports/pdf/ed91ff3e658457fd.pdf similarity index 100% rename from tests/data/cc/analysis/certs/reports/pdf/ebd276cca70fd723.pdf rename to tests/data/cc/analysis/certs/reports/pdf/ed91ff3e658457fd.pdf diff --git a/tests/data/cc/analysis/certs/reports/txt/ebd276cca70fd723.txt b/tests/data/cc/analysis/certs/reports/txt/ed91ff3e658457fd.txt similarity index 100% rename from tests/data/cc/analysis/certs/reports/txt/ebd276cca70fd723.txt rename to tests/data/cc/analysis/certs/reports/txt/ed91ff3e658457fd.txt diff --git a/tests/data/cc/analysis/certs/targets/pdf/ebd276cca70fd723.pdf b/tests/data/cc/analysis/certs/targets/pdf/ed91ff3e658457fd.pdf similarity index 100% rename from tests/data/cc/analysis/certs/targets/pdf/ebd276cca70fd723.pdf rename to tests/data/cc/analysis/certs/targets/pdf/ed91ff3e658457fd.pdf diff --git a/tests/data/cc/analysis/certs/targets/txt/ebd276cca70fd723.txt b/tests/data/cc/analysis/certs/targets/txt/ed91ff3e658457fd.txt similarity index 100% rename from tests/data/cc/analysis/certs/targets/txt/ebd276cca70fd723.txt rename to tests/data/cc/analysis/certs/targets/txt/ed91ff3e658457fd.txt diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json index 38fb7fd0..04be80ca 100644 --- a/tests/data/cc/analysis/reference_dataset.json +++ b/tests/data/cc/analysis/reference_dataset.json @@ -15,7 +15,7 @@ "certs": [ { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "c30de3192d2e8ec2", + "dgst": "3129688580711e08", "status": "archived", "category": "Other Devices and Systems", "name": "Océ Digital Access Controller (DAC) R10.1.5 for use in the Océ VarioPrint 1055, 1055 BC, 1055 DP, 1065, 1075, 2062, 2075, 2075 DP printer/copier/scanner products", @@ -583,7 +583,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "53fe111411edfa45", + "dgst": "2c47b65953dcffb3", "status": "archived", "category": "Other Devices and Systems", "name": "Océ Digital Access Controller (DAC) R9.1.6", @@ -1229,7 +1229,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "692e91451741ef49", + "dgst": "d1b238729b25d745", "status": "archived", "category": "Other Devices and Systems", "name": "Océ Digital Access Controller R8.1.10", diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json index abd4c7a3..5f052c11 100644 --- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json +++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json @@ -15,7 +15,7 @@ "certs": [ { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "d0705c9e6fbaeba3", + "dgst": "11f77cb31b931a57", "status": "active", "category": "Operating Systems", "name": "IBM z/OS Version 2 Release 1", @@ -1339,7 +1339,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "011796336c7b94de", + "dgst": "487cf9415b61b49f", "status": "archived", "category": "Operating Systems", "name": "RACF Element of z/OS Version 2, Release 1", @@ -2288,7 +2288,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "ebc77980250ee68f", + "dgst": "c310425745136fdd", "status": "active", "category": "Operating Systems", "name": "IBM z/OS Version 2 Release 2", diff --git a/tests/data/cc/analysis/vulnerable_dataset.json b/tests/data/cc/analysis/vulnerable_dataset.json index 7978cb5a..cbabe14d 100644 --- a/tests/data/cc/analysis/vulnerable_dataset.json +++ b/tests/data/cc/analysis/vulnerable_dataset.json @@ -16,7 +16,7 @@ "certs": [ { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "ebd276cca70fd723", + "dgst": "ed91ff3e658457fd", "status": "active", "category": "Access Control Devices and Systems", "name": "IBM Security Access Manager for Enterprise Single Sign-On Version 8.2", @@ -97,7 +97,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "37e1b22e5933b0ed", + "dgst": "95e3850bef32f410", "status": "active", "category": "Access Control Devices and Systems", "name": "IBM WebSphere Application Server (WAS) 7.0", diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json index 54781dd5..32475e69 100644 --- a/tests/data/cc/certificate/fictional_cert.json +++ b/tests/data/cc/certificate/fictional_cert.json @@ -1,6 +1,6 @@ { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "a9ccb81a92e547dc", + "dgst": "8049938203b26f7b", "status": "archived", "category": "Sample category", "name": "Sample certificate name", diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json index 5596b597..38c3db5b 100644 --- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json +++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json @@ -16,7 +16,7 @@ "certs": [ { "_type": "sec_certs.sample.cc_maintenance_update.CCMaintenanceUpdate", - "dgst": "cert_8a5e6bcda602920c_update_559ed93dd80320b5", + "dgst": "cert_8f08cacb49a742fb_update_559ed93dd80320b5", "name": "Fortinet FortiGate w/ FortiOS v5.6.7 Build 6022", "report_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20MR%20v1.0e.pdf", "st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20ST%20v1.4%20CCRA.pdf", @@ -93,7 +93,7 @@ "indirect_transitive_cves": null, "scheme_data": null }, - "related_cert_digest": "8a5e6bcda602920c", + "related_cert_digest": "8f08cacb49a742fb", "maintenance_date": "2019-08-26" } ] diff --git a/tests/data/cc/dataset/report_309ac2fd7f2dcf17.txt b/tests/data/cc/dataset/report_e3dcf91ef38ddbf0.txt similarity index 100% rename from tests/data/cc/dataset/report_309ac2fd7f2dcf17.txt rename to tests/data/cc/dataset/report_e3dcf91ef38ddbf0.txt diff --git a/tests/data/cc/dataset/target_309ac2fd7f2dcf17.txt b/tests/data/cc/dataset/target_e3dcf91ef38ddbf0.txt similarity index 100% rename from tests/data/cc/dataset/target_309ac2fd7f2dcf17.txt rename to tests/data/cc/dataset/target_e3dcf91ef38ddbf0.txt diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json index dbf75078..cb47293b 100644 --- a/tests/data/cc/dataset/toy_dataset.json +++ b/tests/data/cc/dataset/toy_dataset.json @@ -16,7 +16,7 @@ "certs": [ { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "309ac2fd7f2dcf17", + "dgst": "e3dcf91ef38ddbf0", "status": "active", "category": "Access Control Devices and Systems", "name": "NetIQ Identity Manager 4.7", @@ -119,7 +119,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "8cf86948f02f047d", + "dgst": "ed7611868f0f9d97", "status": "active", "category": "Access Control Devices and Systems", "name": "Magic SSO V4.0", @@ -227,7 +227,7 @@ }, { "_type": "sec_certs.sample.cc.CCCertificate", - "dgst": "8a5e6bcda602920c", + "dgst": "8f08cacb49a742fb", "status": "active", "category": "Boundary Protection Devices and Systems", "name": "Fortinet FortiGate w/ FortiOS v5.6.7", From 958effe4b47bde59bca32eb64f567365590ba24a Mon Sep 17 00:00:00 2001 From: J08nY Date: Fri, 19 Jul 2024 17:12:58 +0200 Subject: [PATCH 3/6] Fix Spain CC scheme download. --- src/sec_certs/sample/cc_scheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py index 1af70e76..7fb9e18e 100644 --- a/src/sec_certs/sample/cc_scheme.py +++ b/src/sec_certs/sample/cc_scheme.py @@ -1306,7 +1306,7 @@ def get_spain_certified() -> list[dict[str, Any]]: "product_link": urljoin(constants.CC_SPAIN_BASE_URL, tds[0].find("a")["href"]), "category": sns(tds[1].text), "manufacturer": sns(tds[2].text), - "certification_date": sns(tds[3].find("td", class_="djc_value").text), + "certification_date": sns(tds[3].text), } results.append(cert) return results From ed3bac262f8164c6f05a428aa7933fe1545df9b5 Mon Sep 17 00:00:00 2001 From: J08nY Date: Fri, 19 Jul 2024 17:15:20 +0200 Subject: [PATCH 4/6] Fix scheme dataset test. --- tests/cc/test_cc_schemes.py | 2 +- .../auxiliary_datasets/scheme_dataset.json | 856 ++++++++++++++++++ 2 files changed, 857 insertions(+), 1 deletion(-) create mode 100644 tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py index 5ae5a180..9e1e927f 100644 --- a/tests/cc/test_cc_schemes.py +++ b/tests/cc/test_cc_schemes.py @@ -222,4 +222,4 @@ def test_matching(toy_dataset: CCDataset, canada_certified): def test_process_dataset(toy_dataset: CCDataset): toy_dataset.auxiliary_datasets.scheme_dset = toy_dataset.process_schemes(True, only_schemes={"CA"}) toy_dataset._compute_scheme_data() - assert toy_dataset["8a5e6bcda602920c"].heuristics.scheme_data is not None + assert toy_dataset["8f08cacb49a742fb"].heuristics.scheme_data is not None diff --git a/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json b/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json new file mode 100644 index 00000000..64db2e84 --- /dev/null +++ b/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json @@ -0,0 +1,856 @@ +{ + "_type": "sec_certs.dataset.cc_scheme.CCSchemeDataset", + "schemes": { + "CA": { + "_type": "sec_certs.sample.cc_scheme.CCScheme", + "country": "CA", + "timestamp": "2024-07-19T17:14:26.369494", + "lists": { + "INEVALUATION": [ + { + "product": "RICOH Pro C5300S/C5310S, version JE-1.20-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Arbor Edge Defense", + "vendor": "NETSCOUT Systems, Inc.", + "level": "CPP_ND_V2.2E", + "cert_lab": "Lightship Security" + }, + { + "product": "RICOH IM C6500/C8000, version JE-1.20-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Oracle Linux 9.3", + "vendor": "Oracle Corporation", + "level": "PP_OS_V4.3, PKG_TLS_V1.1, PKG_SSH_V1.0", + "cert_lab": "EWA-Canada" + }, + { + "product": "Senetas CN 9000 Series Ethernet Encryptors v5.5.0", + "vendor": "Senetas", + "level": "EAL 2+ (ALC_FLR.2)", + "cert_lab": "Lightship Security" + }, + { + "product": "Xerox® AltaLink™ C8130/C8135/C8145/C8155/C8170 & B8145/B8155/B8170 with SSD", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "RICOH IM C7010, version JE-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Xerox® AltaLink™ C8130/C8135/C8145/C8155/C8170 & B8145/B8155/B8170 with HDD", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "KAYTUS Server Baseboard Management Controller 7.11.00", + "vendor": "KAYTUS Systems Pte Ltd.", + "level": "EAL 2+ (ALC_FLR.2)", + "cert_lab": "EWA-Canada" + }, + { + "product": "Senetas CN 4000/6000 Series Ethernet Encryptors v5.5.0", + "vendor": "Senetas", + "level": "EAL 2+ (ALC_FLR.2)", + "cert_lab": "Lightship Security" + }, + { + "product": "Cisco cEdge Routers running IOS XE 17.12 with SD-WAN 20.12", + "vendor": "Cisco Systems, Inc.", + "level": "EAL 2 + (ALC_FLR.2)", + "cert_lab": "Lightship Security" + }, + { + "product": "Xerox® B410 Printer and C410 Color Printer with Firmware Version 222.030", + "vendor": "Xerox Corporation", + "level": "CPP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Lexmark MX532, MX632, CX532, and CX635 Multi-Function Printers with Firmware Version 222.030", + "vendor": "Lexmark International, Inc.", + "level": "CPP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Oracle Identity Governance 12c", + "vendor": "Oracle Corporation", + "level": "PP_ESM_ICM_V2.1", + "cert_lab": "Lightship Security" + }, + { + "product": "Dell Encryption Personal Edition Version 10.10.4", + "vendor": "Dell Inc.", + "level": "PP_APP_V1.4, MOD_FE_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Lexmark MX532, MX632, CX532 and CX635 Multi-Function Printers with Hard Drive and with Firmware Version 222.030", + "vendor": "Lexmark International, Inc.", + "level": "CPP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Lexmark MS632 and CS632 Single Function Printers with Firmware Version 222.030", + "vendor": "Lexmark International, Inc.", + "level": "CPP_HCD_V1.0", + "cert_lab": "Lightship Security" + }, + { + "product": "Fortinet FortiManager 7.0", + "vendor": "Fortinet, Inc.", + "level": "EAL 4+ (ALC_FLR.3)", + "cert_lab": "EWA-Canada" + }, + { + "product": "Fortinet FortiAnalyzer 7.0.5", + "vendor": "Fortinet, Inc.", + "level": "EAL 4+ (ALC_FLR.3)", + "cert_lab": "EWA-Canada" + }, + { + "product": "NETSCOUT® Omnis™ Cyber Intelligence with Omnis™ CyberStream v6.3.4", + "vendor": "NETSCOUT", + "level": "CPP_ND_V2.2E", + "cert_lab": "Lightship Security" + }, + { + "product": "NetScaler Version 13.1", + "vendor": "Cloud Software Group", + "level": "CPP_ND_2.2E", + "cert_lab": "Lightship Security" + }, + { + "product": "NETSCOUT nGenius for Flows with nGenius Collector v6.3", + "vendor": "NETSCOUT", + "level": "CPP_ND_V2.2E", + "cert_lab": "Lightship Security" + } + ], + "CERTIFIED": [ + { + "product": "High Sec Labs FA10A-4 and FA10AO-4 Firmware Version 40000-0E7 Peripheral Sharing Devices", + "vendor": "High Sec Labs Ltd.", + "level": "PP_PSD_V4.0, MOD_AO_V1.0", + "certification_date": "2024-07-09" + }, + { + "product": "Belkin Secure KVM models F1DN102KVM-HA-3, F1DN202KVM-HA-3, F1DN104KVM-HA-3, F1DN204KVM-HA-3 Version 33303-C6C6", + "vendor": "Belkin International, Inc.", + "level": "PP_PSS_V3.0", + "certification_date": "2024-06-11" + }, + { + "product": "CipherDriveOne 2.0.1", + "vendor": "KLC Group LLC", + "level": "CPP_FDE_AA_V2.0E", + "certification_date": "2024-06-27" + }, + { + "product": "Dell PowerScale OneFS v9.5", + "vendor": "Dell Technologies", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2024-05-22" + }, + { + "product": "Xerox® AltaLink™ EC8036 and EC8056", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "certification_date": "2024-06-25" + }, + { + "product": "Layer7 API Gateway v10.1.00", + "vendor": "Broadcom Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2024-05-24" + }, + { + "product": "Xerox® VersaLink™ C625 and B625 with eMMC", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "certification_date": "2024-05-08" + }, + { + "product": "Xerox® VersaLink™ C625 and B625 with HDD", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "certification_date": "2024-05-14" + }, + { + "product": "Xerox® VersaLink™ C415 and B415 with HDD", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "certification_date": "2024-04-24" + }, + { + "product": "Xerox® VersaLink™ C415 and B415 with eMMC", + "vendor": "Xerox Corporation", + "level": "PP_HCD_V1.0", + "certification_date": "2024-05-03" + }, + { + "product": "High Sec Labs FI11H-M, FI11D-M, FI11PH-M Firmware Version 44404-E7E7 Peripheral Sharing Devices", + "vendor": "High Sec Labs Ltd.", + "level": "PP_PSD_V4.0 with MOD_KM_V1.0, MOD_VI_V1.0", + "certification_date": "2024-04-29" + }, + { + "product": "High Sec Labs SC21H-4, SC82PH-4, SC162PH-4, SMX42D-M, SMX42H-M, SMX42P-M, SMX82D-M, SMX82H-M, SMX82P-M Firmware Version 44404-E7E7 Peripheral Sharing Devices", + "vendor": "High Sec Labs Ltd.", + "level": "PP_PSD_V4.0 with MOD_KM_V1.0, MOD_VI_V1.0", + "certification_date": "2024-04-29" + }, + { + "product": "NETSCOUT® nGeniusONE® with InfiniStreamNG® v6.3.3", + "vendor": "NETSCOUT Systems Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2024-04-04" + }, + { + "product": "Adder AS-4CR Multi-Domain Card Reader Firmware Version 40040-0E7", + "vendor": "Adder Technology", + "level": "PP_PSD_V4.0 with MOD_UA_V1.0", + "certification_date": "2024-01-05" + }, + { + "product": "Oracle VM Server for SPARC 3.6 and Oracle Solaris 11.4", + "vendor": "Oracle Corporation", + "level": "PP_BASE_VIRTUALIZATION_v1.1; MOD_SV_V1.1; PKG_SSH_V1.0; PKG_TLS_V1.1", + "certification_date": "2024-01-25" + }, + { + "product": "RICOH IM 370F/460F version JE-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-12-07" + }, + { + "product": "High Sec Labs SK21PH-4, SK41PH-4, DK22PH-4, DK42PH-4, SK81PH-4, DK82PH-4, SX42PH-4, SX82PH-4 Firmware Version 44404-E7E7 Peripheral Sharing Devices", + "vendor": "High Sec Labs Ltd.", + "level": "PP_PSD_V4.0 with MOD_AO_V1.0, MOD_KM_V1.0, MOD_VI_V1.0", + "certification_date": "2023-11-20" + }, + { + "product": "RICOH IM 370 version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-12-08" + }, + { + "product": "Dell ECS v3.8.0.3", + "vendor": "Dell Technologies", + "level": "EAL2+ (ALC_FLR.2)", + "certification_date": "2023-11-14" + }, + { + "product": "Dell PowerMax with PowerMaxOS 10 Solutions Enabler 10.0, and Unisphere for PowerMax 10.0", + "vendor": "Dell Technologies", + "level": "EAL2+ (ALC_FLR.2)", + "certification_date": "2023-10-20" + }, + { + "product": "BAE Systems STOP™ 8.8.2", + "vendor": "BAE Systems", + "level": "PP_OS_V4.2.1", + "certification_date": "2023-09-15" + }, + { + "product": "RICOH IM 2500/3000/3500/4000/5000/6000 version JE-1.10-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-09-08" + }, + { + "product": "RICOH IM 7000/8000/9000/9000T version JE-1.10-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-09-08" + }, + { + "product": "Oracle Access Management 12c", + "vendor": "Oracle Corporation", + "level": "PP_ESM_PM_V2.1, PP_ESM_AC_V2.1", + "certification_date": "2023-09-20" + }, + { + "product": "RICOH IM C530F / C530FB version E-1.10-H", + "vendor": "Ricoh Company, Ltd", + "level": "PP_HCD_V1.0", + "certification_date": "2023-08-29" + }, + { + "product": "RICOH Pro C5300S/C5310S Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company, Ltd", + "level": "PP_HCD_V1.0", + "certification_date": "2023-07-28" + }, + { + "product": "Lexmark CX730, CX930, CX931 and MX931 Multi-Function Printers with Trusted Platform Module and Hard Drive and without Fax and with Firmware Version 081.234", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-06-23" + }, + { + "product": "Lexmark MX521, MX931, CX730, CX930, and CX931 Multi-Function Printers with Trusted Platform Module, without Fax and Hard Drive and with Firmware Version 081.234", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-06-23" + }, + { + "product": "Lexmark MX421, MX432, MX521, MX622, MX721, MX722, MX725, MX931, CX622, CX625, CX730, CX735, CX930, CX931, CX942, CX943 and CX944 Multi-Function Printers with Trusted Platform Module and Fax and without Hard Drive and with Firmware Version 081.234", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-06-22" + }, + { + "product": "Lexmark MS622, MS822, MS826, CS622, CS730, CS735, CS820, and CS943 Single Function Printers with Trusted Platform Module and Firmware Version 081.234", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-06-23" + }, + { + "product": "RICOH IM C2010/C2510/C2519/C3010/C3510/C3519/C4510/C5510/C6010 version JE-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-06-29" + }, + { + "product": "Lexmark MX432, MX522, MX622, MX721, MX722, MX822, MX826, MX931, CX622, CX625, CX730, CX735, CX820, CX825, CX860, CX930, CX931, CX942, CX943, and CX944 MFPs with TPM, Fax and Hard Drive with firmware version 081.234", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-05-24" + }, + { + "product": "RICOH IM C6500/C8000 Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-04-27" + }, + { + "product": "RICOH IM 2500/3000/3500/4000/5000/6000 Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-05-01" + }, + { + "product": "RICOH IM 7000/8000/9000 Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-05-01" + }, + { + "product": "RICOH IM C300/C300F/C300FLT/C400F/C400SRF/C400FLT Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-04-27" + }, + { + "product": "RICOH IM 550/600/600SR Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-04-27" + }, + { + "product": "RICOH IM C2000 / C2000LT / C2500 / C2500LT / C3000 / C3000LT / C3500 / C3500LT / C4500 /C4500LT / C5500 / C5500LT/ C6000 / C6000LT Enhanced Security Firmware version E-1.00-H", + "vendor": "Ricoh Company Ltd.", + "level": "PP_HCD_V1.0", + "certification_date": "2023-04-27" + }, + { + "product": "Oracle Linux 8.4", + "vendor": "Oracle Corporation", + "level": "PP_OS_V4.2.1, PKG_SSH_V1.0", + "certification_date": "2023-04-12" + }, + { + "product": "TestStream Management Software v5.3.0 on nGenius 3900 Series Switches", + "vendor": "NetScout Systems, Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2023-04-17" + }, + { + "product": "Cisco Catalyst 9800 Series Wireless Controllers and Access Points 17.6", + "vendor": "Cisco Systems, Inc.", + "level": "CPP_ND_V2.2E, PP_WLAN_AS_EP_V1.0", + "certification_date": "2023-03-20" + }, + { + "product": "Oracle Linux 7.6 UEK 5 KVM & Virtualization Manager 4.3", + "vendor": "Oracle Corporation", + "level": "PP_BASE_VIRTUALIZATION_V1.0, EP_SV_V1.0, PP_SSH_EP_V1.0", + "certification_date": "2023-03-03" + }, + { + "product": "High Sec Labs SK41PHU-4, DK42PHU-4, SX42PHU-4, SX82PHU-4, SC42DHU-4, SC42PHU-4 Firmware Version 44444-E7E7 Peripheral Sharing Devices", + "vendor": "High Sec Labs Ltd.", + "level": "PP_PSD_V4.0 with MOD_AO_V1.0, MOD_KM_V1.0, MOD_UA_V1.0, MOD_VI_V1.0", + "certification_date": "2022-12-20" + }, + { + "product": "Adder AVS-4228, AVS-42216, XDS441, XDS441FX Firmware Version 44404-E7E7 Peripheral Sharing Devices\n\t\t\tAdder AVS-4228, AVS-42216 Firmware Version 44404-E7E7 Peripheral Sharing Devices (August 2023)", + "vendor": "Adder Technology", + "level": "PP_PSD_V4.0 with MOD_KM_V1.0, MOD_VI_V1.0", + "certification_date": "2022-12-07" + }, + { + "product": "SentinelOne Singularity Complete Version S", + "vendor": "SentinelOne, Inc.", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-12-19" + }, + { + "product": "Dell EMC Unity OE v5.2", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-12-13" + }, + { + "product": "FortiAnalyzer 6.2.8", + "vendor": "Fortinet, Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2022-12-12" + }, + { + "product": "RICOH IM C300/C300F/C400F/C400SRF version JE-1.10-H", + "vendor": "Ricoh Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2022-11-23" + }, + { + "product": "RICOH IM 550/600/600SR version E-1.10-H", + "vendor": "Ricoh Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2022-11-23" + }, + { + "product": "RICOH IM C5300S/C5310S version JE-1.10-H", + "vendor": "Ricoh Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2022-11-14" + }, + { + "product": "RICOH IM C6500/C8000 version JE-1.10-H", + "vendor": "Ricoh Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2022-11-14" + }, + { + "product": "Fortinet FortiManager 6.2.8", + "vendor": "Fortinet, Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2022-11-07" + }, + { + "product": "Cisco 900 Series Integrated Services Routers running IOS v15.9", + "vendor": "Cisco Systems, Inc.", + "level": "CPP_ND_V2.2E with MOD_VPNGW_v1.1", + "certification_date": "2022-11-02" + }, + { + "product": "RICOH IM C2000/C2000LT/C2500/C2500LT/C3000/C3000LT/C3500/C3500LT/ C4500/C4500LT/C5500/C5500LT/C6000/C6000LT version JE-1.20-H", + "vendor": "Ricoh Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2022-09-28" + }, + { + "product": "Dell EMC™ Data Domain® v7.2", + "vendor": "Dell EMC™", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-09-26" + }, + { + "product": "Citrix Hypervisor® 8.2 LTSR Premium Edition (CU1)", + "vendor": "Citrix Systems, Inc.", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-08-23" + }, + { + "product": "McAfee Endpoint Security 10.7.x with ePolicy Orchestrator 5.10.x", + "vendor": "Trellix", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-07-28" + }, + { + "product": "NetApp StorageGRID 11.5", + "vendor": "NetApp Inc.", + "level": "EAL 2+ (ALC_FLR.1)", + "certification_date": "2022-07-15" + }, + { + "product": "AhnLab CPP 1.0", + "vendor": "AhnLab, Inc.", + "level": "EAL 2+ (ALC_FLR.1)", + "certification_date": "2022-07-14" + }, + { + "product": "nGenius 5000 and 7000 Series Packet Flow Switches with PFOS 6.0.6", + "vendor": "NETSCOUT Systems, Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2022-06-07" + }, + { + "product": "Trend Micro Deep Security 20", + "vendor": "Trend Micro Inc.", + "level": "EAL 2+ (ALC_FLR.1)", + "certification_date": "2022-05-31" + }, + { + "product": "Adder AVS-4112, AVS-2112, AVS-4114, AVS-4214, AVS-2114, AVS-2214, AVS-4128, AVS-4124, AVS-1124, AVS-4224 Firmware Version 44404-E7E7, Peripheral Sharing Devices", + "vendor": "Adder Technology", + "level": "PP_PSD_v4.0, MOD_AO_v1.0, MOD_KM_v1.0, MOD_VI_v1.0", + "certification_date": "2022-04-20" + }, + { + "product": "Keysight Technologies Vision Series Network Packet Broker v5.7.1\n\t\t\tKeysight Technologies Vision Series Network Packet Broker v5.10.0 (January 2024)", + "vendor": "Keysight Technologies", + "level": "CPP_ND_v2.2E", + "certification_date": "2022-03-04" + }, + { + "product": "Belkin F1DN102MOD-xx-4, F1DN202MOD-xx-4, F1DN104MOD-xx-4, F1DN204MOD-xx-4, F1DN108MOD-xx-4, F1DN208MOD-xx-4 Firmware Version 44404-E7E7 Peripheral Sharing Devices", + "vendor": "Belkin International, Inc.", + "level": "PP_PSD_v4.0, MOD_KM_v1.0, MOD_VI_v1.0", + "certification_date": "2021-07-11" + }, + { + "product": "Belkin F1DN002MOD-KM-4, F1DN004MOD-KM-4 and F1DN-FLTR-HID-4 Firmware Version 40404-0E7 Peripheral Sharing Devices", + "vendor": "Belkin International, Inc.", + "level": "PP_PSD_v4.0, MOD_KM_v1.0", + "certification_date": "2022-01-24" + }, + { + "product": "Cisco Web Security Appliance with AsyncOS 11.8", + "vendor": "Cisco Systems, Inc.", + "level": "CPP_ND_v2.2E", + "certification_date": "2022-02-28" + }, + { + "product": "FortiGate/FortiOS Version 6.2.7", + "vendor": "Fortinet, Inc.", + "level": "CPP_ND_V2.2E w/ MOD_CPP_FW_v1.4e, MOD_VPNGW_v1.1,MOD_IPS_V1.0", + "certification_date": "2022-01-14" + }, + { + "product": "Fortinet FortiGate™ Next Generation Firewalls with FortiOS 6.2.7", + "vendor": "Fortinet, Incorporated", + "level": "EAL 4+ (ALC_FLR.3)", + "certification_date": "2021-10-15" + }, + { + "product": "NetApp Element Software 12.2 on SolidFire Appliances", + "vendor": "NetApp, Inc.", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2022-03-08" + }, + { + "product": "Sphyrna Security Unidirectional Gateway - Data Diode Identifier: 2010-UG100-SSI", + "vendor": "Sphyrna Security Incorporated", + "level": "EAL4+ (ADV_INT.2, ALC_CMC.5, ALC_CMS.5, ALC_DVS.2, ALC_FLR.3, ATE_DPT.2 and AVA_VAN.4)", + "certification_date": "2021-09-07" + }, + { + "product": "NetApp E-Series & EF-Series with SANtricity OS 11.70", + "vendor": "NetApp, Inc.", + "level": "NDcPP v2.2e", + "certification_date": "2021-09-21" + }, + { + "product": "Blackline Systems Corporation BSC-CDS Unidirectional Subsystem PN: 710-0185-00", + "vendor": "Blackline Systems Corporation", + "level": "EAL 4+ (ADV_INT.2, ALC_CMC.5, ALC_CMS.5, ALC_DVS.2, ALC_FLR.3, ATE_DPT.2 and AVA_VAN.4)", + "certification_date": "2021-09-15" + }, + { + "product": "RICOH IM C530F/C530FB version E-1.00-H", + "vendor": "RICOH Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-10-13" + }, + { + "product": "RICOH IM 2500/3000/3500/4000/5000/6000 version JE-1.00-H", + "vendor": "RICOH Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-10-18" + }, + { + "product": "RICOH IM 7000/8000/9000/9000T version JE-1.00-H", + "vendor": "RICOH Company, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-10-18" + }, + { + "product": "Cisco Email Security Appliance with AsyncOS 13.0", + "vendor": "Cisco Systems, Inc", + "level": "cPP_ND_v2.1", + "certification_date": "2021-09-20" + }, + { + "product": "Belkin F1DN104KVM-UNN4, F1DN204KVM-UNN4, F1DN102KVM-UNN4, F1DN202KVM-UNN4 Firmware Version 44404-E7E7 Peripheral Sharing Devices\n\t\t\tBelkin F1DN104KVM-UNN4, F1DN204KVM-UNN4, F1DN102KVM-UNN4, F1DN202KVM-UNN4, F1DN104KVM-HA-4, F1DN204KVM-HA-4, F1DN102KVM-HA-4, F1DN202KVM-HA-4 Firmware Version 44404-E7E7 Peripheral Sharing Devices (April 2023)", + "vendor": "Belkin International, Inc.", + "level": "PP_PSD_v4.0, MOD_AO_V1.0, MOD_KM_V1.0, MOD_VI_V1.0", + "certification_date": "2021-07-08" + }, + { + "product": "Belkin F1DN104KVM-UN-4, F1DN204KVM-UN-4, F1DN102KVM-UN-4, F1DN202KVM-UN-4, F1DN108KVM-UN-4, F1DN208KVM-UN-4, F1DN116KVM-UN-4 Firmware Version 44444-E7E7 Peripheral Sharing Devices", + "vendor": "Belkin International, Inc.", + "level": "PP_PSD_V4.0, MOD_AO_V1.0, MOD_KM_V1.0, MOD_UA_V1.0, MOD_VI_V1.0", + "certification_date": "2021-04-29" + }, + { + "product": "Oracle Linux 7.6", + "vendor": "Oracle Corporation", + "level": "PP_OS_V4.2.1 PP_SSH_EP_v1.0", + "certification_date": "2021-07-19" + }, + { + "product": "Crunchy Certified PostgreSQL 12.5", + "vendor": "Crunchy Data Solutions, Inc.", + "level": "DBMS PP Base Package, V2.12", + "certification_date": "2021-03-16" + }, + { + "product": "OPSWAT MetaDefender Core v4.19.0 & MetaDefender Kiosk v4.4.5", + "vendor": "OPSWAT, Inc.", + "level": "EAL 2+ (ALC_FLR.1)", + "certification_date": "2021-03-03" + }, + { + "product": "CipherDrive v1.2.2\n\t\t\tCipherDrive v1.2.3 (January 2023)", + "vendor": "KLC Group LLC", + "level": "CPP_FDE_AA_V2.0E", + "certification_date": "2021-02-17" + }, + { + "product": "Oracle Solaris 11.4", + "vendor": "Oracle Corporation", + "level": "PP_OS_v4.2.1, PP_SSH_EP_v1.0", + "certification_date": "2021-02-08" + }, + { + "product": "Lexmark MS622, MS822, MS826, CS622, CS720, CS725, CS820, CS921 and CS923 w/firmware 073.239 and Lexmark Secure Element (P/N 57X0185)", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-02-03" + }, + { + "product": "Lexmark MX421, MX521, MX622, MX721, MX722, MX725, CX622, CX625, CX725 w/firmware 073.239 and Lexmark Secure Element (P/N 57X0185)", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-01-26" + }, + { + "product": "Lexmark MX522, MX622h, MX721h, MX722h, MX822, MX826, CX622h, CX625h, CX725h, CX820, CX825, CX860, CX920, CX921, CX922, CX923, CX924, M C550SRF, M C550FG w/firmware 073.239 and Lexmark Secure Element (P/N 57X0185)", + "vendor": "Lexmark International, Inc.", + "level": "PP_HCD_V1.0", + "certification_date": "2021-01-14" + }, + { + "product": "Fortinet FortiGate/FortiOS 6.0.9", + "vendor": "Fortinet Inc.", + "level": "CPP_FW_V2.0E w/EP_IPS_V2.11, EP_VPN_GW_V2.1", + "certification_date": "2021-01-05" + }, + { + "product": "RICOH PRO C5300S/C5310S, version JE-1.00-H", + "vendor": "RICOH COMPANY, LTD", + "level": "PP_HCD_V1.0", + "certification_date": "2020-12-21" + }, + { + "product": "RICOH IM C6500/C8000, version JE-1.00-H", + "vendor": "RICOH COMPANY, LTD", + "level": "PP_HCD_V1.0", + "certification_date": "2020-12-17" + }, + { + "product": "RICOH IM 550/600/600SR, version E-1.00-H", + "vendor": "RICOH COMPANY, LTD", + "level": "PP_HCD_V1.0", + "certification_date": "2020-12-17" + }, + { + "product": "RICOH IM C300/C300F/C400F/C400SRF, version JE-1.00-H", + "vendor": "RICOH COMPANY, LTD", + "level": "PP_HCD_V1.0", + "certification_date": "2020-12-16" + }, + { + "product": "Citrix Virtual Apps and Desktops 7 1912 LTSR Premium Edition\n\n\t\t\tCitrix Virtual Apps and Desktops 7 2203 LTSR Premium Edition (CVAD) (May 2022)", + "vendor": "Citrix Systems, Inc.", + "level": "EAL 2 + (ALC_FLR.2)", + "certification_date": "2020-10-26" + }, + { + "product": "Samsung 5G gNB AU, DU v19.A", + "vendor": "Samsung Electronics Co. Ltd.", + "level": "CPP_ND_V2.2E", + "certification_date": "2020-11-10" + }, + { + "product": "McAfee Change Control and Application Control 8.3.0 with ePolicy Orchestrator 5.10.0", + "vendor": "McAfee, LLC.", + "level": "EAL2 + (ALC_FLR.2)", + "certification_date": "2020-10-16" + }, + { + "product": "Vormetric Data Security Manager V6000, Version 6.3", + "vendor": "Thales DIS CPL USA, Inc.", + "level": "PP_ESM_PM_V2.1", + "certification_date": "2020-10-07" + }, + { + "product": "Tripwire Enterprise Version 8.8.2.2\n\t\t\tTripwire Enterprise Version 8.9.1 (August 2022)", + "vendor": "Tripwire, Inc.", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-09-03" + }, + { + "product": "Dell EMC XtremIO v6.3.1-5 with the 6.3.1-5 Storage Controller Software", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-10-14" + }, + { + "product": "Netscout nGeniusPULSE Server v3.2", + "vendor": "NETSCOUT Systems, Inc", + "level": "CPP_ND_V2.1", + "certification_date": "2020-10-05" + }, + { + "product": "Dell EMC™ VxFlex 3.0.1.208 with VxFlex Ready Node 14G Hardware", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-09-09" + }, + { + "product": "Dell EMC Networking SmartFabric OS10 v10.5.1", + "vendor": "Dell EMC", + "level": "CPP_ND_V2.1", + "certification_date": "2020-09-15" + }, + { + "product": "Dell EMC™ SupportAssist Enterprise 4.0 with Policy Manager 6.8", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-08-06" + }, + { + "product": "Ixia, A Keysight Business Vision Series Network Packet Broker v5.3.0", + "vendor": "Ixia, A Keysight Business", + "level": "CPP_ND_V2.1", + "certification_date": "2020-07-22" + }, + { + "product": "Fortinet FortiGate 6000 Series w/ FortiOS 5.6", + "vendor": "Fortinet, Inc.", + "level": "CPP_FW_V2.0e; EP_VPN_GW_V2.1; EP_IPS_V2.11", + "certification_date": "2020-07-29" + }, + { + "product": "Dell EMC™ VxRail™ 4.7", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-06-30" + }, + { + "product": "Market Central SecureSwitch® Fiber Optic Switch Models: A, B, C, D, 1:1, 2:1, 3:1, 4:1, 5:1, 6:1, 7:1 and 8:1\n\t\t\tMarket Central SecureSwitch® Fiber Optic Switch Models: A, B, C, D, 1:1, 2:1, 3:1, 4:1, 5:1, 6:1, 7:1 and 8:1 (March 2022)", + "vendor": "Market Central, Inc.", + "level": "EAL 4+ (ALC_FLR.1)", + "certification_date": "2020-06-25" + }, + { + "product": "Aruba Mobility Master with ArubaOS 8.2", + "vendor": "Aruba, a Hewlett Packard Enterprise company", + "level": "CPP_ND_v2.1", + "certification_date": "2020-07-10" + }, + { + "product": "Symantec Privileged Access Manager v3.3.0.1085", + "vendor": "Broadcom", + "level": "PP_ESM_PM_V2.1", + "certification_date": "2020-05-31" + }, + { + "product": "Veritas NetBackup™ 8.2 and NetBackup 5240 Appliance Release 3.2", + "vendor": "Veritas Technologies", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-02-28" + }, + { + "product": "Dell EMC™ Isilon with OneFS v8.2.0.0", + "vendor": "Dell EMC", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-01-31" + }, + { + "product": "Dell MX7000 Modular Chassis with Management Module v1.00.10", + "vendor": "Dell Technologies", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2020-01-07" + }, + { + "product": "RICOH IM C2000 / C2500 / C3000 / C3500 / C4500 / C5500 / C6000 v.JE-1.00-H", + "vendor": "RICOH COMPANY, LTD.", + "level": "PP_HCD_V1.0", + "certification_date": "2020-01-06" + }, + { + "product": "Arista Networks Switches EOS 4.22.1FX-CC", + "vendor": "Arista Networks, Inc.", + "level": "CPP_ND_V2.1", + "certification_date": "2019-12-03" + }, + { + "product": "Integrated Dell Remote Access Controller 9", + "vendor": "Dell Technologies", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2019-11-13" + }, + { + "product": "AhnLab MDS, MDS with MTA, and MDS Manager v2.1", + "vendor": "AhnLab, Inc.", + "level": "CPP_ND_V2.1", + "certification_date": "2019-10-29" + }, + { + "product": "Dell EMC™ Avamar® v18.1", + "vendor": "Dell EMC™", + "level": "EAL 2+ (ALC_FLR.2)", + "certification_date": "2019-10-09" + }, + { + "product": "AhnLab EPP, EDR 1.0 and V3 Endpoint Security 9.0", + "vendor": "AhnLab", + "level": "EAL 2+ (ALC_FLR.1)", + "certification_date": "2019-08-27" + }, + { + "product": "Fortinet FortiProxy v1.0", + "vendor": "Fortinet, Inc.", + "level": "CPP_ND_V2.0E", + "certification_date": "2019-08-08" + }, + { + "product": "NETSCOUT Sightline and Threat Mitigation System v9.7", + "vendor": "NETSCOUT Systems Inc.", + "level": "CPP_ND_V2.2E", + "certification_date": "2024-05-14" + } + ] + } + } + } +} \ No newline at end of file From 9eaa9c27a0130ee99c0e715730b9cd07b0bb9c83 Mon Sep 17 00:00:00 2001 From: J08nY Date: Fri, 19 Jul 2024 18:50:57 +0200 Subject: [PATCH 5/6] Fix USA scheme download. --- src/sec_certs/constants.py | 7 +- src/sec_certs/sample/cc_scheme.py | 213 +++++++----------- tests/cc/test_cc_schemes.py | 1 - .../auxiliary_datasets/scheme_dataset.json | 2 +- 4 files changed, 89 insertions(+), 134 deletions(-) diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py index e9b6d882..69f0c6cd 100644 --- a/src/sec_certs/constants.py +++ b/src/sec_certs/constants.py @@ -145,7 +145,6 @@ CC_SWEDEN_ARCHIVED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/arkiverade-certifikat-aldre-an-5-ar/" CC_TURKEY_ARCHIVED_URL = "https://statik.tse.org.tr/upload/tr/dosya/icerikyonetimi/3300/03112021143434-2.pdf" CC_USA_BASE_URL = "https://www.niap-ccevs.org" -CC_USA_PRODUCT_URL = CC_USA_BASE_URL + "/Product/" -CC_USA_CERTIFIED_URL = CC_USA_BASE_URL + "/Product/PCL.cfm" -CC_USA_INEVAL_URL = CC_USA_BASE_URL + "/Product/PINE.cfm" -CC_USA_ARCHIVED_URL = CC_USA_BASE_URL + "/Product/Archived.cfm" +CC_USA_PRODUCTS_URL = CC_USA_BASE_URL + "/api/project/product/pcl_products/" +CC_USA_FILES_URL = CC_USA_BASE_URL + "/api/file/get_pcl_files/" +CC_USA_GETFILE_URL = CC_USA_BASE_URL + "/api/file/get_public_file/" diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py index 7fb9e18e..51dfd0f8 100644 --- a/src/sec_certs/sample/cc_scheme.py +++ b/src/sec_certs/sample/cc_scheme.py @@ -65,12 +65,13 @@ ] -def _get(url: str, session, **kwargs) -> Response: +def _getq(url: str, params, session=None, **kwargs) -> Response: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=InsecureRequestWarning) conn = session if session else requests resp = conn.get( url, + params=params, headers={"User-Agent": "sec-certs.org"}, verify=False, **kwargs, @@ -80,6 +81,10 @@ def _get(url: str, session, **kwargs) -> Response: return resp +def _get(url: str, session=None, **kwargs) -> Response: + return _getq(url, None, session, **kwargs) + + def _get_page(url: str, session=None) -> BeautifulSoup: return BeautifulSoup(_get(url, session).content, "html5lib") @@ -1444,6 +1449,78 @@ def get_turkey_certified() -> list[dict[str, Any]]: return results +def _get_usa(args, enhanced: bool, artifacts: bool): # noqa: C901 + # TODO: There is more information in the API (like about PPs, etc.) + def map_cert(cert, files=None): # noqa: C901 + result = { + "product": cert["product_name"], + "id": f"CCEVS-VR-VID{cert['product_id']}", + "url": constants.CC_USA_BASE_URL + f"/product/{cert['product_id']}", + "certification_date": cert["certification_date"], + "expiration_date": cert["sunset_date"], + "category": cert["tech_type"], + "vendor": cert["vendor_id_name"], + "evaluation_facility": cert["assigned_lab_name"], + "scheme": cert["submitting_country_id_code"], + } + if files: + for file in files["eval_files"]: + if file["file_label"] == "Validation Report": + dt = datetime.fromisoformat(file["uploaded_on"]) + result["id"] += f"-{dt.year}" + result["report_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" + if artifacts: + result["report_hash"] = _get_hash(result["report_link"]).hex() + elif file["file_label"] == "CC Certificate": + result["cert_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" + if artifacts: + result["cert_hash"] = _get_hash(result["cert_link"]).hex() + elif file["file_label"] == "Security Target": + result["target_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" + if artifacts: + result["target_hash"] = _get_hash(result["target_link"]).hex() + elif file["file_label"] == "Assurance Activity Report (AAR)": + result["aar_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" + if artifacts: + result["aar_hash"] = _get_hash(result["aar_link"]).hex() + elif file["file_label"] == "Administrative Guide (AGD)": + result["agd_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" + if artifacts: + result["agd_hash"] = _get_hash(result["agd_link"]).hex() + + return result + + session = requests.session() + results = [] + offset = 0 + got = 0 + while True: + resp = _getq( + constants.CC_USA_PRODUCTS_URL, + {"limit": 100, "offset": offset, **args}, + session, + ) + json = resp.json() + count = json["count"] + for cert in json["results"]["products"]: + got += 1 + if "from_cc_portal" in cert: + continue + files = None + if enhanced: + resp = _getq( + constants.CC_USA_FILES_URL, + {"product_id": cert["product_id"]}, + session, + ) + files = resp.json() + results.append(map_cert(cert, files)) + offset += 100 + if got >= count: + break + return results + + def get_usa_certified( # noqa: C901 enhanced: bool = True, artifacts: bool = False ) -> list[dict[str, Any]]: @@ -1454,83 +1531,11 @@ def get_usa_certified( # noqa: C901 :param artifacts: Whether to download and compute artifact hashes (way slower, even more data). :return: The entries. """ - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = _get_page(constants.CC_USA_CERTIFIED_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - vendor_span = tds[0].find("span", class_="b u") - product_link = tds[0].find("a") - scheme_img = tds[6].find("img") - # Only return the US certifications. - if scheme_img["title"] != "USA": - continue - cert: dict[str, Any] = { - "product": sns(product_link.text), - "vendor": sns(vendor_span.text), - "product_link": urljoin(constants.CC_USA_PRODUCT_URL, product_link["href"]), - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "certification_date": sns(tds[4].text), - "assurance_maintenance_date": sns(tds[5].text), - } - if enhanced: - e: dict[str, Any] = {} - if not cert["product_link"]: - continue - cert_page = _get_page(cert["product_link"]) - details = cert_page.find("div", class_="txt2 lma") - for span in details.find_all("span"): - title = sns(span.text) - if not title: - continue - sibling = span.next_sibling - value = sns(sibling.text) - if "Certificate Date" in title: - e["certification_date"] = value - elif "Product Type" in title: - e["product_type"] = value - elif "Conformance Claim" in title: - e["cc_claim"] = value - elif "Validation Report Number" in title: - e["cert_id"] = value - elif "PP Identifier" in title: - e["protection_profile"] = sns(span.find_next_sibling("a").text) - elif "CC Testing Lab" in title: - e["evaluation_facility"] = sns(span.find_next_sibling("a").text) - links = cert_page.find_all("a", class_="pseudobtn1") - for link in links: - name = sns(link.text) - href = urljoin(constants.CC_USA_BASE_URL, sns(link["href"])) - if not name: - continue - if "CC Certificate" in name: - e["cert_link"] = href - if artifacts: - e["cert_hash"] = _get_hash(href).hex() - elif "Security Target" in name: - e["target_link"] = href - if artifacts: - e["target_hash"] = _get_hash(href).hex() - elif "Validation Report" in name: - e["report_link"] = href - if artifacts: - e["report_hash"] = _get_hash(href).hex() - elif "Assurance Activity" in name: - e["assurance_activity_link"] = href - if artifacts: - e["assurance_activity_hash"] = _get_hash(href).hex() - elif "Administrative Guide" in name: - guides = e.setdefault("administrative_guides", []) - guide = {"link": href} - guides.append(guide) - if artifacts: - guide["hash"] = _get_hash(href).hex() - cert["enhanced"] = e - results.append(cert) - return results + return _get_usa( + {"certification_status": "Certified", "publish_status": "Published"}, + enhanced, + artifacts, + ) def get_usa_in_evaluation() -> list[dict[str, Any]]: @@ -1539,29 +1544,7 @@ def get_usa_in_evaluation() -> list[dict[str, Any]]: :return: The entries. """ - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = _get_page(constants.CC_USA_INEVAL_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - vendor_span = tds[0].find("span", class_="b u") - product_name = None - for child in tds[0].children: - if isinstance(child, NavigableString): - product_name = sns(child) - break - cert = { - "vendor": sns(vendor_span.text), - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "kickoff_date": sns(tds[4].text), - } - if product_name: - cert["product"] = product_name - results.append(cert) - return results + return _get_usa({"status": "In Progress", "publish_status": "Published"}, False, False) def get_usa_archived() -> list[dict[str, Any]]: @@ -1570,33 +1553,7 @@ def get_usa_archived() -> list[dict[str, Any]]: :return: The entries. """ - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = _get_page(constants.CC_USA_ARCHIVED_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - scheme_img = tds[5].find("img") - # Only return the US certifications. - if scheme_img["title"] != "USA": - continue - vendor_span = tds[0].find("span", class_="b u") - product_name = None - for child in tds[0].children: - if isinstance(child, NavigableString): - product_name = sns(child) - break - cert = { - "vendor": sns(vendor_span.text), - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "certification_date": sns(tds[4].text), - } - if product_name: - cert["product"] = product_name - results.append(cert) - return results + return _get_usa({"status": "Archived", "publish_status": "Published"}, False, False) class EntryType(Enum): diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py index 9e1e927f..1740cb27 100644 --- a/tests/cc/test_cc_schemes.py +++ b/tests/cc/test_cc_schemes.py @@ -178,7 +178,6 @@ def test_turkey(): @pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException) def test_usa(): - pytest.skip() certified = CCSchemes.get_usa_certified() assert len(certified) != 0 assert absolute_urls(certified) diff --git a/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json b/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json index 64db2e84..3b7348d8 100644 --- a/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json +++ b/tests/data/cc/dataset/auxiliary_datasets/scheme_dataset.json @@ -4,7 +4,7 @@ "CA": { "_type": "sec_certs.sample.cc_scheme.CCScheme", "country": "CA", - "timestamp": "2024-07-19T17:14:26.369494", + "timestamp": "2024-07-19T17:15:33.014552", "lists": { "INEVALUATION": [ { From ee63131f13f3900083f41e0731d1a477546410c2 Mon Sep 17 00:00:00 2001 From: J08nY Date: Sun, 21 Jul 2024 14:40:53 +0200 Subject: [PATCH 6/6] Fix USA scheme parsing on Python < 3.11. Datetime.fromisoformat changed behavior then. --- src/sec_certs/sample/cc_scheme.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py index 51dfd0f8..2785551d 100644 --- a/src/sec_certs/sample/cc_scheme.py +++ b/src/sec_certs/sample/cc_scheme.py @@ -18,6 +18,7 @@ import requests import tabula from bs4 import BeautifulSoup, NavigableString, Tag +from dateutil.parser import isoparse from requests import Response from urllib3.connectionpool import InsecureRequestWarning @@ -1466,7 +1467,7 @@ def map_cert(cert, files=None): # noqa: C901 if files: for file in files["eval_files"]: if file["file_label"] == "Validation Report": - dt = datetime.fromisoformat(file["uploaded_on"]) + dt = isoparse(file["uploaded_on"]) result["id"] += f"-{dt.year}" result["report_link"] = constants.CC_USA_GETFILE_URL + f"?file_id={file['file_id']}" if artifacts: