Merge branch 'bump-req-python-to-3-10' into reference-analysis

.github/workflows/docs.yml

@@ -13,7 +13,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install external dependencies
         run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
       - name: Install sec-certs and deps

.github/workflows/pre-commit.yml

@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install build dependencies
         run: python -m pip install build
       - name: Build distributions

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
           cache: "pip"
           cache-dependency-path: |
             requirements/test_requirements.txt

Dockerfile

@@ -19,9 +19,8 @@ ENV NB_GID=1000
 ENV HOME /home/${USER}
 
 #installing dependencies
-RUN apt-get update
-RUN apt-get install python3 -y
-RUN apt-get install python3-pip -y
+RUN apt-get update && apt-get upgrade -y
+RUN apt-get install python3.10 -y
 RUN apt-get install python3-venv -y
 RUN apt-get install git -y
 RUN apt-get install curl -y

docs/installation.md

@@ -10,7 +10,7 @@ pip install -U sec-certs
 python -m spacy download en_core_web_sm
 ```
 
-Note, that `Python>=3.8` is required.
+Note, that `Python>=3.10` is required.
 
 :::
 
@@ -47,4 +47,4 @@ If you're not using Docker, you must install the dependencies as described below
 - [Java](https://www.java.com/en) is needed to parse tables in FIPS pdf documents, must be available from `PATH`.
 - Some imported libraries have non-trivial dependencies to resolve:
     - [pdftotext](https://github.com/jalan/pdftotext) requires [Poppler](https://poppler.freedesktop.org/) to be installed. We've experienced issues with older versions of Poppler (`0.x`), make sure to install `20.x` version of these libraries.
-    - [tesseract](https://github.com/tesseract-ocr/tesseract) is required for OCR of malformed PDF documents, together with data files for English, French and German.
+    - [tesseract](https://github.com/tesseract-ocr/tesseract) is required for OCR of malformed PDF documents, together with data files for English, French and German.

pyproject.toml

@@ -19,14 +19,12 @@
     "License :: OSI Approved :: MIT License",
     "Topic :: Security",
     "Topic :: Security :: Cryptography",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
   ]
-  requires-python = ">=3.8"
+  requires-python = ">=3.10"
   dynamic = ["version"]
   dependencies = [
     "beautifulsoup4",
@@ -111,7 +109,7 @@
   ]
   src = ["src", "tests"]
   line-length = 120
-  target-version = "py38"
+  target-version = "py310"
 
   [tool.ruff.mccabe]
     max-complexity = 10

src/sec_certs/cli.py

@@ -3,10 +3,10 @@
 
 import logging
 import sys
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
-from typing import Callable
 
 import click
 from pydantic import ValidationError

src/sec_certs/dataset/cc.py

@@ -4,10 +4,11 @@
 import locale
 import shutil
 import tempfile
+from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import ClassVar, Iterator, cast
+from typing import ClassVar, cast
 
 import numpy as np
 import pandas as pd

src/sec_certs/dataset/cc_scheme.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Mapping
 from pathlib import Path
-from typing import Mapping
 
 from sec_certs import constants
 from sec_certs.dataset.json_path_dataset import JSONPathDataset

src/sec_certs/dataset/cpe.py

@@ -2,9 +2,10 @@
 
 import logging
 import tempfile
+from collections.abc import Iterator
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Iterator
+from typing import Any
 
 import pandas as pd
 

src/sec_certs/dataset/dataset.py

@@ -8,10 +8,11 @@
 import shutil
 import tempfile
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Generic, Iterator, TypeVar, cast
+from typing import Any, Generic, TypeVar, cast
 
 import pandas as pd
 

src/sec_certs/dataset/fips_algorithm.py

@@ -3,9 +3,9 @@
 import itertools
 import logging
 import re
+from collections.abc import Iterator
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Iterator
 
 import pandas as pd
 from bs4 import BeautifulSoup

src/sec_certs/dataset/fips_iut.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Iterator, Mapping
 
 import requests
 

src/sec_certs/dataset/fips_mip.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
 from datetime import date
 from operator import attrgetter
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Iterator, Mapping
 
 import requests
 

src/sec_certs/model/cc_matching.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import contextlib
-from typing import Any, Iterable, Mapping, Sequence
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any
 
 from sec_certs.configuration import config
 from sec_certs.model.matching import AbstractMatcher

src/sec_certs/model/cpe_matching.py

@@ -4,7 +4,7 @@
 import logging
 import operator
 import re
-from typing import Pattern
+from re import Pattern
 
 from rapidfuzz import fuzz
 

src/sec_certs/model/fips_matching.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import typing
+from collections.abc import Iterable, Mapping, Sequence
 from datetime import date
-from typing import Iterable, Mapping, Sequence
 
 from sec_certs.configuration import config
 from sec_certs.model.matching import AbstractMatcher

src/sec_certs/model/matching.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from heapq import heappop, heappush
-from typing import Any, Generic, Sequence, TypeVar
+from typing import Any, Generic, TypeVar
 
 from rapidfuzz import fuzz
 

src/sec_certs/model/reference_finder.py

@@ -1,18 +1,19 @@
 from __future__ import annotations
 
-from typing import Callable, Dict, List, Optional, Set, TypeVar
+from collections.abc import Callable
+from typing import TypeVar
 
 from sec_certs.sample.certificate import Certificate, References
 
 CertSubType = TypeVar("CertSubType", bound=Certificate)
-Certificates = Dict[str, CertSubType]
-ReferencedByDirect = Dict[str, Set[str]]
-ReferencedByIndirect = Dict[str, Set[str]]
-ReferencesType = Dict[str, Dict[str, Optional[Set[str]]]]
-IDMapping = Dict[str, List[str]]
-UnknownReferences = Dict[str, Set[str]]
+Certificates = dict[str, CertSubType]
+ReferencedByDirect = dict[str, set[str]]
+ReferencedByIndirect = dict[str, set[str]]
+ReferencesType = dict[str, dict[str, set[str] | None]]
+IDMapping = dict[str, list[str]]
+UnknownReferences = dict[str, set[str]]
 IDLookupFunc = Callable[[CertSubType], str]
-ReferenceLookupFunc = Callable[[CertSubType], Set[str]]
+ReferenceLookupFunc = Callable[[CertSubType], set[str]]
 
 
 # TODO: All of this can and should be rewritten on top of networkx or some other graph library.

src/sec_certs/model/sar_transformer.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
 import logging
-from typing import Dict, Iterable, cast
+from collections.abc import Iterable
+from typing import cast
 
 from sec_certs.sample.cc import CCCertificate
 from sec_certs.sample.sar import SAR, SAR_DICT_KEY
@@ -68,13 +69,13 @@ def report_keywords_may_have_sars(sample: CCCertificate):
         sec_level_sars = SARTransformer._parse_sars_from_security_level_list(cert.security_level)
 
         if st_keywords_may_have_sars(cert):
-            st_dict: dict = cast(Dict, cert.pdf_data.st_keywords)
+            st_dict: dict = cast(dict, cert.pdf_data.st_keywords)
             st_sars = SARTransformer._parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst)
         else:
             st_sars = set()
 
         if report_keywords_may_have_sars(cert):
-            report_dict: dict = cast(Dict, cert.pdf_data.report_keywords)
+            report_dict: dict = cast(dict, cert.pdf_data.report_keywords)
             report_sars = SARTransformer._parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst)
         else:
             report_sars = set()

src/sec_certs/model/transitive_vulnerability_finder.py

@@ -2,9 +2,10 @@
 
 import logging
 from collections import Counter
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Callable, Dict, Optional, Set, TypeVar
+from typing import TypeVar
 
 from sec_certs.sample.certificate import Certificate, References
 from sec_certs.serialization.json import ComplexSerializableType
@@ -25,8 +26,8 @@ class TransitiveCVEs(ComplexSerializableType):
     indirect_transitive_cves: set[str] | None = field(default=None)
 
 
-Certificates = Dict[str, CertSubType]
-Vulnerabilities = Dict[str, Dict[str, Optional[Set[str]]]]
+Certificates = dict[str, CertSubType]
+Vulnerabilities = dict[str, dict[str, set[str] | None]]
 
 
 class TransitiveVulnerabilityFinder:

src/sec_certs/sample/cc_scheme.py

@@ -6,11 +6,12 @@
 import math
 import tempfile
 import warnings
+from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any, Callable, ClassVar, Iterable
+from typing import Any, ClassVar
 from urllib.parse import urljoin
 
 import requests

src/sec_certs/sample/fips.py

@@ -2,10 +2,11 @@
 
 import itertools
 import re
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from datetime import date
 from pathlib import Path
-from typing import Any, Callable, ClassVar, Final, Literal
+from typing import Any, ClassVar, Final, Literal
 
 import dateutil
 import numpy as np

src/sec_certs/sample/fips_iut.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
 from datetime import date, datetime
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Iterator, Mapping
 
 import requests
 from bs4 import BeautifulSoup, Tag

src/sec_certs/sample/fips_mip.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
 from datetime import date, datetime
 from enum import Enum
 from functools import total_ordering
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Iterator, Mapping
 
 import requests
 from bs4 import BeautifulSoup, Tag

src/sec_certs/serialization/json.py

@@ -3,10 +3,11 @@
 import copy
 import gzip
 import json
+from collections.abc import Callable
 from datetime import date, datetime
 from functools import wraps
 from pathlib import Path
-from typing import Any, Callable, TypeVar
+from typing import Any, TypeVar
 
 from sec_certs import constants
 

src/sec_certs/utils/helpers.py

@@ -4,11 +4,12 @@
 import logging
 import re
 import time
+from collections.abc import Collection
 from contextlib import nullcontext
 from datetime import datetime
 from functools import partial
 from pathlib import Path
-from typing import Any, Collection
+from typing import Any
 
 import numpy as np
 import pkgconfig

src/sec_certs/utils/parallel_processing.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import time
+from collections.abc import Callable, Iterable
 from multiprocessing import cpu_count
 from multiprocessing.pool import Pool, ThreadPool
-from typing import Any, Callable, Iterable
+from typing import Any
 
 from sec_certs.configuration import config
 from sec_certs.utils.tqdm import tqdm