drop python 3.8

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x", "3.12.x"]
+        python-version: ["3.9.x", "3.10.x", "3.11.x", "3.12.x"]
 
     defaults:
       run:
@@ -49,7 +49,6 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
-          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_common.yml

@@ -37,8 +37,6 @@ jobs:
         python-version: ["3.11.x"]
         # Test all python versions on ubuntu only
         include:
-          - python-version: "3.8.x"
-            os: "ubuntu-latest"
           - python-version: "3.9.x"
             os: "ubuntu-latest"
           - python-version: "3.10.x"
@@ -79,7 +77,6 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
-          version: 1.8.5
 
       # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
       # - name: Load cached venv

README.md

@@ -28,7 +28,7 @@ Be it a Google Colab notebook, AWS Lambda function, an Airflow DAG, your local l
 
 ## Installation
 
-dlt supports Python 3.8+.
+dlt supports Python 3.9+.
 
 ```sh
 pip install dlt

dlt/cli/deploy_command_helpers.py

@@ -2,6 +2,8 @@
 import abc
 import os
 import yaml
+import ast
+
 from yaml import Dumper
 from itertools import chain
 from typing import List, Optional, Sequence, Tuple, Any, Dict
@@ -22,7 +24,7 @@
 from dlt.common.git import get_origin, get_repo, Repo
 from dlt.common.configuration.specs.runtime_configuration import get_default_pipeline_name
 from dlt.common.typing import StrAny
-from dlt.common.reflection.utils import evaluate_node_literal, ast_unparse
+from dlt.common.reflection.utils import evaluate_node_literal
 from dlt.common.pipeline import LoadInfo, TPipelineState, get_dlt_repos_dir
 from dlt.common.storages import FileStorage
 from dlt.common.utils import set_working_dir
@@ -312,7 +314,7 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio
                 if f_r_value is None:
                     fmt.warning(
                         "The value of `dev_mode` in call to `dlt.pipeline` cannot be"
-                        f" determined from {ast_unparse(f_r_node).strip()}. We assume that you know"
+                        f" determined from {ast.unparse(f_r_node).strip()}. We assume that you know"
                         " what you are doing :)"
                     )
                 if f_r_value is True:
@@ -330,7 +332,7 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio
                     raise CliCommandInnerException(
                         "deploy",
                         "The value of 'pipelines_dir' argument in call to `dlt_pipeline` cannot be"
-                        f" determined from {ast_unparse(p_d_node).strip()}. Pipeline working dir"
+                        f" determined from {ast.unparse(p_d_node).strip()}. Pipeline working dir"
                         " will be found. Pass it directly with --pipelines-dir option.",
                     )
 
@@ -341,7 +343,7 @@ def parse_pipeline_info(visitor: PipelineScriptVisitor) -> List[Tuple[str, Optio
                     raise CliCommandInnerException(
                         "deploy",
                         "The value of 'pipeline_name' argument in call to `dlt_pipeline` cannot be"
-                        f" determined from {ast_unparse(p_d_node).strip()}. Pipeline working dir"
+                        f" determined from {ast.unparse(p_d_node).strip()}. Pipeline working dir"
                         " will be found. Pass it directly with --pipeline-name option.",
                     )
             pipelines.append((pipeline_name, pipelines_dir))

dlt/cli/source_detection.py

@@ -4,7 +4,7 @@
 
 from dlt.common.configuration import is_secret_hint
 from dlt.common.configuration.specs import BaseConfiguration
-from dlt.common.reflection.utils import creates_func_def_name_node, ast_unparse
+from dlt.common.reflection.utils import creates_func_def_name_node
 from dlt.common.typing import is_optional_type
 
 from dlt.sources import SourceReference
@@ -63,7 +63,7 @@ def find_source_calls_to_replace(
     for calls in visitor.known_sources_resources_calls.values():
         for call in calls:
             transformed_nodes.append(
-                (call.func, ast.Name(id=pipeline_name + "_" + ast_unparse(call.func)))
+                (call.func, ast.Name(id=pipeline_name + "_" + ast.unparse(call.func)))
             )
 
     return transformed_nodes

dlt/common/libs/pydantic.py

@@ -248,20 +248,12 @@ def _process_annotation(t_: Type[Any]) -> Type[Any]:
             return Annotated[_process_annotation(a_t), tuple(a_m)]  # type: ignore[return-value]
         elif is_list_generic_type(t_):
             l_t: Type[Any] = get_args(t_)[0]
-            try:
-                return get_origin(t_)[_process_annotation(l_t)]  # type: ignore[no-any-return]
-            except TypeError:
-                # this is Python3.8 fallback. it does not support indexers on types
-                return List[_process_annotation(l_t)]  # type: ignore
+            return get_origin(t_)[_process_annotation(l_t)]  # type: ignore[no-any-return]
         elif is_dict_generic_type(t_):
             k_t: Type[Any]
             v_t: Type[Any]
             k_t, v_t = get_args(t_)
-            try:
-                return get_origin(t_)[k_t, _process_annotation(v_t)]  # type: ignore[no-any-return]
-            except TypeError:
-                # this is Python3.8 fallback. it does not support indexers on types
-                return Dict[k_t, _process_annotation(v_t)]  # type: ignore
+            return get_origin(t_)[k_t, _process_annotation(v_t)]  # type: ignore[no-any-return]
         elif is_union_type(t_):
             u_t_s = tuple(_process_annotation(u_t) for u_t in extract_union_types(t_))
             return Union[u_t_s]  # type: ignore[return-value]

dlt/common/reflection/utils.py

@@ -1,13 +1,7 @@
 import ast
 import inspect
-from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, Callable
-
-try:
-    import astunparse
 
-    ast_unparse: Callable[[ast.AST], str] = astunparse.unparse
-except ImportError:
-    ast_unparse = ast.unparse  #  type: ignore[attr-defined, unused-ignore]
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, Callable
 
 from dlt.common.typing import AnyFun
 
@@ -31,7 +25,7 @@ def get_literal_defaults(node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) ->
     literal_defaults: Dict[str, str] = {}
     for arg, default in zip(reversed(args), reversed(defaults)):
         if default:
-            literal_defaults[str(arg.arg)] = ast_unparse(default).strip()
+            literal_defaults[str(arg.arg)] = ast.unparse(default).strip()
 
     return literal_defaults
 
@@ -105,7 +99,7 @@ def rewrite_python_script(
             script_lines.append(source_script_lines[last_line][last_offset : node.col_offset])  # type: ignore[attr-defined]
 
         # replace node value
-        script_lines.append(ast_unparse(t_value).strip())
+        script_lines.append(ast.unparse(t_value).strip())
         last_line = node.end_lineno - 1  # type: ignore[attr-defined]
         last_offset = node.end_col_offset  # type: ignore[attr-defined]
 

dlt/common/runners/pool_runner.py

@@ -26,7 +26,7 @@ class NullExecutor(Executor):
     Provides a uniform interface for `None` pool type
     """
 
-    def submit(self, fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> Future[T]:
+    def submit(self, fn: Callable[P, T], /, *args: P.args, **kwargs: P.kwargs) -> Future[T]:
         """Run the job and return a Future"""
         fut: Future[T] = Future()
         try:

dlt/common/typing.py

@@ -55,15 +55,11 @@
     # in versions of Python>=3.10.
     UnionType = Never
 
-if sys.version_info[:3] >= (3, 9, 0):
-    from typing import _SpecialGenericAlias, _GenericAlias  # type: ignore[attr-defined]
-    from types import GenericAlias  # type: ignore[attr-defined]
+from typing import _SpecialGenericAlias, _GenericAlias  # type: ignore[attr-defined]
+from types import GenericAlias
 
-    typingGenericAlias: Tuple[Any, ...] = (_GenericAlias, _SpecialGenericAlias, GenericAlias)
-else:
-    from typing import _GenericAlias  # type: ignore[attr-defined]
+typingGenericAlias: Tuple[Any, ...] = (_GenericAlias, _SpecialGenericAlias, GenericAlias)
 
-    typingGenericAlias = (_GenericAlias,)
 
 from dlt.common.pendulum import timedelta, pendulum
 

dlt/reflection/script_visitor.py

@@ -1,9 +1,9 @@
 import inspect
 import ast
-from ast import NodeVisitor
+from ast import NodeVisitor, unparse
 from typing import Any, Dict, List
 
-from dlt.common.reflection.utils import find_outer_func_def, ast_unparse
+from dlt.common.reflection.utils import find_outer_func_def
 
 import dlt.reflection.names as n
 
@@ -67,9 +67,9 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
             for deco in node.decorator_list:
                 # decorators can be function calls, attributes or names
                 if isinstance(deco, (ast.Name, ast.Attribute)):
-                    alias_name = ast_unparse(deco).strip()
+                    alias_name = ast.unparse(deco).strip()
                 elif isinstance(deco, ast.Call):
-                    alias_name = ast_unparse(deco.func).strip()
+                    alias_name = ast.unparse(deco.func).strip()
                 else:
                     raise ValueError(
                         self.source_segment(deco), type(deco), "Unknown decorator form"
@@ -86,7 +86,7 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
     def visit_Call(self, node: ast.Call) -> Any:
         if self._curr_pass == 2:
             # check if this is a call to any of known functions
-            alias_name = ast_unparse(node.func).strip()
+            alias_name = ast.unparse(node.func).strip()
             fn = self.func_aliases.get(alias_name)
             if not fn:
                 # try a fallback to "run" function that may be called on pipeline or source

dlt/sources/rest_api/__init__.py

@@ -1,7 +1,7 @@
 """Generic API Source"""
 from copy import deepcopy
 from typing import Any, Dict, List, Optional, Generator, Callable, cast, Union
-import graphlib  # type: ignore[import,unused-ignore]
+import graphlib
 from requests.auth import AuthBase
 
 import dlt
@@ -229,7 +229,7 @@ def rest_api_resources(config: RESTAPIConfig) -> List[DltResource]:
 
 def create_resources(
     client_config: ClientConfig,
-    dependency_graph: graphlib.TopologicalSorter,
+    dependency_graph: graphlib.TopologicalSorter,  # type: ignore[type-arg]
     endpoint_resource_map: Dict[str, Union[EndpointResource, DltResource]],
     resolved_param_map: Dict[str, Optional[List[ResolvedParam]]],
 ) -> Dict[str, DltResource]:

dlt/sources/rest_api/config_setup.py

@@ -12,7 +12,7 @@
     cast,
     NamedTuple,
 )
-import graphlib  # type: ignore[import,unused-ignore]
+import graphlib
 import string
 from requests import Response
 
@@ -277,10 +277,12 @@ def make_parent_key_name(resource_name: str, field_name: str) -> str:
 def build_resource_dependency_graph(
     resource_defaults: EndpointResourceBase,
     resource_list: List[Union[str, EndpointResource, DltResource]],
-) -> Tuple[
-    Any, Dict[str, Union[EndpointResource, DltResource]], Dict[str, Optional[List[ResolvedParam]]]
+) -> Tuple[  # type: ignore[type-arg]
+    graphlib.TopologicalSorter,
+    Dict[str, Union[EndpointResource, DltResource]],
+    Dict[str, Optional[List[ResolvedParam]]],
 ]:
-    dependency_graph = graphlib.TopologicalSorter()
+    dependency_graph: graphlib.TopologicalSorter = graphlib.TopologicalSorter()  # type: ignore[type-arg]
     resolved_param_map: Dict[str, Optional[List[ResolvedParam]]] = {}
     endpoint_resource_map = expand_and_index_resources(resource_list, resource_defaults)
 

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version=3.8
+python_version=3.9
 ignore_missing_imports=false
 strict_optional=false
 warn_redundant_casts=true