Merge branch 'develop'

.github/workflows/changelog.yml

@@ -20,7 +20,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ci:skip-changelog') && github.event.pull_request.user.login != 'pre-commit-ci[bot]' && github.event.pull_request.user.login != 'dependabot[bot]'"
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/codeql-analysis.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       uses: actions/setup-python@v4
@@ -84,7 +84,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       uses: actions/setup-python@v4

.github/workflows/dev-release.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/get-matrix.yml

@@ -72,7 +72,7 @@ jobs:
       matrix-webdav: ${{ toJson(fromJson(steps.matrix-webdav.outputs.result)[steps.key-webdav.outputs.key]) }}
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 2
 

.github/workflows/nightly.yml

@@ -342,7 +342,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       uses: actions/setup-python@v4

.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/test-clickhouse.yml

@@ -40,7 +40,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-core.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v4

.github/workflows/test-ftp.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v4

.github/workflows/test-ftps.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v4

.github/workflows/test-greenplum.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-hdfs.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-hive.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-kafka.yml

@@ -73,7 +73,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-local-fs.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-mongodb.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-mssql.yml

@@ -41,7 +41,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-mysql.yml

@@ -40,7 +40,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-oracle.yml

@@ -43,7 +43,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-postgres.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-s3.yml

@@ -40,7 +40,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-sftp.yml

@@ -33,7 +33,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v4

.github/workflows/test-teradata.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Java ${{ inputs.java-version }}
       uses: actions/setup-java@v3

.github/workflows/test-webdav.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v4

.github/workflows/tests.yml

@@ -325,7 +325,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python ${{ env.DEFAULT_PYTHON }}
       uses: actions/setup-python@v4

.pre-commit-config.yaml

@@ -69,15 +69,15 @@ repos:
   - id: black
     language_version: python3
 - repo: https://github.com/asottile/blacken-docs
-  rev: 1.15.0
+  rev: 1.16.0
   hooks:
   - id: blacken-docs
 - repo: meta
   hooks:
   - id: check-hooks-apply
   - id: check-useless-excludes
 - repo: https://github.com/PyCQA/autoflake
-  rev: v2.2.0
+  rev: v2.2.1
   hooks:
   - id: autoflake
     args:

CONTRIBUTING.rst

@@ -170,7 +170,7 @@ Without docker-compose
 
     To run Greenplum tests, you should:
 
-    * Download `Pivotal connector for Spark <https://onetl.org.readthedocs.build/en/latest/db_connection/greenplum/prerequisites.html>`_
+    * Download `Pivotal connector for Spark <https://onetl.readthedocs.io/en/latest/connection/db_connection/greenplum/prerequisites.html>`_
     * Either move it to ``~/.ivy2/jars/``, or pass file path to ``CLASSPATH``
     * Set environment variable ``ONETL_DB_WITH_GREENPLUM=true`` to enable adding connector to Spark session
 

README.rst

@@ -321,7 +321,7 @@ Read data from MSSQL, transform & write to Hive.
         extra={"ApplicationIntent": "ReadOnly"},
     ).check()
 
-    # >>> INFO:|MSSQL| Connection is available.
+    # >>> INFO:|MSSQL| Connection is available
 
     # Initialize DB reader
     reader = DBReader(
@@ -408,7 +408,7 @@ Download files from SFTP & upload them to HDFS.
         password="somepassword",
     ).check()
 
-    # >>> INFO:|SFTP| Connection is available.
+    # >>> INFO:|SFTP| Connection is available
 
     # Initialize downloader
     file_downloader = FileDownloader(
@@ -546,7 +546,7 @@ Read files directly from S3 path, convert them to dataframe, transform it and th
         spark=spark,
     ).check()
 
-    # >>> INFO:|SparkS3| Connection is available.
+    # >>> INFO:|SparkS3| Connection is available
 
     # Describe file format and parsing options
     csv = CSV(
@@ -577,7 +577,7 @@ Read files directly from S3 path, convert them to dataframe, transform it and th
         ],
     )
 
-    # Initialize file reader
+    # Initialize file df reader
     reader = FileDFReader(
         connection=spark_s3,
         source_path="/remote/tests/Report",  # path on S3 there *.csv files are located

codecov.yml

@@ -2,86 +2,5 @@ coverage:
   status:
     project:
       default:
-        target: 93%
+        target: 94%
         threshold: 1%
-
-flags:
-  core:
-    paths:
-    - onetl/base/base_connection.py
-    - onetl/hwm/**
-    - onetl/impl/*model*.py
-    - onetl/impl/*options*.py
-    - onetl/strategy/**
-    - onetl/hooks/**
-    - onetl/plugins/**
-    - onetl/exception.py
-    - onetl/log.py
-    - onetl/_internal.py
-  db:
-    paths:
-    - onetl/base/*db*.py
-    - onetl/base/*df*.py
-    - onetl/core/db*/**
-    - onetl/db_connection/db_connection.py
-    - onetl/db_connection/dialect_mixins/**
-    - onetl/db_connection/jdbc*.py
-  clickhouse:
-    paths:
-    - onetl/db_connection/clickhouse.py
-  greenplum:
-    paths:
-    - onetl/db_connection/greenplum.py
-    carryforward: true  # if someone creates pull request from a fork, do not fail if Greenplum coverage is 0%
-  hive:
-    paths:
-    - onetl/db_connection/hive.py
-  mongodb:
-    paths:
-    - onetl/db_connection/mongodb.py
-  mssql:
-    paths:
-    - onetl/db_connection/mongodb.py
-  mysql:
-    paths:
-    - onetl/db_connection/mongodb.py
-  oracle:
-    paths:
-    - onetl/db_connection/oracle.py
-  postgres:
-    paths:
-    - onetl/db_connection/postgres.py
-  teradata:
-    paths:
-    - onetl/db_connection/teradata.py
-  file:
-    paths:
-    - onetl/base/*file*.py
-    - onetl/base/*path*.py
-    - onetl/base/contains_exception.py
-    - onetl/core/file*/**
-    - onetl/core/kerberos_helpers.py
-    - onetl/file_connection/file_connection.py
-    - onetl/impl/*path*.py
-    - onetl/impl/*file*.py
-    - onetl/impl/*directory*.py
-  ftp:
-    paths:
-    - onetl/file_connection/ftp.py
-  ftps:
-    paths:
-    - onetl/file_connection/ftps.py
-  hdfs:
-    paths:
-    - onetl/file_connection/hdfs.py
-  s3:
-    paths:
-    - onetl/file_connection/s3.py
-  sftp:
-    paths:
-    - onetl/file_connection/sftp.py
-  webdav:
-    paths:
-    - onetl/file_connection/webdav.py
-  nightly:
-    joined: false