Merge branch 'develop'

.env.docker

@@ -87,6 +87,16 @@ ONETL_SFTP_PORT=2222
 ONETL_SFTP_USER=onetl
 ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
 
+# Samba
+ONETL_SAMBA_HOST=samba
+ONETL_SAMBA_PROTOCOL=SMB
+ONETL_SAMBA_UID=1000
+ONETL_SAMBA_GID=1000
+ONETL_SAMBA_PORT=445
+ONETL_SAMBA_SHARE=SmbShare
+ONETL_SAMBA_USER=onetl
+ONETL_SAMBA_PASSWORD=awd123fd1
+
 # Webdav
 ONETL_WEBDAV_HOST=webdav
 ONETL_WEBDAV_PORT=80

.env.local

@@ -87,6 +87,16 @@ export ONETL_SFTP_PORT=2222
 export ONETL_SFTP_USER=onetl
 export ONETL_SFTP_PASSWORD=AesujeifohgoaCu0Boosiet5aimeitho
 
+# Samba
+export ONETL_SAMBA_HOST=localhost
+export ONETL_SAMBA_PROTOCOL=SMB
+export ONETL_SAMBA_UID=1000
+export ONETL_SAMBA_GID=1000
+export ONETL_SAMBA_PORT=445
+export ONETL_SAMBA_SHARE=SmbShare
+export ONETL_SAMBA_USER=onetl
+export ONETL_SAMBA_PASSWORD=awd123fd1
+
 # Webdav
 export ONETL_WEBDAV_HOST=localhost
 export ONETL_WEBDAV_PORT=8000

.github/workflows/data/greenplum/matrix.yml

@@ -7,7 +7,7 @@ min: &min
 
 max: &max
   # Greenplum connector does not support Spark 3.3+
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.10'
   java-version: 11
   os: ubuntu-latest

.github/workflows/data/local-fs/matrix.yml

@@ -4,12 +4,18 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-avro: &avro
+min_avro: &min_avro
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
+min_excel: &min_excel
+  spark-version: 3.2.4
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
 max: &max
   spark-version: 3.4.1
   python-version: '3.11'
@@ -25,12 +31,15 @@ latest: &latest
 matrix:
   small:
   - <<: *max
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   full:
   - <<: *min
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   - <<: *max
   nightly:
   - <<: *min
-  - <<: *avro
+  - <<: *min_avro
+  - <<: *min_excel
   - <<: *latest

.github/workflows/data/mongodb/matrix.yml

@@ -1,6 +1,6 @@
 min: &min
   # MongoDB connector does not support Spark 2
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest

.github/workflows/data/s3/matrix.yml

@@ -2,7 +2,7 @@ min: &min
   # prior image versions returns empty content of bucket root, some kind of bug
   minio-version: 2021.3.17
   # Minimal Spark version with Hadoop 3.x support
-  spark-version: 3.2.3
+  spark-version: 3.2.4
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest

.github/workflows/data/samba/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/samba/matrix.yml

@@ -0,0 +1,18 @@
+min: &min
+  python-version: '3.7'
+  os: ubuntu-latest
+
+max: &max
+  python-version: '3.11'
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - server-version: latest
+    <<: *max
+  full: &full
+  - server-version: latest
+    <<: *min
+  - server-version: latest
+    <<: *max
+  nightly: *full

.github/workflows/data/samba/tracked.txt

@@ -0,0 +1 @@
+**/samba*

.github/workflows/get-matrix.yml

@@ -41,6 +41,8 @@ on:
         value: ${{ jobs.get-matrix.outputs.matrix-s3 }}
       matrix-sftp:
         value: ${{ jobs.get-matrix.outputs.matrix-sftp }}
+      matrix-samba:
+        value: ${{ jobs.get-matrix.outputs.matrix-samba }}
       matrix-webdav:
         value: ${{ jobs.get-matrix.outputs.matrix-webdav }}
 
@@ -69,6 +71,7 @@ jobs:
       matrix-hdfs: ${{ toJson(fromJson(steps.matrix-hdfs.outputs.result)[steps.key-hdfs.outputs.key]) }}
       matrix-s3: ${{ toJson(fromJson(steps.matrix-s3.outputs.result)[steps.key-s3.outputs.key]) }}
       matrix-sftp: ${{ toJson(fromJson(steps.matrix-sftp.outputs.result)[steps.key-sftp.outputs.key]) }}
+      matrix-samba: ${{ toJson(fromJson(steps.matrix-samba.outputs.result)[steps.key-samba.outputs.key]) }}
       matrix-webdav: ${{ toJson(fromJson(steps.matrix-webdav.outputs.result)[steps.key-webdav.outputs.key]) }}
     steps:
     - name: Checkout code
@@ -635,6 +638,36 @@ jobs:
       with:
         cmd: yq -o=json '.matrix' .github/workflows/data/sftp/matrix.yml
 
+    - name: Check if Samba files are changed
+      id: changed-samba
+      uses: tj-actions/changed-files@v35
+      with:
+        files_from_source_file: .github/workflows/data/samba/tracked.txt
+        files_ignore_from_source_file: .github/workflows/data/samba/ignored.txt
+
+    - name: Print Samba files changed
+      run: |
+        echo '${{ steps.changed-samba.outputs.all_changed_files }}'
+
+    - name: Calculate Samba matrix key
+      id: key-samba
+      run: |
+        if ${{ inputs.nightly }}; then
+          key=nightly
+        elif ${{ steps.changed-base.outputs.any_changed }} || ${{ steps.changed-file.outputs.any_changed }} || ${{ steps.changed-samba.outputs.any_changed }}; then
+          key=full
+        else
+          key=small
+        fi
+        echo key=$key
+        echo key=$key >> $GITHUB_OUTPUT
+
+    - name: Get Samba matrix
+      id: matrix-samba
+      uses: mikefarah/[email protected]
+      with:
+        cmd: yq -o=json '.matrix' .github/workflows/data/samba/matrix.yml
+
     - name: Check if WebDAV files are changed
       id: changed-webdav
       uses: tj-actions/changed-files@v35

.github/workflows/nightly.yml

@@ -303,6 +303,21 @@ jobs:
       os: ${{ matrix.os }}
       with-cache: false
 
+  tests-samba:
+    name: Run Samba tests (server=${{ matrix.server-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
+    needs: [get-matrix]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.get-matrix.outputs.matrix-samba) }}
+
+    uses: ./.github/workflows/test-samba.yml
+    with:
+      server-version: ${{ matrix.server-version }}
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+      with-cache: false
+
   tests-webdav:
     name: Run WebDAV tests (server=${{ matrix.openwebdavssh-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
     needs: [get-matrix]
@@ -338,6 +353,7 @@ jobs:
     - tests-hdfs
     - tests-s3
     - tests-sftp
+    - tests-samba
     - tests-webdav
 
     steps:

.github/workflows/test-samba.yml

@@ -0,0 +1,81 @@
+name: Tests for Samba
+on:
+  workflow_call:
+    inputs:
+      server-version:
+        required: true
+        type: string
+      python-version:
+        required: true
+        type: string
+      os:
+        required: true
+        type: string
+      with-cache:
+        required: false
+        type: boolean
+        default: true
+
+jobs:
+  test-samba:
+    name: Run Samba tests (server=${{ inputs.server-version }}, python=${{ inputs.python-version }}, os=${{ inputs.os }})
+    runs-on: ${{ inputs.os }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Cache pip
+      uses: actions/cache@v3
+      if: inputs.with-cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-python-${{ inputs.python-version }}-tests-samba-${{ hashFiles('requirements/core.txt', 'requirements/samba.txt', 'requirements/tests/base.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-python-${{ inputs.python-version }}-tests-samba-${{ hashFiles('requirements/core.txt', 'requirements/samba.txt', 'requirements/tests/base.txt') }}
+          ${{ runner.os }}-python-${{ inputs.python-version }}-tests-samba-
+
+    - name: Upgrade pip
+      run: python -m pip install --upgrade pip setuptools wheel
+
+    - name: Install dependencies
+      run: |
+        pip install -I -r requirements/core.txt -r requirements/samba.txt -r requirements/tests/base.txt
+
+    # Replace with Github Actions' because of custom parameter for samba container start
+    - name: Start Samba
+      run: |
+        docker compose down -v --remove-orphans
+        docker compose up -d samba
+      env:
+        SAMBA_IMAGE: elswork/samba:${{ inputs.server-version }}
+        COMPOSE_PROJECT_NAME: ${{ github.run_id }}-samba${{ inputs.server-version }}
+
+    - name: Wait for Samba to be ready
+      run: |
+        ./docker/wait-for-it.sh -h localhost -p 445 -t 60
+
+    - name: Run tests
+      run: |
+        mkdir reports/ || echo "Directory exists"
+        sed '/^$/d' ./.env.local | sed '/^#/d' | sed 's/^/export /' > ./env
+        source ./env
+        ./pytest_runner.sh -m samba
+
+    - name: Shutdown Samba
+      if: always()
+      run: |
+        docker compose down -v --remove-orphans
+      env:
+        COMPOSE_PROJECT_NAME: ${{ github.run_id }}-samba${{ inputs.server-version }}
+
+    - name: Upload coverage results
+      uses: actions/upload-artifact@v3
+      with:
+        name: samba-${{ inputs.server-version }}-python-${{ inputs.python-version }}-os-${{ inputs.os }}
+        path: reports/*

.github/workflows/tests.yml

@@ -287,6 +287,21 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
 
+  tests-samba:
+    name: Run Samba tests (server=${{ matrix.server-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
+    needs: [get-matrix]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.get-matrix.outputs.matrix-samba) }}
+
+    uses: ./.github/workflows/test-samba.yml
+    with:
+      server-version: ${{ matrix.server-version }}
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+
+
   tests-webdav:
     name: Run WebDAV tests (server=${{ matrix.webdav-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
     needs: [get-matrix]
@@ -321,6 +336,7 @@ jobs:
     - tests-hdfs
     - tests-s3
     - tests-sftp
+    - tests-samba
     - tests-webdav
 
     steps:

.pre-commit-config.yaml

@@ -59,12 +59,12 @@ repos:
   - id: rst-inline-touching-normal
   - id: text-unicode-replacement-char
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.10.1
+  rev: v3.13.0
   hooks:
   - id: pyupgrade
     args: [--py37-plus, --keep-runtime-typing]
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.9.1
   hooks:
   - id: black
     language_version: python3

.readthedocs.yml

@@ -14,6 +14,7 @@ python:
     - ftp
     - ftps
     - hdfs
+    - samba
     - s3
     - sftp
     - webdav