diff --git a/.github/workflows/test_destination_filesystem_sftp.yml b/.github/workflows/test_destination_filesystem_sftp.yml deleted file mode 100644 index 74a2fdf626..0000000000 --- a/.github/workflows/test_destination_filesystem_sftp.yml +++ /dev/null @@ -1,83 +0,0 @@ - -name: test | filesystem sftp - -on: - pull_request: - branches: - - master - - devel - workflow_dispatch: - schedule: - - cron: '0 2 * * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 - RUNTIME__LOG_LEVEL: ERROR - RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }} - - ACTIVE_DESTINATIONS: "[\"filesystem-sftp\"]" - ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" - -jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} - - run_loader: - name: test | filesystem sftp tests - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - defaults: - run: - shell: bash - runs-on: "ubuntu-latest" - - steps: - - - name: Check out - uses: actions/checkout@master - - - name: Start SFTP server - run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: "3.10.x" - - - name: Install Poetry - uses: snok/install-poetry@v1.3.2 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - - - name: Install dependencies - run: poetry install --no-interaction -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline - - - run: | - poetry run pytest tests/load/filesystem_sftp - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - - run: | - poetry run pytest tests/load/filesystem_sftp - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - - name: Stop SFTP server - if: always() - run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" down -v diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 78ea23ec1c..bf159afeb0 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -74,6 +74,9 @@ jobs: - name: Start weaviate run: docker compose -f ".github/weaviate-compose.yml" up -d + + - name: Start SFTP server + run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d - name: Setup Python uses: actions/setup-python@v4 @@ -110,3 +113,7 @@ jobs: - name: Stop weaviate if: always() run: docker compose -f ".github/weaviate-compose.yml" down -v + + - name: Stop SFTP server + if: always() + run: docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" down -v diff --git a/tests/load/filesystem_sftp/__init__.py b/tests/load/filesystem_sftp/__init__.py index c5e215e0c5..0d23f8002b 100644 --- a/tests/load/filesystem_sftp/__init__.py +++ b/tests/load/filesystem_sftp/__init__.py @@ -1,3 +1,3 @@ from tests.utils import skip_if_not_active -skip_if_not_active("filesystem-sftp") +skip_if_not_active("filesystem") diff --git a/tests/utils.py b/tests/utils.py index 6ba5771d3d..1b81881470 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -40,7 +40,6 @@ "postgres", "snowflake", "filesystem", - "filesystem-sftp", "weaviate", "dummy", "motherduck", @@ -55,7 +54,6 @@ } NON_SQL_DESTINATIONS = { "filesystem", - "filesystem-sftp", "weaviate", "dummy", "qdrant",