Merge pull request #772 from dlt-hub/devel

master merge for 0.3.x releases

.dockerignore

@@ -9,5 +9,11 @@ __pycache__
 .egg-info
 _storage
 _test_storage
+.vscode
 Dockerfile
-.md
+.md
+_secrets
+docs
+tests
+logs
+experiments

.editorconfig

@@ -14,5 +14,5 @@ max_line_length = 200
 # Use tabs for indentation (Makefiles require tabs)
 indent_style = tab
 
-[*.{yaml,yml}]
+[*.{yaml,yml,js,md}]
 indent_size = 2

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,119 @@
+---
+name: Bug Report
+description: Report a problem with dlt
+labels: ["kind:bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for reporting a bug for dlt! Please fill out the sections below.
+        If you are not sure if this is a bug or not, please join our [Slack](https://join.slack.com/t/dlthub-community/shared_invite/zt-1n5193dbq-rCBmJ6p~ckpSFK4hCF2dYA)
+        and ask in the #3-technical-help channel.
+  - type: input
+    attributes:
+      label: dlt version
+      description: What dlt version are you using? You can find this information in the output of `dlt --version`.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the problem
+      description: Outline the problem or issue you encountered.
+      placeholder: >
+        Share the context in which the issue occurred, and details of what happened.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Expected behavior
+      description: Describe your expectation of how dlt should behave.
+      placeholder: >
+        Elaborate on why you believe the observed behavior is incorrect. Including exact error messages,
+        log fragments or screenshots. Attach files by dragging and dropping here.
+  - type: textarea
+    attributes:
+      label: Steps to reproduce
+      description: >
+        How can we replicate the issue? If it's not straightforward to reproduce, please join our [Slack](https://join.slack.com/t/dlthub-community/shared_invite/zt-1n5193dbq-rCBmJ6p~ckpSFK4hCF2dYA)
+        and ask in the #3-technical-help channel.
+      placeholder: >
+        Provide a step-by-step description of how to reproduce the problem you are running into.
+        Please be as specific as possible by providing a link to a public repository with a
+        minimalistic reproducer or a code snippet that can be used to reproduce the problem.
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Operating system
+      description: What operating system are you using?
+      multiple: true
+      options:
+        - "Linux"
+        - "macOS"
+        - "Windows"
+        - "Other"
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Runtime environment
+      description: >
+        What kind of deployment do you have? If you use a Managed Service, consider first using regular
+        channels of reporting issues for the service.
+      multiple: false
+      options:
+        - "Local"
+        - "Virtual Machine"
+        - "Docker, Docker Compose"
+        - "Kubernetes"
+        - "Airflow"
+        - "Astronomer"
+        - "Google Cloud Composer"
+        - "Google Colab"
+        - "GitHub Codespaces"
+        - "Other"
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Python version
+      description: What Python version are you using?
+      multiple: false
+      options:
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: dlt data source
+      description: What data source are you using?
+      placeholder: >
+        E.g. dlt verified source, API, CSV, etc.
+  - type: dropdown
+    attributes:
+      label: dlt destination
+      description: What type of destination are you using?
+      multiple: true
+      options:
+        - Google BigQuery
+        - DuckDB
+        - Filesystem & buckets
+        - Postgres
+        - Amazon Redshift
+        - Snowflake
+        - AWS Athena / Glue Catalog
+        - MotherDuck
+        - Weaviate
+  - type: textarea
+    attributes:
+      label: Other deployment details
+      description: Offer more details about your deployment.
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Is there anything else you'd like us to know?
+      placeholder: >
+        E.g. issue frequency, workarounds, related issues, suggestions on how to fix, etc.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,6 @@
+---
+blank_issues_enabled: true
+contact_links:
+  - name: Ask a question or get support on dlt Slack
+    url: https://join.slack.com/t/dlthub-community/shared_invite/zt-1n5193dbq-rCBmJ6p~ckpSFK4hCF2dYA
+    about: Need help or support? Join our dlt community on Slack and get assistance.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,44 @@
+---
+name: Feature Request
+description: Suggest a feature for dlt
+labels: ["kind:enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a feature for dlt!
+        If you like to discuss your idea first, please join our [Slack](https://join.slack.com/t/dlthub-community/shared_invite/zt-1n5193dbq-rCBmJ6p~ckpSFK4hCF2dYA)
+        and pose your questions in the #3-technical-help channel.
+        For minor features and improvements, feel free to open a [pull request](https://github.com/dlt-hub/dlt/pulls) directly.
+  - type: textarea
+    attributes:
+      label: Feature description
+      description: Describe the feature you'd like to see.
+      placeholder: >
+        What would you like to happen?
+  - type: dropdown
+    attributes:
+      label: Are you a dlt user?
+      description: Please tell us how you are using dlt.
+      multiple: false
+      options:
+        - "Yes, I'm already a dlt user."
+        - "Yes, I run dlt in production."
+        - "Yes, I use it for fun."
+        - "I'd consider using dlt, but it's lacking a feature I need."
+  - type: textarea
+    attributes:
+      label: Use case
+      description: Is your feature request related to a problem? Please describe.
+      placeholder: >
+        What problem are you trying to solve?
+  - type: textarea
+    attributes:
+      label: Proposed solution
+      description: Describe the solution you'd like to see.
+      placeholder: >
+        What would you like to see changed?
+  - type: textarea
+    attributes:
+      label: Related issues
+      description: Is there currently another issue associated with this?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,25 @@
+<!--
+Thank you for submitting a pull request! Please provide a brief description of your changes below.
+-->
+### Description
+
+
+<!--
+Please link any related issues. This helps us keep the PR focused and merge it faster.
+-->
+### Related Issues
+
+- Fixes #...
+- Closes #...
+- Resolves #...
+
+<!--
+Provide any additional context about the PR here.
+-->
+### Additional Context
+
+<!--
+Please ensure that
+    - you have read the [Contributing to dlt](../CONTRIBUTING.md) guide.
+    - you have run the tests locally and they have passed before submitting your PR.
+-->

.github/weaviate-compose.yml

@@ -0,0 +1,34 @@
+version: '3.4'
+services:
+  weaviate:
+    command:
+    - --host
+    - 0.0.0.0
+    - --port
+    - '8080'
+    - --scheme
+    - http
+    image: semitechnologies/weaviate:1.21.1
+    ports:
+    - 8080:8080
+    volumes:
+    - weaviate_data
+    restart: on-failure:0
+    environment:
+      QUERY_DEFAULTS_LIMIT: 25
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      DEFAULT_VECTORIZER_MODULE: 'none'
+      ENABLE_MODULES: 'text2vec-contextionary,text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai'
+      CONTEXTIONARY_URL: contextionary:9999
+      CLUSTER_HOSTNAME: 'node1'
+  contextionary:
+    environment:
+      OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75
+      EXTENSIONS_STORAGE_MODE: weaviate
+      EXTENSIONS_STORAGE_ORIGIN: http://weaviate:8080
+      NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE: 5
+      ENABLE_COMPOUND_SPLITTING: 'false'
+    image: semitechnologies/contextionary:en0.16.0-v1.2.1
+    ports:
+    - 9999:9999

.github/workflows/get_docs_changes.yml

@@ -0,0 +1,48 @@
+name: get docs changes
+
+on:
+  workflow_call:
+    outputs:
+      changes_outside_docs:
+        description: "Changes outside docs"
+        value: ${{ jobs.get_docs_changes.outputs.changes_outside_docs }}
+
+env:
+  EXCLUDED_FILE_PATTERNS: '^docs/|^README.md|^LICENSE\.txt|\.editorconfig|\.gitignore'
+
+
+jobs:
+  get_docs_changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Check changes outside docs
+        id: check_changes
+        run: |
+          echo "base.sha: ${{ github.event.pull_request.base.sha }}"
+          echo "head.sha: ${{ github.event.pull_request.head.sha }}"
+
+          merge_base_sha=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
+
+          echo "merge_base_sha: $merge_base_sha"
+
+          changes_outside_docs=$(
+            git diff --name-only $merge_base_sha ${{ github.event.pull_request.head.sha }} \
+              | grep -vE "${{ env.EXCLUDED_FILE_PATTERNS }}" || true
+          )
+
+          echo $changes_outside_docs
+          if [ -z "$changes_outside_docs" ]; then
+            echo "No changes outside docs. Skipping tests."
+            echo "changes_outside_docs=false" >> $GITHUB_OUTPUT
+          else
+            echo "Changes detected outside docs."
+            echo "changes_outside_docs=true" >> $GITHUB_OUTPUT
+          fi

.github/workflows/lint.yml

@@ -5,12 +5,18 @@ on:
   pull_request:
     branches:
       - master
+      - devel
   workflow_dispatch:
 
 jobs:
+  get_docs_changes:
+    uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
     name: Runs mypy, flake and bandit
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+
     runs-on: ubuntu-latest
 
     steps:
@@ -19,7 +25,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10.x"
 
@@ -32,17 +38,23 @@ jobs:
 
       - name: Load cached venv
         id: cached-poetry-dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
       - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
+        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --all-extras --with airflow
 
-      - name: Install self
-        run: poetry install --no-interaction
+      # - name: Install self
+      #   run: poetry install --no-interaction
 
       - name: Run lint
         run: make lint
+
+      # - name: print envs
+      #   run: |
+      #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
+      #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
+      #     echo "The GitHub repo is: $GITHUB_REPOSITORY"