Sync from soda-core

.env.example

@@ -44,3 +44,13 @@ ATHENA_SCHEMA=***
 
 # Create and test checks with views instead of tables
 TEST_WITH_VIEWS=false
+
+CONTRACTS_POSTGRES_HOST=***
+CONTRACTS_POSTGRES_USERNAME=***
+CONTRACTS_POSTGRES_PASSWORD=***
+CONTRACTS_POSTGRES_DATABASE=***
+
+ATLAN_API_KEY=***
+
+FABRIC_ENDPOINT=***
+FABRIC_DWH=***

.github/workflows/build-docker.yml

@@ -8,35 +8,30 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      - 
-        name: check if a version tag
+      - name: check if a version tag
         id: check-version-tag
         run: |
           if [[ ${{ github.event.client_payload.tag }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
               echo ::set-output name=match::true
           fi
-      - 
-        name: Sleep for 900s
+      - name: Sleep for 900s
         if: steps.check-version-tag.outputs.match == 'true'
         uses: juliangruber/sleep-action@v1
         with:
           time: 900s
-      - 
-        name: check if a version tag in ref
+      - name: check if a version tag in ref
         if: steps.check-version-tag.outputs.match == 'true'
         id: get-version-tag-in-ref
         run: |
           if [[ ${{ github.event.client_payload.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
               echo ::set-output name=versiontag::$(echo "${{github.event.client_payload.ref}}" | cut -d / -f 3)
           fi
-      -
-        name: Checkout
+      - name: Checkout
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.client_payload.ref }}
-      -
-        name: Docker meta
+      - name: Docker meta
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         id: meta
         uses: docker/metadata-action@v4
@@ -45,27 +40,25 @@ jobs:
             sodadata/soda-core
           tags: |
             type=raw,value=${{ github.event.client_payload.tag }}
-      -
-        name: Set up QEMU
+            type=semver,pattern=v{{major}}.{{minor}},value=${{ github.event.client_payload.tag }}
+            type=semver,pattern=v{{major}},value=${{ github.event.client_payload.tag }}
+      - name: Set up QEMU
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         uses: docker/setup-buildx-action@v2
-      -
-        name: Login to DockerHub
+      - name: Login to DockerHub
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Build and push
+      - name: Build and push
         if: github.event.client_payload.tag == steps.get-version-tag-in-ref.outputs.versiontag
         uses: docker/build-push-action@v3
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/main.workflow.yml

@@ -17,6 +17,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
+        with:
+            python-version: '3.11.x'
       - uses: pre-commit/[email protected]
         with:
           extra_args: --all-files
@@ -49,11 +51,10 @@ jobs:
     env:
       DATA_SOURCE: ${{ matrix.data-source }}
       PYTHON_VERSION: ${{ matrix.python-version }}
-      SNOWFLAKE_HOST: ${{ secrets.SNOWFLAKE_HOST }}
-      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
-      SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
-      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_CI_ACCOUNT }}
+      SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_CI_USERNAME }}
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_CI_DATABASE }}
       SNOWFLAKE_SCHEMA: "public"
       BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }}
       BIGQUERY_DATASET: "test"
@@ -169,6 +170,35 @@ jobs:
       - name: Test with tox
         run: |
           tox -- soda -k soda/scientific
+  test-contracts:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.9"
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsasl2-dev
+          python -m pip install --upgrade pip
+          cat dev-requirements.in | grep tox | xargs pip install
+
+      - name: Test with tox
+        run: |
+          tox -- soda -k soda/contracts
+
   publish-pypi:
     name: Build & Publish Package
     if: contains(github.ref, 'refs/tags/')

.github/workflows/pr.workflow.yml

@@ -13,6 +13,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
+        with:
+            python-version: '3.11.x'
       - uses: pre-commit/[email protected]
         with:
           extra_args: --all-files
@@ -35,15 +37,13 @@ jobs:
           - "duckdb"
           - "dask"
 
-
     env:
       DATA_SOURCE: ${{ matrix.data-source }}
       PYTHON_VERSION: ${{ matrix.python-version }}
-      SNOWFLAKE_HOST: ${{ secrets.SNOWFLAKE_HOST }}
-      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
-      SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
-      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
+      SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_CI_ACCOUNT }}
+      SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_CI_USERNAME }}
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+      SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_CI_DATABASE }}
       SNOWFLAKE_SCHEMA: "public"
       BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }}
       BIGQUERY_DATASET: "test"
@@ -61,7 +61,7 @@ jobs:
       MYSQL_PASSWORD: sodacore
       MYSQL_ROOT_PASSWORD: sodacore
       SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }}
-      
+
     steps:
       - uses: actions/checkout@v3
 
@@ -81,8 +81,8 @@ jobs:
 
       - name: Test with tox
         run: |
-          tox --exit-and-dump-after 3600 -- soda -k soda/core
-          tox --exit-and-dump-after 3600 -- soda -k soda/${{ matrix.data-source }}
+          tox -- soda -k soda/core
+          tox -- soda -k soda/${{ matrix.data-source }}
         env:
           test_data_source: ${{ matrix.data-source }}
 
@@ -113,7 +113,7 @@ jobs:
 
       - name: Test with tox
         run: |
-          tox --exit-and-dump-after 3600 -- soda -k soda/core
+          tox -- soda -k soda/core
         env:
           test_data_source: postgres
           WESTMALLE: BETTER_THAN_LA_TRAPPE
@@ -145,4 +145,33 @@ jobs:
 
       - name: Test with tox
         run: |
-          tox --exit-and-dump-after 3600 -- soda -k soda/scientific
+          tox -- soda -k soda/scientific
+
+  test-contracts:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.9"
+
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsasl2-dev
+          python -m pip install --upgrade pip
+          cat dev-requirements.in | grep tox | xargs pip install
+
+      - name: Test with tox
+        run: |
+          tox -- soda -k soda/contracts

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ files: ^soda/
 exclude: antlr/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: check-added-large-files
@@ -18,24 +18,25 @@ repos:
       - id: debug-statements
       - id: detect-private-key
       - id: end-of-file-fixer
-  - repo: https://github.com/humitos/mirrors-autoflake.git
-    rev: v1.1
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
     hooks:
       - id: autoflake
         args: ["--in-place", "--remove-all-unused-imports"]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
-        args: [--py37-plus]
+        exclude: _models?\.py$
+        args: [--py38-plus, --keep-runtime-typing]
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         additional_dependencies: [toml]
         name: Sort imports using isort
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.10.0
     hooks:
       - id: black
         name: Run black formatter

.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "#00D891"  # Primary color
+backgroundColor = "#F5F7F7"  # Background color
+# secondaryBackgroundColor = "#00D891"  # Color for the sidebar and other secondary backgrounds
+textColor = "#262730"  # Primary text color
+font = "sans serif"  # Font style (e.g., "sans serif", "serif", "monospace")

README.md

@@ -7,8 +7,13 @@
   <a href="https://join.slack.com/t/soda-community/shared_invite/zt-m77gajo1-nXJF7JtbbRht2zwaiLb9pg"><img alt="Slack" src="https://img.shields.io/badge/chat-slack-green.svg"></a>
   <a href="#"><img src="https://static.pepy.tech/personalized-badge/soda-core?period=total&units=international_system&left_color=black&right_color=green&left_text=Downloads"></a>
 </p>
-<br />
 
+<hr />
+
+> [!IMPORTANT]  
+> **🚀 We're hiring! Are you passionate about open-source and love working on projects like Soda Core? Join our team as a Software Engineer and help shape the future of data quality tools. [Apply now!](https://careers.soda.io/o/software-engineer-data-testing-python-data-engineering-mediorsenior?source=gh-core)**
+
+<hr />
 
 &#10004;  An open-source, CLI tool and Python library for data quality testing<br />
 &#10004;  Compatible with the <a href="https://docs.soda.io/soda-cl/soda-cl-overview.html" target="_blank">Soda Checks Language (SodaCL)</a>  <br />
@@ -22,7 +27,12 @@ When it runs a scan on a dataset, Soda Core executes the checks to find invalid,
 
 #### Soda Library 
 
-Consider using **[Soda Library](https://docs.soda.io/soda/quick-start-sip.html)**, an extension of Soda Core that offers more features and functionality, and enables you to connect to a [Soda Cloud](https://docs.soda.io/soda-cloud/overview.html) account to collaborate with your team on data quality.
+Consider migrating to **[Soda Library](https://docs.soda.io/soda/quick-start-sip.html)**, an extension of Soda Core that offers more features and functionality, and enables you to connect to a [Soda Cloud](https://docs.soda.io/soda-cloud/overview.html) account to collaborate with your team on data quality.
+* Use [Group by](https://docs.soda.io/soda-cl/group-by.html) and [Group Evolution](https://docs.soda.io/soda-cl/group-evolution.html) configurations to intelligently group check results
+* Leverage [Reconciliation checks](https://docs.soda.io/soda-cl/recon.html) to compare data between data sources for data migration projects.
+* Use [Schema Evolution](https://docs.soda.io/soda-cl/schema.html#define-schema-evolution-checks) checks to automatically validate schemas.
+* Set up [Anomaly Detection](https://docs.soda.io/soda-cl/anomaly-detection.html) checks to automatically learn patterns and discover anomalies in your data.
+
 [Install Soda Library](https://docs.soda.io/soda-library/install.html) and get started with a 45-day free trial.
 
 <br />

dev-requirements.in

@@ -1,16 +1,20 @@
-pip-tools~=6.5
+pip-tools~=7.3
 pytest~=7.0
 python-dotenv~=1.0
-tox~=4.6
-tox-docker~=4.1
+tox~=4.12
+tox-docker~=5.0
 pytest-html~=3.1
 pytest-cov~=3.0
 faker~=13.3
-tbump~=6.7
+tbump~=6.11
 black==22.6.0
 typing_extensions>=4.3.0,<5
 urllib3~=1.26
 pygments~=2.11
 readme-renderer~=32.0
 certifi>=2022.12.07
 wheel>=0.38.1
+docutils<0.21 # 0.21 dropped py38 support, remove this after py38 support is gone
+pre-commit<3.6 # 3.6 dropped py38, remove this after py38 support is gone
+requests>=2.32.3
+