azure-pipelines.yml

# This file is responsible for configuring the `great_expectations` pipeline (https://dev.azure.com/great-expectations/great_expectations/_build)
#
# The pipeline is run under the following conditions:
#   - On the develop branch when a weekly release is being cut
#   - On the develop branch as scheduled by the below cron job
#
#  `great_expectations` runs the entire test suite and is meant to safeguard the codebase against any errors that may have slipped through the cracks
#  from `dependency_graph`. Additionally, it is responsible for deployment during the weekly release cut.

schedules:
- cron: 0 */3 * * *
  displayName: Scheduled Runs
  branches:
    include:
    - develop
  always: false # Will only trigger if the state of the codebase has changed since the last scheduled run

trigger:
  tags:
    include:
      - '*' # Should be SemVer for a successful release but any tag will trigger the build

resources:
  containers:
  - container: postgres
    image: postgres:11
    ports:
    - 5432:5432
    env:
      POSTGRES_DB: "test_ci"
      POSTGRES_HOST_AUTH_METHOD: "trust"
  - container: mysql
    image: mysql:8.0.20
    ports:
      - 3306:3306
    env:
      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
      MYSQL_DATABASE: test_ci
  - container: mssql
    image: mcr.microsoft.com/mssql/server:2019-latest
    env:
      ACCEPT_EULA: Y
      MSSQL_SA_PASSWORD: ReallyStrongPwd1234%^&*
      MSSQL_DB: test_ci
      MSSQL_PID: Developer
    ports:
      - 1433:1433
  - container: trino
    image: trinodb/trino:400
    ports:
      - 8088:8080

# The pipeline is run under two primary conditions: if cutting a release or as scheduled by the above cron job.
variables:
  isReleasePrep: $[contains(variables['Build.SourceBranch'], 'release')]
  isRelease: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')]
  isScheduled: $[and(eq(variables['Build.SourceBranch'], 'refs/heads/develop'), eq(variables['Build.Reason'], 'Schedule'))]
  isManual: $[eq(variables['Build.Reason'], 'Manual')]

stages:
  - stage: lint
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
      - job: lint
        condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: 3.7
            displayName: 'Use Python 3.7'

          - script: |
              pip install isort[requirements]==5.10.1 flake8==5.0.4 black==22.3.0 pyupgrade==2.7.2 invoke==1.7.1
              EXIT_STATUS=0
              invoke fmt --check || EXIT_STATUS=$?
              invoke lint || EXIT_STATUS=$?
              pyupgrade --py3-plus || EXIT_STATUS=$?
              exit $EXIT_STATUS

  - stage: custom_checks
    pool:
      vmImage: 'ubuntu-latest'

    jobs:
    - job: type_hint_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
      steps:
      - script: |
          pip install --requirement reqs/requirements-dev-contrib.txt
          invoke type-coverage
        name: TypeHintChecker

    - job: static_type_check
      condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
      steps:
      - script: |
          pip install --requirement reqs/requirements-dev-contrib.txt
          invoke type-check --install-types --warn-unused-ignores --pretty
        name: StaticTypeCheck

    - job: docstring_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
      steps:
      - bash: python scripts/check_docstring_coverage.py
        name: DocstringChecker

    - job: unused_import_checker
      condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
      steps:
      - script: |
          pip install --requirement reqs/requirements-dev-contrib.txt
          # https://www.flake8rules.com/rules/F401.html - Prunes the dgtest graph to improve accuracy
          flake8 --select F401 great_expectations tests
        name: UnusedImportChecker

  - stage: docker_tests
    dependsOn: [lint, custom_checks]
    condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
    pool:
      vmImage: 'ubuntu-latest'
    jobs:
      - job: make_suffix
        steps:
          - bash: |
              EPOCH=`date +%s`
              SUFFIX="develop_${EPOCH}_${RANDOM}"
              echo "Generated image suffix: ${SUFFIX}"
              echo "##vso[task.setvariable variable=IMAGE_SUFFIX;isOutput=true]${SUFFIX}"
            name: suffix

      - job: build_push
        dependsOn: make_suffix
        condition: succeeded()
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
          GE_USAGE_STATISTICS_URL: "https://qa.stats.greatexpectations.io/great_expectations/v1/usage_statistics"
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              # We repull the develop branch. We could build the local branch that is already present but we should
              # prune it's depth since we don't need the git history.
              CMD="docker buildx build -f docker/Dockerfile.tests --tag greatexpectations/test:${DOCKER_TAG} "
              CMD+="--target test --build-arg PYTHON_VERSION=$(python.version) "
              CMD+="--build-arg GE_USAGE_STATISTICS_URL=\"$(GE_USAGE_STATISTICS_URL)\" "
              CMD+="--build-arg SOURCE=github --build-arg BRANCH=develop ."
              echo ${CMD}
              eval ${CMD}
            displayName: 'Build python $(python.version) image'

          - bash: |
              docker login -u ${DOCKER_USER} -p ${DOCKER_TOKEN}
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PUSH="docker push greatexpectations/test:${DOCKER_TAG}"
              echo ${PUSH}
              eval ${PUSH}
              echo "Docker image can be found at https://hub.docker.com/r/greatexpectations/test/tags"
              echo "Pull it locally using: docker pull greatexpectations/test:${DOCKER_TAG}"
            displayName: 'Push python $(python.version) image'
            env:
              DOCKER_USER: $(DOCKER_USER)
              DOCKER_TOKEN: $(DOCKER_TOKEN)

      - job: test_comprehensive
        dependsOn: [make_suffix, build_push]
        condition: succeeded()
        timeoutInMinutes: 120
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        services:
          postgres: postgres
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              mkdir coverage
              PYTEST="docker run -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) "
              PYTEST+="-e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION) "
              PYTEST+="--network=host --mount type=bind,source=${PWD}/coverage,target=/coverage "
              # We don't run e2e because we don't want to test connections to external dependencies here.
              PYTEST+="greatexpectations/test:${DOCKER_TAG} /bin/bash -c \"pytest -m 'not e2e' "
              PYTEST+="--random-order --postgresql --cloud "
              # We should be able to specify --cov-report=xml:path but this doesn't work so we write to the default
              # path and then mv it.
              PYTEST+="--junitxml=/coverage/junit/test-results.xml --cov=. --cov-report=xml --cov-report=html:/coverage/htmlcov "
              PYTEST+="--ignore=tests/cli --ignore=tests/integration/usage_statistics "
              # The onboarding data assistant tests do not succeed in random order and are run on their own below.
              PYTEST+="--ignore=tests/rule_based_profiler/data_assistant/test_onboarding_data_assistant.py "
              # The last \" closes the quote from bash -c \"
              PYTEST+="&& mv coverage.xml /coverage/coverage.xml\""

              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest $(python.version)'
            env:
              AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
              AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
              AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

          - task: PublishTestResults@2
            condition: succeededOrFailed()
            inputs:
              testResultsFiles: '$(System.DefaultWorkingDirectory)/coverage/**/test-*.xml'
              testRunTitle: 'Publish test results for Python $(python.version)'

          - task: PublishCodeCoverageResults@1
            inputs:
              codeCoverageTool: Cobertura
              summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage/coverage.xml'
              reportDirectory: '$(System.DefaultWorkingDirectory)/coverage/htmlcov'

      - job: test_onboarding_data_assistant
        dependsOn: [ make_suffix, build_push ]
        condition: succeeded()
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run greatexpectations/test:${DOCKER_TAG} "
              # These tests currently do no succeed in random order.
              PYTEST+="pytest tests/rule_based_profiler/data_assistant/test_onboarding_data_assistant.py"
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest Onboarding Data Assistant'

      - job: test_spark
        dependsOn: [ make_suffix, build_push ]
        condition: succeeded()
        timeoutInMinutes: 120
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run greatexpectations/test:${DOCKER_TAG} "
              # We do NOT randomize the spark tests.
              # Spark only allows 1 context per JVM process, so we share it between tests.
              # Currently these tests order dependent.
              PYTEST+="pytest --spark --ignore=tests/cli --ignore=tests/integration/usage_statistics"
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest Spark'

      - job: test_mysql
        dependsOn: [ make_suffix, build_push ]
        condition: succeeded()
        timeoutInMinutes: 120
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        services:
          mysql: mysql
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              printf "Waiting for MySQL database to accept connections: "
              until mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --execute "SHOW DATABASES"; do
                printf '.'
                sleep 1;
              done;
              printf "\n"
            displayName: 'Wait for MySQL'
          - bash: |
              mkdir script
              echo "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));" > script/mysql_setup.sql
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) "
              PYTEST+="-e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION) "
              PYTEST+="--network=host --mount type=bind,source=${PWD}/script,target=/script "
              PYTEST+="greatexpectations/test:${DOCKER_TAG} /bin/bash -c \""
              PYTEST+="mysql --host=localhost --protocol=TCP --port=3306 --user=root --password='' --reconnect < /script/mysql_setup.sql && "
              # The last \" closes the quote from bash -c \".
              PYTEST+="pytest --mysql --random-order tests/test_definitions tests/expectations\""
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest MySQL'
            env:
              AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
              AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
              AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

      - job: test_trino
        dependsOn: [make_suffix, build_push]
        condition: succeeded()
        timeoutInMinutes: 120
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        services:
          trino: trino
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              printf 'Waiting for Trino database to accept connections'
              sleep 30
            displayName: 'Wait for Trino'
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) "
              PYTEST+="-e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION) "
              PYTEST+="--network=host greatexpectations/test:${DOCKER_TAG} "
              PYTEST+="pytest --trino --random-order tests/test_definitions tests/expectations"
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest Trino'
            env:
              AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
              AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
              AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

      - job: test_cli
        dependsOn: [make_suffix, build_push]
        condition: succeeded()
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        services:
          postgres: postgres
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run --network=host greatexpectations/test:${DOCKER_TAG} "
              PYTEST+="pytest --postgresql --spark --aws-integration tests/cli"
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest cli'

      - job: test_min_versions
        dependsOn: [make_suffix, build_push]
        condition: succeeded()
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
              min_numpy: '1.18.5'
              min_pandas: '1.1.0'
            Python38:
              python.version: '3.8'
              min_numpy: '1.18.5'
              min_pandas: '1.1.0'
            Python39:
              python.version: '3.9'
              min_numpy: '1.19.3'
              min_pandas: '1.1.3'
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) "
              PYTEST+="-e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -e AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION) "
              PYTEST+="--network=host greatexpectations/test:${DOCKER_TAG} bash -c \""
              PYTEST+="printf 'Y\nY\n' | pip uninstall numpy pandas && "
              PYTEST+="printf 'Y\nY\n' | pip install numpy==$(min_numpy) pandas==$(min_pandas) && "
              PYTEST+="pytest --no-sqlalchemy --random-order --ignore=tests/cli --ignore=tests/integration/usage_statistic "
              PYTEST+="--ignore=tests/rule_based_profiler/data_assistant/test_onboarding_data_assistant.py\""
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest min versions'
            env:
              AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
              AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
              AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

      - job: test_usage_stats
        dependsOn: [make_suffix, build_push]
        condition: succeeded()
        strategy:
          matrix:
            Python37:
              python.version: '3.7'
            Python38:
              python.version: '3.8'
            Python39:
              python.version: '3.9'
        variables:
          IMAGE_SUFFIX: $[ dependencies.make_suffix.outputs['suffix.IMAGE_SUFFIX'] ]
        steps:
          - bash: |
              DOCKER_TAG="$(python.version)_$(IMAGE_SUFFIX)"
              PYTEST="docker run --network=host greatexpectations/test:${DOCKER_TAG} "
              PYTEST+="pytest --no-sqlalchemy --aws-integration --random-order tests/integration/usage_statistics"
              echo ${PYTEST}
              eval ${PYTEST}
            displayName: 'Pytest Usage Stats'


  - stage: import_ge
    dependsOn: [lint]
    pool:
      vmImage: 'ubuntu-latest'
    jobs:
      - job: import_ge
        condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))

        strategy:
         matrix:
           Python37:
             python.version: '3.7'
           Python38:
             python.version: '3.8'
           Python39:
             python.version: '3.9'


        steps:
         - task: UsePythonVersion@0
           inputs:
             versionSpec: '$(python.version)'
           displayName: 'Use Python $(python.version)'

         - bash: python -m pip install --upgrade pip
           displayName: 'Update pip'

         - script: |
             pip install  .
           displayName: 'Install GX and required dependencies (i.e. not sqlalchemy)'

         - script: |
             python -c "import great_expectations as gx; print('Successfully imported GX Version:', gx.__version__)"
           displayName: 'Import Great Expectations'

  - stage: db_integration
    pool:
      vmImage: 'ubuntu-latest'
    dependsOn: [lint, import_ge, custom_checks]
    jobs:
      - job: mssql
        condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
        timeoutInMinutes: 120

        services:
          mssql: mssql

        variables:
          python.version: '3.8'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip
            displayName: 'Update pip'

          - script: |
              sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt

          - script: |
              pip install \
                --requirement reqs/requirements-dev-test.txt \
                --requirement reqs/requirements-dev-sqlalchemy.txt \
                --constraint constraints-dev.txt
              pip install --requirement requirements.txt
              pip install .
            displayName: 'Install dependencies'

          - script: |
              pip install pytest pytest-cov pytest-azurepipelines pytest-random-order
              pytest tests/test_definitions tests/expectations \
                --mssql \
                --napoleon-docstrings \
                --junitxml=junit/test-results.xml \
                --cov=. \
                --cov-report=xml \
                --cov-report=html
            displayName: 'Pytest mssql'

  - stage: deploy
    condition: and(succeeded(), eq(variables.isRelease, true))
    pool:
      vmImage: 'ubuntu-latest'
    dependsOn: [import_ge, custom_checks, lint, db_integration, docker_tests]
    jobs:
      - job: deploy
        variables:
          python.version: '3.8'

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '$(python.version)'
            displayName: 'Use Python $(python.version)'

          - bash: python -m pip install --upgrade pip
            displayName: 'Update pip'

          - script: |
              pip install twine
              pip install wheel
              git config --global user.email "team@greatexpectations.io"
              git config --global user.name "Great Expectations"
            displayName: 'Prepare packaging'

          # Build the python distribution from source
          - script: |
              python setup.py sdist
              python setup.py bdist_wheel
            displayName: 'Build distribution'

          - task: TwineAuthenticate@1
            inputs:
              pythonUploadServiceConnection: pypi

          # Use command line script to 'twine upload', use -r to pass the repository name and --config-file to pass the environment variable set by the authenticate task.
          - script: |
              python -m twine upload -r great-expectations --config-file $(PYPIRC_PATH) dist/*
            displayName: 'Upload'