azure-pipelines.yml

# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python

trigger:
- main

jobs:
- job: 'EvalChanges'
  displayName: 'Analyze changed files to determine which job to run'
  pool:
    vmImage: 'macOS-10.15'
  steps:
  # We want to enforce the following rules for PRs:
  # * if all modifications are to README.md
  #     no testing is needed
  # * if there are modifications to docs/* or to any code
  #     then docs need to be built to verify consistency
  # * if there are modifications to notebooks/* or to any code
  #     then notebooks need to be run to verify consistency
  # * for any code changes (or changes to metadata files)
  #     linting and testing should be run
  # For a PR build, HEAD will be the merge commit, and we want to diff against the base branch,
  #  which will be the first parent: HEAD^ 
  # (For non-PR changes, we will always perform all CI tasks)
  - powershell: |
      if ($env:BUILD_REASON -eq 'PullRequest') {
        $editedFiles = git diff HEAD^ --name-only
        $editedFiles # echo edited files to enable easier debugging
        $codeChanges = $false
        $docChanges = $false
        $nbChanges = $false
        $changeType = "none"
        foreach ($file in $editedFiles) {
          switch -Wildcard ($file) {
            "README.md" { Continue }
            "econml/_version.py" { Continue }
            "prototypes/*" { Continue }
            "images/*" { Continue }
            "doc/*" { $docChanges = $true; Continue }
            "notebooks/*" { $nbChanges = $true; Continue }
            default { $codeChanges = $true; Continue }
          }
        }
      }
      Write-Host "##vso[task.setvariable variable=buildDocs;isOutput=true]$(($env:BUILD_REASON -ne 'PullRequest') -or ($docChanges -or $codeChanges))"
      Write-Host "##vso[task.setvariable variable=buildNbs;isOutput=true]$(($env:BUILD_REASON -ne 'PullRequest') -or ($nbChanges -or $codeChanges))"
      Write-Host "##vso[task.setvariable variable=testCode;isOutput=true]$(($env:BUILD_REASON -ne 'PullRequest') -or $codeChanges)"
    name: output
    displayName: 'Determine type of code change'

- template: azure-pipelines-steps.yml
  parameters:
    versions: ['3.8']
    images: ['ubuntu-18.04']
    package: '-e .[all]'
    job:
      job: 'Docs'
      displayName: 'Build documentation'  
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.buildDocs'], 'True')
      steps:
      - script: 'sudo apt-get -yq install graphviz'
        displayName: 'Install graphviz'

      - script: 'pip install sklearn-contrib-lightning'
        displayName: 'Install lightning'        

      - script: 'pip install git+https://github.com/slundberg/shap.git@d1d2700acc0259f211934373826d5ff71ad514de'
        displayName: 'Install specific version of shap'  

      - script: 'pip install sphinx sphinx_rtd_theme'
        displayName: 'Install sphinx'

      - script: 'python setup.py build_sphinx -W'
        displayName: 'Build documentation'

      - publish: 'build/sphinx/html'
        artifact: 'Documentation'
        displayName: 'Publish documentation as artifact'

      - script: 'python setup.py build_sphinx -b doctest'
        displayName: 'Run doctests'


- template: azure-pipelines-steps.yml
  parameters:
    versions: ['3.8']
    images: ['ubuntu-18.04']
    package: '-e .[tf,plt]'
    job:
      job: 'Notebooks_cust'
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.buildNbs'], 'True')
      displayName: 'Notebooks (Customer Solutions)'
      steps:
      # Work around https://github.com/pypa/pip/issues/9542
      - script: 'pip install -U numpy~=1.21.0'
        displayName: 'Upgrade numpy'
        
      - script: 'pip install pytest pytest-runner jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm && pip list && python setup.py pytest'
        displayName: 'Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "notebook"'
          NOTEBOOK_DIR_PATTERN: 'CustomerScenarios'

      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Notebooks'
        condition: succeededOrFailed()

- template: azure-pipelines-steps.yml
  parameters:
    versions: ['3.8']
    images: ['ubuntu-18.04']
    package: '-e .[tf,plt]'
    job:
      job: 'Notebooks_noncust'
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.buildNbs'], 'True')
      displayName: 'Notebooks (except Customer Solutions)'
      steps:
      # Work around https://github.com/pypa/pip/issues/9542
      - script: 'pip install -U numpy~=1.21.0'
        displayName: 'Upgrade numpy'
        
      - script: 'pip install pytest pytest-runner jupyter jupyter-client nbconvert nbformat seaborn xgboost tqdm && python setup.py pytest'
        displayName: 'Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "notebook"'
          NOTEBOOK_DIR_PATTERN: '(?!CustomerScenarios)'

      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Notebooks'
        condition: succeededOrFailed()


# - job: 'AutoML'
#   dependsOn: 'EvalChanges'
#   condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
#   variables:
#     python.version: '3.6'
#   pool:
#     vmImage: 'ubuntu-18.04'
#   steps:
#   - template: azure-pipelines-steps.yml
#     parameters:
#       body:
#         - task: AzureCLI@2
#           displayName: 'AutoML tests'
#           inputs:
#             azureSubscription: 'automl'
#             scriptLocation: 'inlineScript'
#             scriptType: 'pscore'
#             powerShellIgnoreLASTEXITCODE: '' # string for now due to https://github.com/microsoft/azure-pipelines-tasks/issues/12266
#             inlineScript: |
#               $env:SUBSCRIPTION_ID = az account show --query id -o tsv
#               python setup.py pytest
#           env:
#             WORKSPACE_NAME: 'testWorkspace'
#             RESOURCE_GROUP: 'testingAutoMLEconML'
#             PYTEST_ADDOPTS: '-m "automl" -n 0'
#             COVERAGE_PROCESS_START: 'setup.cfg'

#         - task: PublishTestResults@2
#           displayName: 'Publish Test Results **/test-results.xml'
#           inputs:
#             testResultsFiles: '**/test-results.xml'
#             testRunTitle: 'AutoML'
#           condition: succeededOrFailed()
#       package: '.[automl]'

- template: azure-pipelines-steps.yml
  parameters:
    versions: ['3.8']
    images: ['macOS-10.15']
    job:
      job: 'Linting'
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
      steps:
      - script: 'pip install pycodestyle && pycodestyle econml'
        failOnStderr: true
        displayName: Linting

- template: azure-pipelines-steps.yml
  parameters:
    package: '-e .[tf,plt]'
    job:
      job: Tests_main
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
      displayName: 'Run tests (main)'
      steps:
      - script: 'pip install pytest pytest-runner "coverage<6.4.1;python_version==''3.6''" "coverage;python_version>''3.6''" && python setup.py pytest'
        displayName: 'Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "not (notebook or automl or dml or serial or cate_api)" -n 2'
          COVERAGE_PROCESS_START: 'setup.cfg'
      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Python $(python.version), image $(imageName)'
        condition: succeededOrFailed()

      - task: PublishCodeCoverageResults@1
        displayName: 'Publish Code Coverage Results'
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'

- template: azure-pipelines-steps.yml
  parameters:
    package: '-e .[tf,plt]'
    job:
      job: Tests_dml
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
      displayName: 'Run tests (DML)'
      steps:
      - script: 'pip install pytest pytest-runner "coverage<6.4.1;python_version==''3.6''" "coverage;python_version>''3.6''" && python setup.py pytest'
        displayName: 'Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "dml"'
          COVERAGE_PROCESS_START: 'setup.cfg'
      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Python $(python.version), image $(imageName)'
        condition: succeededOrFailed()

      - task: PublishCodeCoverageResults@1
        displayName: 'Publish Code Coverage Results'
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'

- template: azure-pipelines-steps.yml
  parameters:
    # exclude macOS since these tests frequently break there
    images: ['ubuntu-18.04', 'windows-2019']
    package: '-e .[tf,plt]'
    job:
      job: Tests_serial
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
      displayName: 'Run tests (Serial)'
      steps:
      - script: 'pip install pytest pytest-runner "coverage<6.4.1;python_version==''3.6''" "coverage;python_version>''3.6''" && python setup.py pytest'
        displayName: 'Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "serial" -n 1'
          COVERAGE_PROCESS_START: 'setup.cfg'
      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Python $(python.version), image $(imageName)'
        condition: succeededOrFailed()

      - task: PublishCodeCoverageResults@1
        displayName: 'Publish Code Coverage Results'
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'

- template: azure-pipelines-steps.yml
  parameters:
    package: '-e .[tf,plt]'
    job:
      job: Tests_CATE_API
      dependsOn: 'EvalChanges'
      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
      displayName: 'Run tests (Other)'
      steps:
      - script: 'pip install pytest pytest-runner "coverage<6.4.1;python_version==''3.6''" "coverage;python_version>''3.6''"'
        displayName: 'Install pytest'
      - script: 'python setup.py pytest'
        displayName: 'CATE Unit tests'
        env:
          PYTEST_ADDOPTS: '-m "cate_api" -n auto'
          COVERAGE_PROCESS_START: 'setup.cfg'
      - task: PublishTestResults@2
        displayName: 'Publish Test Results **/test-results.xml'
        inputs:
          testResultsFiles: '**/test-results.xml'
          testRunTitle: 'Python $(python.version), image $(imageName)'
        condition: succeededOrFailed()

      - task: PublishCodeCoverageResults@1
        displayName: 'Publish Code Coverage Results'
        inputs:
          codeCoverageTool: Cobertura
          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'