.pre-commit-config.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
---
default_stages: [commit, push]
default_language_version:
  # force all unspecified python hooks to run python3
  python: python3
minimum_pre_commit_version: "1.20.0"
repos:
  - repo: https://github.com/Lucas-C/pre-commit-hooks
    rev: v1.1.9
    hooks:
      - id: forbid-tabs
        exclude: ^docs/Makefile$
      - id: insert-license
        name: Add license for all SQL files
        files: \.sql$
        exclude: ^\.github/.*$
        args:
          - --comment-style
          - "/*||*/"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all other files
        exclude: ^\.github/.*$
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
        files: >
          \.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|^Dockerfile.*$
      - id: insert-license
        name: Add license for all rst files
        exclude: ^\.github/.*$
        args:
          - --comment-style
          - "||"
          - --license-filepath
          - license-templates/LICENSE.rst
          - --fuzzy-match-generates-todo
        files: \.rst$
      - id: insert-license
        name: Add license for all JS/CSS files
        files: \.(js|css)$
        exclude: ^\.github/.*$|^airflow/www/static/.*|^airflow/www_rbac/static/.*$
        args:
          - --comment-style
          - "/**| *| */"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all JINJA template files
        files: ^airflow/www/templates/.*\.html$|^docs/templates/.*\.html$|^airflow/contrib/plugins/metastore_browser/templates/.*\.html$  # yamllint disable-line rule:line-length
        exclude: ^\.github/.*$
        args:
          - --comment-style
          - "{#||#}"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all shell files
        exclude: ^\.github/.*$
        files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all python files
        exclude: ^\.github/.*$
        types: [python]
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all XML files
        exclude: ^\.github/.*$
        files: .*\.xml$
        args:
          - --comment-style
          - "<!--||-->"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all yaml files
        exclude: ^\.github/.*$
        types: [yaml]
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all md files
        files: \.md$
        exclude: ^\.github/.*$
        args:
          - --comment-style
          - "<!--|| -->"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
  - repo: https://github.com/thlorenz/doctoc.git
    rev: v1.4.0
    hooks:
      - id: doctoc
        name: Add TOC for md files
        files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.md$|^dev/README.md$
        args:
          - "--maxlevel"
          - "2"
  - repo: meta
    hooks:
      - id: check-hooks-apply
  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v3.1.0
    hooks:
      - id: check-merge-conflict
      - id: debug-statements
      - id: detect-private-key
      - id: end-of-file-fixer
      - id: mixed-line-ending
      - id: check-executables-have-shebangs
      - id: check-xml
      - id: trailing-whitespace
  - repo: https://github.com/pre-commit/pygrep-hooks
    rev: v1.5.1
    hooks:
      - id: rst-backticks
      - id: python-no-log-warn
  - repo: https://github.com/adrienverge/yamllint
    rev: v1.24.2
    hooks:
      - id: yamllint
        name: Check yaml files with yamllint
        entry: yamllint -c yamllint-config.yml
        types: [yaml]
        exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/templates/.*\.yaml$
    ##
    ## Dear committer.
    ##
    ## If you ever come here to add the missing isort step here - hear a little warning.
    ##
    ## Initially isort will cause surprising duplicates of urlparse and other urllib related methods.
    ## The urllib imports seem broken for python 2 but they are actually fine due to future
    ## backport aliases installed elsewhere in the code (implicitly) - in 6 places.
    ##
    ## When you decide how to fix it (likely talking to other people in community) and you push
    ## build to CI you will find terrible truth that in Airflow 1.10 modules are so much
    ## cross-dependent, that imports in a number of places have to be done in specific order and
    ## if this is not followed properly, circular imports kick-in and you are doomed.
    ##
    ## Running isort breaks the import House of Cards and there is no easy way to fix it short of
    ## splitting a number of files and probably breaking compatibility.
    ##
    ## Luckily this has been fixed in Airflow 2.0 by proper untangling of the cross-dependencies and
    ## 1.10.* branch is really in maintenance mode, so do not really waste your time here.
    ##
    ## Unless you really want of course. But then either delete this comment or increase the counter
    ## below after you give up.
    ##
    ## Total hours wasted here = 3
    ##
  - repo: local
    hooks:
      - id: shellcheck
        name: Check Shell scripts syntax correctness
        language: docker_image
        entry: koalaman/shellcheck:stable -x -a
        files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$|\.bats$
      - id: lint-dockerfile
        name: Lint dockerfile
        language: system
        entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
        files: ^Dockerfile.*$
        pass_filenames: true
      - id: setup-order
        name: Checks for an order of dependencies in setup.py
        language: python
        files: ^setup.py$
        pass_filenames: false
        require_serial: true
        entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
      - id: update-breeze-file
        name: Update output of breeze command in BREEZE.rst
        entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
        language: system
        files: ^BREEZE.rst$|^breeze$|^breeze-complete$
        pass_filenames: false
        require_serial: true
      - id: update-local-yml-file
        name: Update mounts in the local yml file
        entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
        language: system
        files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
        pass_filenames: false
        require_serial: true
      - id: update-extras
        name: Update extras in documentation
        entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
        language: python
        files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
        pass_filenames: false
        require_serial: true
      - id: pydevd
        language: pygrep
        name: Check for pydevd debug statements accidentally left
        entry: "pydevd.*settrace\\("
        pass_filenames: true
        files: \.py$
      - id: language-matters
        language: pygrep
        name: Check for language that we do not accept as community
        description: Please use "deny_list" or "allow_list"  instead.
        entry: "(?i)(black|white)[_-]?list"
        pass_filenames: true
        exclude: >
          (?x)
          ^airflow/contrib/hooks/cassandra_hook.py$|
          ^airflow/operators/hive_stats_operator.py$|
          ^tests/contrib/hooks/test_cassandra_hook.py|
          ^CHANGELOG.txt
      - id: dont-use-safe-filter
        language: pygrep
        name: Don't use safe in templates
        description: the Safe filter is error-prone, use Markup() in code instead
        entry: "\\|\\s*safe"
        files: \.html$
        pass_filenames: true
      - id: python2-fastcheck
        language: pygrep
        name: Find common Python 3 vs. 2.7 compatibility problems
        entry: >
          (?x)
            (?!.*\#\ noqa)(?!.*//\ noqa)(           # Exclude lines with '# noqa' or '// noqa' comment
                .super\(\).*|                       # Matches super() call from Python 3
                ^\s*def\s*\S*\([^:#)]*:.*|          # Matches function param with Python3 type
                ^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.*  # Matches -> return value syntax from Python3
            )$
        files: \.py$
        exclude: ^dev|^scripts
        pass_filenames: true
      - id: python2-compile
        name: Compile code using python2
        language: system
        entry: python2.7 -m py_compile
        files: \.py$
        exclude: ^dev|^scripts
        pass_filenames: true
        require_serial: true
      - id: pydevd
        language: pygrep
        name: Check for pydevd debug statements accidentally left
        entry: "pydevd.*settrace\\("
        files: \.py$
        pass_filenames: true
      - id: check-integrations
        name: Check if integration list is aligned
        entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
        language: system
        pass_filenames: false
        files: ^common/_common_values.sh$|^breeze-complete$
      - id: incorrect-use-of-LoggingMixin
        language: pygrep
        name: Make sure LoggingMixin is not used alone
        entry: "LoggingMixin\\(\\)"
        files: \.py$
        pass_filenames: true
      - id: build
        name: Check if image build is needed
        entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
        language: system
        always_run: true
        pass_filenames: false
      - id: check-apache-license
        name: Check if licenses are OK for Apache
        entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
        language: system
        files: ^.*LICENSE.*$|^.*LICENCE.*$
        pass_filenames: false
        require_serial: true
      - id: airflow-config-yaml
        name: Checks for consistency between config.yml and default_config.cfg
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
        files: "^airflow/config_templates/config.yml$|^airflow/config_templates/default_airflow.cfg$"
        pass_filenames: false
        require_serial: true
        additional_dependencies: ['pyyaml']
      - id: mypy
        name: Run mypy
        language: system
        entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
        files: \.py$
        exclude: ^dev
        require_serial: true
      - id: flake8
        name: Run flake8
        language: system
        entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
        files: \.py$
        exclude: ^dev
        pass_filenames: true
      - id: bat-tests
        name: Run BATS bash tests for changed bash files
        language: system
        entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
        files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
        pass_filenames: false