.pre-commit-config.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
---
default_stages: [commit, push]
default_language_version:
  python: python3
  node: 18.6.0
minimum_pre_commit_version: '2.0.0'
repos:
  - repo: meta
    hooks:
      - id: identity
        name: Print input to the static check hooks for troubleshooting
      - id: check-hooks-apply
        name: Check if all hooks apply to the repository
  - repo: https://github.com/thlorenz/doctoc.git
    rev: v2.2.0
    hooks:
      - id: doctoc
        name: Add TOC for Markdown and RST files
        files:
          ^CONTRIBUTING\.md$|^README\.md$|^UPDATING.*\.md$|^chart/UPDATING.*\.md$|^dev/.*\.md$|^dev/.*\.rst$
        exclude: ^.*/.*_vendor/
        args:
          - "--maxlevel"
          - "2"
  - repo: https://github.com/Lucas-C/pre-commit-hooks
    rev: v1.3.1
    hooks:
      - id: insert-license
        name: Add license for all SQL files
        files: \.sql$
        exclude: |
          (?x)
          ^\.github/|
          ^.*/.*_vendor/
        args:
          - --comment-style
          - "/*||*/"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all RST files
        exclude: ^\.github/.*$|^.*/.*_vendor/|newsfragments/.*\.rst$
        args:
          - --comment-style
          - "||"
          - --license-filepath
          - license-templates/LICENSE.rst
          - --fuzzy-match-generates-todo
        files: \.rst$
      - id: insert-license
        name: Add license for all CSS/JS/JSX/PUML/TS/TSX files
        files: \.(css|jsx?|puml|tsx?)$
        exclude: ^\.github/.*$|^.*/.*_vendor/
        args:
          - --comment-style
          - "/*!| *| */"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all JINJA template files
        files: ^airflow/www/templates/.*\.html$
        exclude: ^\.github/.*$|^.*/.*_vendor/
        args:
          - --comment-style
          - "{#||#}"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all Shell files
        exclude: ^\.github/.*$|^.*/.*_vendor/|^dev/breeze/autocomplete/.*$
        files: \.bash$|\.sh$
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all Python files
        exclude: ^\.github/.*$|^.*/.*_vendor/
        files: \.py$|\.pyi$
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all XML files
        exclude: ^\.github/.*$|^.*/.*_vendor/
        files: \.xml$
        args:
          - --comment-style
          - "<!--||-->"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all Helm template files
        files: ^chart/templates/.*
        args:
          - --comment-style
          - "{{/*||*/}}"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all YAML files except Helm templates
        exclude: ^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*
        types: [yaml]
        files: \.ya?ml$
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all Markdown files
        files: \.md$
        exclude: ^\.github/.*$|PROVIDER_CHANGES.*\.md$|^.*/.*_vendor/
        args:
          - --comment-style
          - "<!--|| -->"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
      - id: insert-license
        name: Add license for all other files
        exclude: ^\.github/.*$|^.*/.*_vendor/
        args:
          - --comment-style
          - "|#|"
          - --license-filepath
          - license-templates/LICENSE.txt
          - --fuzzy-match-generates-todo
        files: >
          \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
  - repo: https://github.com/psf/black
    rev: 22.12.0
    hooks:
      - id: black
        name: Run black (Python formatter)
        args: [--config=./pyproject.toml]
        exclude: ^.*/.*_vendor/|^airflow/contrib/
  - repo: local
    hooks:
      - id: update-common-sql-api-stubs
        name: Check and update common.sql API stubs
        entry: ./scripts/ci/pre_commit/pre_commit_update_common_sql_api_stubs.py
        language: python
        files: ^scripts/ci/pre_commit/pre_commit_update_common_sql_api\.py|^airflow/providers/common/sql/.*\.pyi?$
        additional_dependencies: ['rich>=12.4.4', 'mypy==1.2.0', 'black==22.12.0', 'jinja2']
        pass_filenames: false
        require_serial: true
      - id: update-black-version
        name: Update black versions everywhere
        entry: ./scripts/ci/pre_commit/pre_commit_update_black_version.py
        language: python
        files: ^.pre-commit-config.yaml$
        additional_dependencies: ['pyyaml']
        pass_filenames: false
        require_serial: true
      - id: ruff
        name: ruff
        language: python
        require_serial: true
        pass_filenames: true
        # Since ruff makes use of multiple cores we _purposefully_ don't run this in docker so it can use the
        # host CPU to it's fullest
        entry: ruff --fix --force-exclude
        additional_dependencies: ['ruff==0.0.272']
        files: \.pyi?$
        exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py
  - repo: https://github.com/asottile/blacken-docs
    rev: 1.13.0
    hooks:
      - id: blacken-docs
        name: Run black on Python code blocks in documentation files
        args:
          - --line-length=110
          - --target-version=py37
          - --target-version=py38
          - --target-version=py39
          - --target-version=py310
        alias: black
        additional_dependencies: [black==22.12.0]
  - repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v4.4.0
    hooks:
      - id: check-merge-conflict
        name: Check that merge conflicts are not being committed
      - id: debug-statements
        name: Detect accidentally committed debug statements
      - id: check-builtin-literals
        name: Require literal syntax when initializing builtin types
        exclude: ^.*/.*_vendor/
      - id: detect-private-key
        name: Detect if private key is added to the repository
        exclude: ^docs/apache-airflow-providers-ssh/connections/ssh.rst$
      - id: end-of-file-fixer
        name: Make sure that there is an empty line at the end
        exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^docs/apache-airflow/img/.*\.sha256
      - id: mixed-line-ending
        name: Detect if mixed line ending is used (\r vs. \r\n)
        exclude: ^.*/.*_vendor/
      - id: check-executables-have-shebangs
        name: Check that executables have shebang
        exclude: ^.*/.*_vendor/
      - id: check-xml
        name: Check XML files with xmllint
        exclude: ^.*/.*_vendor/
      - id: trailing-whitespace
        name: Remove trailing whitespace at end of line
        exclude: ^.*/.*_vendor/|^docs/apache-airflow/img/.*\.dot|^images/breeze/output.*$
      - id: fix-encoding-pragma
        name: Remove encoding header from Python files
        exclude: ^.*/.*_vendor/
        args:
          - --remove
      - id: pretty-format-json
        name: Format JSON files
        args:
          - --autofix
          - --no-sort-keys
          - --indent
          - "4"
        files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
        pass_filenames: true
  - repo: https://github.com/pre-commit/pygrep-hooks
    rev: v1.10.0
    hooks:
      - id: rst-backticks
        name: Check if RST files use double backticks for code
        exclude: ^.*/.*_vendor/
      - id: python-no-log-warn
        name: Check if there are no deprecate log warn
        exclude: ^.*/.*_vendor/
  - repo: https://github.com/adrienverge/yamllint
    rev: v1.29.0
    hooks:
      - id: yamllint
        name: Check YAML files with yamllint
        entry: yamllint -c yamllint-config.yml --strict
        types: [yaml]
        exclude: ^.*airflow\.template\.yaml$|^.*init_git_sync\.template\.yaml$|^.*/.*_vendor/|^chart/(?:templates|files)/.*\.yaml$|openapi/.*\.yaml$|^\.pre-commit-config\.yaml$
  - repo: https://github.com/ikamensh/flynt
    rev: '0.77'
    hooks:
      - id: flynt
        name: Run flynt string format converter for Python
        exclude: |
          (?x)
          ^.*/.*_vendor/
        args:
         # If flynt detects too long text it ignores it. So we set a very large limit to make it easy
         # to split the text by hand. Too long lines are detected by flake8 (below),
         # so the user is informed to take action.
         - --line-length
         - '99999'
  - repo: https://github.com/codespell-project/codespell
    rev: v2.2.2
    hooks:
      - id: codespell
        name: Run codespell to check for common misspellings in files
        entry: bash -c 'echo "If you think that this failure is an error, consider adding the word(s)
          to the codespell dictionary at docs/spelling_wordlist.txt.
          The word(s) should be in lowercase." && exec codespell "$@"' --
        language: python
        types: [text]
        exclude: ^.*/.*_vendor/|^airflow/www/static/css/material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$
        args:
          - --ignore-words=docs/spelling_wordlist.txt
          - --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
          - --exclude-file=.codespellignorelines
  - repo: local
    # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because
    # we try to minimise the number of passes that must happen in order to apply some of the changes
    # done by pre-commits. Some of the pre-commits not only check for errors but also fix them. This means
    # that output from an earlier pre-commit becomes input to another pre-commit. Splitting the local
    # scripts of our and adding some other non-local pre-commit in-between allows us to handle such
    # changes quickly - especially when we want the early modifications from the first local group
    # to be applied before the non-local pre-commits are run
    hooks:
      - id: replace-bad-characters
        name: Replace bad characters
        entry: ./scripts/ci/pre_commit/pre_commit_replace_bad_characters.py
        language: python
        types: [file, text]
        exclude: ^.*/.*_vendor/|^clients/gen/go\.sh$|^\.gitmodules$
        additional_dependencies: ['rich>=12.4.4']
      - id: lint-openapi
        name: Lint OpenAPI using spectral
        language: docker_image
        entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml
        files: ^airflow/api_connexion/openapi/
      - id: lint-openapi
        name: Lint OpenAPI using openapi-spec-validator
        entry: openapi-spec-validator --schema 3.0.0
        language: python
        additional_dependencies: ['openapi-spec-validator']
        files: ^airflow/api_connexion/openapi/
      - id: lint-dockerfile
        name: Lint Dockerfile
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_lint_dockerfile.py
        files: Dockerfile.*$
        pass_filenames: true
        require_serial: true
      - id: check-setup-order
        name: Check order of dependencies in setup.cfg and setup.py
        language: python
        files: ^setup\.cfg$|^setup\.py$
        pass_filenames: false
        entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
        additional_dependencies: ['rich>=12.4.4']
      - id: check-extra-packages-references
        name: Checks setup extra packages
        description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file
        language: python
        files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$|^airflow/providers/.*/provider\.yaml$
        pass_filenames: false
        entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
        additional_dependencies: ['rich>=12.4.4']
      - id: check-airflow-provider-compatibility
        name: Check compatibility of Providers with Airflow
        entry: ./scripts/ci/pre_commit/pre_commit_check_provider_airflow_compatibility.py
        language: python
        pass_filenames: true
        files: ^airflow/providers/.*\.py$
        additional_dependencies: ['rich>=12.4.4']
      - id: update-local-yml-file
        name: Update mounts in the local yml file
        entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
        language: python
        files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
        pass_filenames: false
        additional_dependencies: ['rich>=12.4.4']
      - id: update-providers-dependencies
        name: Update cross-dependencies for providers packages
        entry: ./scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
        language: python
        files: ^airflow/providers/.*\.py$|^airflow/providers/.*/provider\.yaml$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$
        pass_filenames: false
        additional_dependencies: ['setuptools', 'rich>=12.4.4', 'pyyaml']
      - id: update-extras
        name: Update extras in documentation
        entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
        language: python
        files: ^setup\.py$|^CONTRIBUTING\.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$
        pass_filenames: false
        additional_dependencies: ['rich>=12.4.4']
      - id: check-extras-order
        name: Check order of extras in Dockerfile
        entry: ./scripts/ci/pre_commit/pre_commit_check_order_dockerfile_extras.py
        language: python
        files: ^Dockerfile$
        pass_filenames: false
        additional_dependencies: ['rich>=12.4.4']
      - id: update-supported-versions
        name: Updates supported versions in documentation
        entry: ./scripts/ci/pre_commit/pre_commit_supported_versions.py
        language: python
        files: ^docs/apache-airflow/installation/supported-versions\.rst$|^scripts/ci/pre_commit/pre_commit_supported_versions\.py$|^README\.md$
        pass_filenames: false
        additional_dependencies: ['tabulate']
      - id: check-revision-heads-map
        name: Check that the REVISION_HEADS_MAP is up-to-date
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
        pass_filenames: false
        additional_dependencies: ['packaging']
      - id: update-version
        name: Update version to the latest version in the documentation
        entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
        language: python
        files: ^docs
        pass_filenames: false
      - id: check-pydevd-left-in-code
        language: pygrep
        name: Check for pydevd debug statements accidentally left
        entry: "pydevd.*settrace\\("
        pass_filenames: true
        files: \.py$
      - id: check-links-to-example-dags-do-not-use-hardcoded-versions
        name: Verify example dags do not use hard-coded version numbers
        description: The links to example dags should use |version| as version specification
        language: pygrep
        entry: >
          (?i)
          .*https://github.*[0-9]/tests/system/providers|
          .*https://github.*/main/tests/system/providers|
          .*https://github.*/master/tests/system/providers|
          .*https://github.*/main/airflow/providers/.*/example_dags/|
          .*https://github.*/master/airflow/providers/.*/example_dags/
        pass_filenames: true
        files: ^docs/apache-airflow-providers-.*\.rst
      - id: check-safe-filter-usage-in-html
        language: pygrep
        name: Don't use safe in templates
        description: the Safe filter is error-prone, use Markup() in code instead
        entry: "\\|\\s*safe"
        files: \.html$
        pass_filenames: true
      - id: check-no-providers-in-core-examples
        language: pygrep
        name: No providers imports in core example DAGs
        description: The core example DAGs have no dependencies other than core Airflow
        entry: "^\\s*from airflow\\.providers.*"
        pass_filenames: true
        files: ^airflow/example_dags/.*\.py$
      - id: check-no-airflow-deprecation-in-providers
        language: pygrep
        name: Do not use DeprecationWarning in providers
        description: Use AirflowProviderDeprecationWarning in providers
        entry: "^\\s*DeprecationWarning*"
        pass_filenames: true
        files: ^airflow/providers/.*\.py$
      - id: check-no-relative-imports
        language: pygrep
        name: No relative imports
        description: Airflow style is to use absolute imports only (except docs building)
        entry: "^\\s*from\\s+\\."
        pass_filenames: true
        files: \.py$
        exclude: ^.*/.*_vendor/|^docs/|^tests/
      - id: check-urlparse-usage-in-code
        language: pygrep
        name: Don't use urlparse in code
        description: urlparse is not recommended, use urlsplit() in code instead
        entry: "^\\s*from urllib\\.parse import ((\\|, )(urlparse\\|urlunparse))+$"
        pass_filenames: true
        files: \.py$
      - id: check-only-new-session-with-provide-session
        name: Check NEW_SESSION is only used with @provide_session
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_new_session_in_provide_session.py
        pass_filenames: true
        files: ^airflow/.+\.py$
      - id: check-for-inclusive-language
        language: pygrep
        name: Check for language that we do not accept as community
        description: Please use more appropriate words for community documentation.
        entry: >
          (?ix)
          (black|white)[_-]?list|
          \bshe\b|
          \bhe\b|
          \bher\b|
          \bhis\b|
          \bmaster\b|
          \bslave\b|
          \bsanity\b|
          \bdummy\b
        pass_filenames: true
        exclude: >
          (?x)
          ^airflow/api_connexion/openapi/v1.yaml$|
          ^airflow/cli/commands/webserver_command.py$|
          ^airflow/config_templates/|
          ^airflow/models/baseoperator.py$|
          ^airflow/operators/__init__.py$|
          ^airflow/providers/amazon/aws/hooks/emr.py$|
          ^airflow/providers/amazon/aws/operators/emr.py$|
          ^airflow/providers/apache/cassandra/hooks/cassandra.py$|
          ^airflow/providers/apache/hive/operators/hive_stats.py$|
          ^airflow/providers/apache/hive/transfers/vertica_to_hive.py$|
          ^airflow/providers/apache/spark/hooks/|
          ^airflow/providers/apache/spark/operators/|
          ^airflow/providers/exasol/hooks/exasol.py$|
          ^airflow/providers/google/cloud/hooks/bigquery.py$|
          ^airflow/providers/google/cloud/operators/cloud_build.py$|
          ^airflow/providers/google/cloud/operators/dataproc.py$|
          ^airflow/providers/google/cloud/operators/mlengine.py$|
          ^airflow/providers/microsoft/azure/hooks/cosmos.py$|
          ^airflow/providers/microsoft/winrm/hooks/winrm.py$|
          ^airflow/www/fab_security/manager.py$|
          ^docs/.*commits.rst$|
          ^docs/apache-airflow/administration-and-deployment/security/webserver.rst$|
          ^docs/apache-airflow-providers-apache-cassandra/connections/cassandra.rst$|
          ^airflow/providers/microsoft/winrm/operators/winrm.py$|
          ^airflow/providers/opsgenie/hooks/opsgenie.py$|
          ^airflow/providers/redis/provider.yaml$|
          ^airflow/serialization/serialized_objects.py$|
          ^airflow/utils/db.py$|
          ^airflow/utils/trigger_rule.py$|
          ^airflow/www/static/css/bootstrap-theme.css$|
          ^airflow/www/static/js/types/api-generated.ts$|
          ^airflow/www/templates/appbuilder/flash.html$|
          ^.*/.*_vendor/|
          ^dev/|
          ^docs/README.rst$|
          ^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store.rst$|
          ^docs/apache-airflow-providers-apache-hdfs/connections.rst$|
          ^docs/apache-airflow-providers-apache-kafka/connections/kafka.rst$|
          ^docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst$|
          ^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$|
          ^docs/conf.py$|
          ^docs/exts/removemarktransform.py$|
          ^scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py$|
          ^tests/|
          ^.pre-commit-config\.yaml$|
          ^.*CHANGELOG\.(rst|txt)$|
          ^.*RELEASE_NOTES\.rst$|
          ^CONTRIBUTORS_QUICK_START.rst$|
          ^.*\.(png|gif|jp[e]?g|tgz|lock)$|
          git
      - id: check-base-operator-partial-arguments
        name: Check BaseOperator and partial() arguments
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
        pass_filenames: false
        files: ^airflow/models/(?:base|mapped)operator\.py$
      - id: check-init-decorator-arguments
        name: Check model __init__ and decorator arguments are in sync
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_sync_init_decorator.py
        pass_filenames: false
        files: ^airflow/models/dag\.py$|^airflow/(?:decorators|utils)/task_group\.py$
      - id: check-base-operator-usage
        language: pygrep
        name: Check BaseOperator[Link] core imports
        description: Make sure BaseOperator[Link] is imported from airflow.models.baseoperator in core
        entry: "from airflow\\.models import.* BaseOperator"
        files: \.py$
        pass_filenames: true
        exclude: >
          (?x)
          ^airflow/decorators/.*$|
          ^airflow/hooks/.*$|
          ^airflow/operators/.*$|
          ^airflow/providers/.*$|
          ^airflow/sensors/.*$|
          ^dev/provider_packages/.*$
      - id: check-base-operator-usage
        language: pygrep
        name: Check BaseOperator[Link] other imports
        description: Make sure BaseOperator[Link] is imported from airflow.models outside of core
        entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
        pass_filenames: true
        files: >
          (?x)
          ^airflow/providers/.*\.py$
        exclude: ^.*/.*_vendor/
      - id: check-decorated-operator-implements-custom-name
        name: Check @task decorator implements custom_operator_name
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_decorator_operator_implements_custom_name.py
        pass_filenames: true
        files: ^airflow/.*\.py$
      - id: check-core-deprecation-classes
        language: pygrep
        name: Verify usage of Airflow deprecation classes in core
        entry: category=DeprecationWarning|category=PendingDeprecationWarning
        files: \.py$
        exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$
        pass_filenames: true
      - id: check-provide-create-sessions-imports
        language: pygrep
        name: Check provide_session and create_session imports
        description: provide_session and create_session should be imported from airflow.utils.session
          to avoid import cycles.
        entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)"
        files: \.py$
        exclude: ^.*/.*_vendor/
        pass_filenames: true
      - id: check-incorrect-use-of-LoggingMixin
        language: pygrep
        name: Make sure LoggingMixin is not used alone
        entry: "LoggingMixin\\(\\)"
        files: \.py$
        exclude: ^.*/.*_vendor/
        pass_filenames: true
      - id: check-daysago-import-from-utils
        language: pygrep
        name: Make sure days_ago is imported from airflow.utils.dates
        entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago"
        files: \.py$
        exclude: ^.*/.*_vendor/
        pass_filenames: true
      - id: check-start-date-not-used-in-defaults
        language: pygrep
        name: start_date not to be defined in default_args in example_dags
        entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
        files: \.*example_dags.*\.py$
        exclude: ^.*/.*_vendor/
        pass_filenames: true
      - id: check-apache-license-rat
        name: Check if licenses are OK for Apache
        entry: ./scripts/ci/pre_commit/pre_commit_check_license.py
        language: python
        files: ^.*LICENSE.*$|^.*LICENCE.*$
        pass_filenames: false
      - id: check-aiobotocore-optional
        name: Check if aiobotocore is an optional dependency only
        entry: ./scripts/ci/pre_commit/pre_commit_check_aiobotocore_optional.py
        language: python
        files: ^airflow/providers/.*/provider\.yaml$
        pass_filenames: true
        additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
        require_serial: true
      - id: check-airflow-config-yaml-consistent
        name: Check consistency between config.yml and default_config.cfg
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
        files: config\.yml$|default_airflow\.cfg$|default\.cfg$
        pass_filenames: false
        require_serial: true
        additional_dependencies: ['pyyaml', 'packaging']
      - id: check-boring-cyborg-configuration
        name: Checks for Boring Cyborg configuration consistency
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_boring_cyborg.py
        pass_filenames: false
        require_serial: true
        additional_dependencies: ['pyyaml', 'termcolor==1.1.0', 'wcmatch==8.2']
      - id: update-in-the-wild-to-be-sorted
        name: Sort INTHEWILD.md alphabetically
        entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.py
        language: python
        files: ^\.pre-commit-config\.yaml$|^INTHEWILD\.md$
        pass_filenames: false
        require_serial: true
      - id: update-installed-providers-to-be-sorted
        name: Sort alphabetically and uniquify installed_providers.txt
        entry: ./scripts/ci/pre_commit/pre_commit_sort_installed_providers.py
        language: python
        files: ^\.pre-commit-config\.yaml$|^scripts/ci/installed_providers\.txt$
        pass_filenames: false
        require_serial: true
      - id: update-spelling-wordlist-to-be-sorted
        name: Sort alphabetically and uniquify spelling_wordlist.txt
        entry: ./scripts/ci/pre_commit/pre_commit_sort_spelling_wordlist.py
        language: python
        files: ^\.pre-commit-config\.yaml$|^docs/spelling_wordlist\.txt$
        require_serial: true
        pass_filenames: false
      - id: lint-helm-chart
        name: Lint Helm Chart
        entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.py
        language: python
        pass_filenames: false
        files: ^chart
        require_serial: true
        additional_dependencies: ['rich>=12.4.4','requests']
      - id: shellcheck
        name: Check Shell scripts syntax correctness
        language: docker_image
        entry: koalaman/shellcheck:v0.8.0 -x -a
        files: \.(bash|sh)$|^hooks/build$|^hooks/push$
        exclude: ^dev/breeze/autocomplete/.*$
      - id: lint-css
        name: stylelint
        entry: "stylelint"
        language: node
        files: ^airflow/www/.*\.(css|sass|scss)$
        # Keep dependency versions in sync w/ airflow/www/package.json
        additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0', 'stylelint-config-prettier@9.0.5']
      - id: compile-www-assets
        name: Compile www assets
        language: node
        stages: ['manual']
        'types_or': [javascript, ts, tsx]
        files: ^airflow/www/
        entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets.py
        pass_filenames: false
        additional_dependencies: ['yarn@1.22.19']
      - id: compile-www-assets-dev
        name: Compile www assets in dev mode
        language: node
        stages: ['manual']
        'types_or': [javascript, ts, tsx]
        files: ^airflow/www/
        entry: ./scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
        pass_filenames: false
        additional_dependencies: ['yarn@1.22.19']
      - id: check-providers-init-file-missing
        name: Provider init file is missing
        pass_filenames: false
        always_run: true
        entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.py
        language: python
      - id: check-providers-subpackages-init-file-exist
        name: Provider subpackage init files are there
        pass_filenames: false
        always_run: true
        entry: ./scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py
        language: python
        require_serial: true
      - id: check-pre-commit-information-consistent
        name: Validate hook IDs & names and sync with docs
        entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py
        args:
          - --max-length=60
        language: python
        files: ^\.pre-commit-config\.yaml$|^scripts/ci/pre_commit/pre_commit_check_pre_commit_hook_names\.py$
        additional_dependencies: ['pyyaml', 'jinja2', 'black==22.12.0', 'tabulate', 'rich>=12.4.4']
        require_serial: true
        pass_filenames: false
      - id: update-breeze-readme-config-hash
        name: Update Breeze README.md with config files hash
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py
        files: ^dev/breeze/pyproject\.toml$|^dev/breeze/setup\.cfg$|^dev/breeze/setup\.py$|^dev/breeze/README\.md$
        pass_filenames: false
        require_serial: true
      - id: check-breeze-top-dependencies-limited
        name: Breeze should have small number of top-level dependencies
        language: python
        entry: ./scripts/tools/check_if_limited_dependencies.py
        files: ^dev/breeze/.*$
        pass_filenames: false
        require_serial: true
        additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml']
      - id: check-system-tests-present
        name: Check if system tests have required segments of code
        entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
        language: python
        files: ^tests/system/.*/example_[^/]*\.py$
        exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$
        pass_filenames: true
        additional_dependencies: ['rich>=12.4.4']
      - id: lint-markdown
        name: Run markdownlint
        description: Checks the style of Markdown files.
        entry: markdownlint
        language: node
        types: [markdown]
        files: \.(md|mdown|markdown)$
        additional_dependencies: ['markdownlint-cli']
      - id: lint-json-schema
        name: Lint JSON Schema files with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --spec-url
          - https://json-schema.org/draft-07/schema
        language: python
        pass_filenames: true
        files: .*\.schema\.json$
        exclude: ^.*/.*_vendor/
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: lint-json-schema
        name: Lint NodePort Service with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --spec-url
          - https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.20.2-standalone/service-v1.json
        language: python
        pass_filenames: true
        files: ^scripts/ci/kubernetes/nodeport\.yaml$
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: lint-json-schema
        name: Lint Docker compose files with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --spec-url
          - https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json
        language: python
        pass_filenames: true
        files: ^scripts/ci/docker-compose/.+\.ya?ml$|docker-compose\.ya?ml$
        exclude: >
          (?x)
          ^scripts/ci/docker-compose/grafana/.|
          ^scripts/ci/docker-compose/.+-config\.ya?ml
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: lint-json-schema
        name: Lint chart/values.schema.json file with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --spec-file
          - chart/values_schema.schema.json
          - chart/values.schema.json
        language: python
        pass_filenames: false
        files: ^chart/values\.schema\.json$|^chart/values_schema\.schema\.json$
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: update-vendored-in-k8s-json-schema
        name: Vendor k8s definitions into values.schema.json
        entry: ./scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py
        language: python
        files: ^chart/values\.schema\.json$
        additional_dependencies: ['requests==2.25.0']
      - id: lint-json-schema
        name: Lint chart/values.yaml file with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --enforce-defaults
          - --spec-file
          - chart/values.schema.json
          - chart/values.yaml
        language: python
        pass_filenames: false
        files: ^chart/values\.yaml$|^chart/values\.schema\.json$
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: lint-json-schema
        name: Lint config_templates/config.yml file with JSON Schema
        entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
        args:
          - --spec-file
          - airflow/config_templates/config.yml.schema.json
        language: python
        pass_filenames: true
        files: ^airflow/config_templates/config\.yml$
        require_serial: true
        additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'PyYAML==5.3.1', 'requests==2.25.0']
      - id: check-persist-credentials-disabled-in-github-workflows
        name: Check that workflow files have persist-credentials disabled
        entry: ./scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
        language: python
        pass_filenames: true
        files: ^\.github/workflows/.*\.yml$
        exclude: ^\.github/workflows/sync_authors\.yml$
        additional_dependencies: ['PyYAML', 'rich>=12.4.4']
      - id: check-docstring-param-types
        name: Check that docstrings do not specify param types
        entry: ./scripts/ci/pre_commit/pre_commit_docstring_param_type.py
        language: python
        pass_filenames: true
        files: \.py$
        exclude: ^.*/.*_vendor/
        additional_dependencies: ['rich>=12.4.4']
      - id: lint-chart-schema
        name: Lint chart/values.schema.json file
        entry: ./scripts/ci/pre_commit/pre_commit_chart_schema.py
        language: python
        pass_filenames: false
        files: ^chart/values\.schema\.json$
        require_serial: true
      - id: update-inlined-dockerfile-scripts
        name: Inline Dockerfile and Dockerfile.ci scripts
        entry: ./scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
        language: python
        pass_filenames: false
        files: ^Dockerfile$|^Dockerfile\.ci$|^scripts/docker/.*$
        require_serial: true
      - id: check-changelog-has-no-duplicates
        name: Check changelogs for duplicate entries
        language: python
        files: CHANGELOG\.(rst|txt)$
        entry: ./scripts/ci/pre_commit/pre_commit_changelog_duplicates.py
        pass_filenames: true
      - id: check-newsfragments-are-valid
        name: Check newsfragments are valid
        language: python
        files: newsfragments/.*\.rst
        entry: ./scripts/ci/pre_commit/pre_commit_newsfragments.py
        pass_filenames: true
        # We sometimes won't have newsfragments in the repo, so always run it so `check-hooks-apply` passes
        # This is fast, so not too much downside
        always_run: true
      - id: update-breeze-cmd-output
        name: Update output of breeze commands in BREEZE.rst
        entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
        language: python
        files: >
          (?x)
          ^BREEZE\.rst$|^dev/breeze/.*$|
          ^\.pre-commit-config\.yaml$|
          ^scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py$|
          ^generated/provider_dependencies.json$
        require_serial: true
        pass_filenames: false
        additional_dependencies: ['rich>=12.4.4', 'rich-click>=1.5', 'inputimeout', 'pyyaml', 'packaging']
      - id: check-example-dags-urls
        name: Check that example dags url include provider versions
        entry: ./scripts/ci/pre_commit/pre_commit_update_example_dags_paths.py
        language: python
        pass_filenames: true
        files: ^docs/.*example-dags\.rst$|^docs/.*index\.rst$
        additional_dependencies: ['rich>=12.4.4', 'pyyaml']
        always_run: true
      - id: check-system-tests-tocs
        name: Check that system tests is properly added
        entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests_hidden_in_index.py
        language: python
        pass_filenames: true
        files: ^docs/apache-airflow-providers-[^/]*/index\.rst$
        additional_dependencies: ['rich>=12.4.4', 'pyyaml']
      - id: check-lazy-logging
        name: Check that all logging methods are lazy
        entry: ./scripts/ci/pre_commit/pre_commit_check_lazy_logging.py
        language: python
        pass_filenames: true
        files: \.py$
        exclude: ^.*/.*_vendor/
        additional_dependencies: ['rich>=12.4.4', 'astor']
      - id: create-missing-init-py-files-tests
        name: Create missing init.py files in tests
        entry: ./scripts/ci/pre_commit/pre_commit_check_init_in_tests.py
        language: python
        additional_dependencies: ['rich>=12.4.4']
        pass_filenames: false
        files: ^tests/.*\.py$
      - id: ts-compile-format-lint-www
        name: TS types generation / ESLint / Prettier against UI files
        language: node
        'types_or': [javascript, ts, tsx, yaml, css, json]
        files: ^airflow/www/static/js/|^airflow/api_connexion/openapi/v1\.yaml$
        entry: ./scripts/ci/pre_commit/pre_commit_www_lint.py
        additional_dependencies: ['yarn@1.22.19']
        pass_filenames: false
      - id: check-tests-unittest-testcase
        name: Check that unit tests do not inherit from unittest.TestCase
        entry: ./scripts/ci/pre_commit/pre_commit_unittest_testcase.py
        language: python
        pass_filenames: true
        files: ^tests/.*\.py$
        ## ADD MOST PRE-COMMITS ABOVE THAT LINE
        # The below pre-commits are those requiring CI image to be built
      - id: mypy-dev
        name: Run mypy for dev
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
        files: ^dev/.*\.py$
        require_serial: true
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
      - id: mypy-core
        name: Run mypy for core
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
        files: \.py$
        exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
        require_serial: true
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
      - id: mypy-providers
        name: Run mypy for providers
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
        files: ^airflow/providers/.*\.py$|^tests/providers/\*\.py$|^tests/system/providers/\*\.py$
        exclude: ^.*/.*_vendor/
        require_serial: true
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
      - id: mypy-docs
        name: Run mypy for /docs/ folder
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
        files: ^docs/.*\.py$
        exclude: ^docs/rtd-deprecation
        require_serial: true
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml']
      - id: check-provider-yaml-valid
        name: Validate provider.yaml files
        entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
        language: python
        files: ^airflow/providers/.*/provider\.yaml$
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'markdown-it-py']
        require_serial: true
      - id: update-migration-references
        name: Update migration ref doc
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_migration_reference.py
        pass_filenames: false
        files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
        additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'markdown-it-py']
      - id: update-er-diagram
        name: Update ER diagram
        language: python
        entry: ./scripts/ci/pre_commit/pre_commit_update_er_diagram.py
        pass_filenames: false
        files: ^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
        additional_dependencies: ['rich>=12.4.4']
        ## ONLY ADD PRE-COMMITS HERE THAT REQUIRE CI IMAGE