.gitlab-ci.yml

stages:
  - bootstrap
  - build
  - post-deploy
  - cleanup

variables:
  PREVIEW_CONFIG: "config/preview.yaml"
  LIVE_CONFIG: "config/live.yaml"
  HUGO_CACHEDIR: "${CI_PROJECT_DIR}/hugo_cache/"
  ARTIFACT_RESOURCE: "public"
  LIVE_DOMAIN: "https://docs.datadoghq.com/"
  PREVIEW_DOMAIN: "https://docs-staging.datadoghq.com/"
  KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: documentation
  KUBERNETES_CPU_REQUEST: 8
  KUBERNETES_MEMORY_REQUEST: 24Gi
  KUBERNETES_MEMORY_LIMIT: 24Gi
  # datadog-ci
  DATADOG_SITE: 'datadoghq.com'
  DATADOG_SUBDOMAIN: 'dd-corpsite'
  FORCE_COLOR: '1'
  GIT_DEPTH: 50
  DEFAULT_TAGS: "environment:${CI_ENVIRONMENT_NAME},job_name:${CI_JOB_NAME},job_stage:${CI_JOB_STAGE},project_name:${CI_PROJECT_NAME},docker_image:${CI_REGISTRY_IMAGE},pipeline_id:${CI_PIPELINE_ID},service:gitlab"

  # gitlab checkout
  DOCS_REPO_NAME: documentation
  REPO_URL: "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/${DOCS_REPO_NAME}.git"
  # yarn/node cache dirs
  YARN_CACHE_FOLDER: "${CI_PROJECT_DIR}/.yarn/cache/"
  PREVIEW_PATTERN: '/.+?\/[a-zA-Z0-9_-]+/'


# ================== copy scripts =============== #
before_script:
  - "[ -d local/bin ] && find local/bin/ -type f -exec cp {} /usr/local/bin \\;"  # load scripts
  - "[ -d local/etc ] && find local/etc/ -type f -exec cp {} /etc \\;"  # load configs
  - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers so they are available in the container
  - "[ -f /etc/profile ] && source /etc/profile" # for golang on path
  - mkdir -p logs
  # set common env variables for this shell
  - export GITHUB_TOKEN=$(get_secret 'github_token')
  - export DATADOG_API_KEY=$(get_secret 'dd-api-key')
  - export DATADOG_APP_KEY=$(get_secret 'dd-app-key')
  - export BRANCH=${CI_COMMIT_REF_NAME}
  - export SLACK_URL=$(get_secret 'slack_url')
  - export PREVIEWS_SLACK_URL=$(get_secret 'PREVIEWS_SLACK_URL')
  - git config --global url."https://${GITHUB_TOKEN}@github.com/DataDog".insteadOf https://github.com/DataDog
  - type build_dogrc &>/dev/null && build_dogrc || echo "build_dogrc not found." # create .dogrc for metrics, events submission
  - export job_start=$(date +%s) # we need the start of each job recorded for the ci reporting


# set policy for the common case
# hugo cache is used only in the build job so lets pull-push
.hugo_cache: &hugo_cache
  - key:
      files:
        - go.sum
    paths:
      - ${HUGO_CACHEDIR}
    policy: pull-push

# set policy for the common case
# yarn cache is used in a lot but only built during build job so lets default to pull and explicit push in build job
.yarn_cache: &yarn_cache
  - key:
      files:
        - yarn.lock
    paths:
      - package.json
      - yarn.lock
      - ${YARN_CACHE_FOLDER}
    policy: pull

.yarn_cache_pull_push: &yarn_cache_pull_push
  - key:
      files:
        - yarn.lock
    paths:
      - package.json
      - yarn.lock
      - ${YARN_CACHE_FOLDER}
    policy: pull-push

# ==================== rules ==================== #
.preview_rules: &preview_rules
  rules:
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH =~ $PREVIEW_PATTERN
      when: on_success

.preview_manual_rules: &preview_manual_rules
  rules:
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH =~ $PREVIEW_PATTERN
      when: manual

.live_rules: &live_rules
  rules:
    - if: $CI_COMMIT_REF_NAME == "master"
      when: on_success

.live_schedule_rules: &live_schedule_rules
  rules:
    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule"
      when: on_success
# ================== templates ================== #
.base_template: &base_template
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/websites-images:webops-site-build-15641318
  tags:
    - "arch:amd64"
  rules:
    - if: $CI_COMMIT_TAG != null # Variable exists on tag workflows
      when: never
    - if: $CI_COMMIT_BRANCH != null # Variable doesnt exists on merge request pipelines or tag pipelines.
      when: on_success


# ================== preview ================== #
# If the branch has a name of <slack-user>/<feature-name> then ci builds a preview site
build_preview:
  <<: *base_template
  <<: *preview_rules
  stage: build
  cache:
    - *hugo_cache
    - *yarn_cache_pull_push
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    CONFIG: ${PREVIEW_CONFIG}
    MESSAGE: ":gift_heart: Your preview site is available!\nNow running tests..."
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config_preview.yaml"
    LOCAL: "False"
  script:
    - dog --config "$HOME/.dogrc" event post "documentation build ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
    - touch Makefile.config
    - make dependencies
    - make config
    - ./node_modules/.bin/jest --testPathPattern=assets/scripts/tests/
    - yarn run prebuild
    - build_site
    # remove service_checks json as we don't need to s3 push that..
    - rm -rf data/service_checks
    - in-isolation deploy_site > logs/deploy.txt
    # remove static images so we can skip from artifact passed in gitlab
    - remove_static_from_repo
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_REF_NAME}> is ready for preview. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}| checks running>." "#31b834"
    - ci_pass_step "${CI_PROJECT_NAME}-${CI_JOB_NAME}" # run at completion of job, if job fails at any point prior this won't run
  artifacts:
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - .github/
      - .vale.ini
      - .htmltest.yml
      - ./logs
      - ./public/redirects.txt
  interruptible: true

link_checks_preview:
  <<: *base_template
  <<: *preview_rules
  stage: post-deploy
  cache: {}
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    GIT_STRATEGY: none
  script:
    - dog --config "$HOME/.dogrc" event post "documentation htmltest ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
    - find ./public -name "*.html" -exec sed -i -e "s#https://docs-staging.datadoghq.com/$CI_COMMIT_REF_NAME/#/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v1/#/api/latest/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v2/#/api/latest/#g" {} +
    - htmltest
  after_script:
    - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers
    - RESULT="⛈ htmltest result; the following issues were found:\n$(tail -n +2 ./tmp/.htmltest/htmltest.log)\n--------------------" # output all but first line
    - if grep -q "not exist" tmp/.htmltest/htmltest.log; then notify_slack "${RESULT}" "#31b834"; fi
  artifacts:
    paths:
      - ./tmp/.htmltest
  interruptible: true

missing_tms_preview:
  <<: *base_template
  <<: *preview_rules
  stage: post-deploy
  environment: "preview"
  allow_failure: true
  cache: {}
  dependencies:
    - build_preview
  script:
    - check_missing_tms
  interruptible: true

algolia_sync_preview:manual:
  <<: *base_template
  stage: post-deploy
  environment: "preview"
  allow_failure: true
  interruptible: true
  script:
    - yarn add atomic-algolia@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/atomic-algolia-v0.3.29.tgz
    - ALGOLIA_APP_ID=$(get_secret 'algolia_preview_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_preview_api_key') yarn run algolia:sync
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: never
    - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_TAG =~ "/^$/"'
      when: manual

algolia_sync_live:
  <<: *base_template
  <<: *live_rules
  stage: post-deploy
  environment: "live"
  allow_failure: true
  interruptible: true
  script:
    - yarn add atomic-algolia@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/atomic-algolia-v0.3.29.tgz
    - ALGOLIA_APP_ID=$(get_secret 'algolia_docsearch_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_docsearch_api_key') yarn run algolia:sync

sourcemaps_preview:
  <<: *base_template
  <<: *preview_rules
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "preview"
  script:
    - yarn install --immutable
    # Rename any js.map files to include SHA if correlating JS file is fingerprinted.  Hugo's asset build system seems to lose the fingerprint value when generating source maps.
    - find ./public -type f -name "*.*.js" -exec bash -c 'x={}; hash=$(basename $x | cut -d '.' -f 2); map=${x/$hash./}.map; mv $map ${map/.js/.$hash.js}' \;
    - DATADOG_API_KEY="$(get_secret 'dd-api-key')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs-staging.datadoghq.com/${CI_COMMIT_REF_NAME}/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_preview
  allow_failure: true
  interruptible: true

post_build_success_status_to_spec_repo:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "preview"
  script:
    - chmod +x ./local/bin/py/submit_github_status_check.py
    - ./local/bin/py/submit_github_status_check.py -i "$(get_secret 'dd-api-spec-repo-app-id')" -k "$(get_secret 'dd-api-spec-repo-app-key')"
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^datadog-api-spec\/generated.*/
      when: on_success
  interruptible: true
  allow_failure: true

post_build_failure_status_to_spec_repo:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "preview"
  script:
    - chmod +x ./local/bin/py/submit_github_status_check.py
    - ./local/bin/py/submit_github_status_check.py -s "failure" -i "$(get_secret 'dd-api-spec-repo-app-id')" -k "$(get_secret 'dd-api-spec-repo-app-key')"
  rules:
    - if: $CI_COMMIT_BRANCH =~ /^datadog-api-spec\/generated.*/
      when: on_failure
  interruptible: true
  allow_failure: true

# ================== live ================== #
build_live:
  <<: *base_template
  <<: *live_rules
  stage: build
  cache:
    - *hugo_cache
    - *yarn_cache_pull_push
  environment: "live"
  retry: 1
  variables:
    CONFIG: ${LIVE_CONFIG}
    URL: ${LIVE_DOMAIN}
    UNTRACKED_EXTRAS: "data"
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config.yaml"
    LOCAL: "False"
  script:
    - dog --config "$HOME/.dogrc" event post "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" "#FFD700"
    - touch Makefile.config
    - make clean-examples
    - make dependencies
    - yarn run prebuild
    - build_site
    - in-isolation deploy_site
    - in-isolation create_artifact
    - in-isolation create_artifact_ignored
    - in-isolation create_artifact_cached
    - remove_static_from_repo
    - ci_pass_step "${CI_PROJECT_NAME}-${CI_JOB_NAME}" # run at completion of job, if job fails at any point prior this won't run
  artifacts:
    when: on_success
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - content/en
      - .htmltest.yml
      - ./public/redirects.txt
    expire_in: 1 week

test_live_link_checks:
  <<: *base_template
  <<: *live_rules
  stage: post-deploy
  cache: {}
  environment: "live"
  variables:
    URL: ${LIVE_DOMAIN}
    GIT_STRATEGY: none
  script:
    - dog --config "$HOME/.dogrc" event post "documentation htmltest ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
    - find ./public -name "*.html" -exec sed -i -e "s#https://docs.datadoghq.com/#/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v1/#/api/latest/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v2/#/api/latest/#g" {} +
    - htmltest
  after_script:
    - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers
    - RESULT="⛈ htmltest result; the following issues were found:\n$(tail -n +2 ./tmp/.htmltest/htmltest.log)\n--------------------" # output all but first line
    - if grep -q "not exist" tmp/.htmltest/htmltest.log; then notify_slack "${RESULT}" "#31b834"; fi
  artifacts:
    paths:
      - ./tmp/.htmltest

sourcemaps_live:
  <<: *base_template
  <<: *live_rules
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "live"
  script:
    - yarn install --immutable
    # Rename any js.map files to include SHA if correlating JS file is fingerprinted. Hugo's asset build system seems to lose the fingerprint value when generating source maps.
    - find ./public -type f -name "*.*.js" -exec bash -c 'x={}; hash=$(basename $x | cut -d '.' -f 2); map=${x/$hash./}.map; mv $map ${map/.js/.$hash.js}' \;
    - DATADOG_API_KEY="$(get_secret 'dd-api-key')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs.datadoghq.com/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_live
  allow_failure: true

evaluate_missing_metrics:
  <<: *base_template
  stage: post-deploy
  allow_failure: true
  cache: {}
  environment: "live"
  script:
    - chmod +x ./local/bin/py/missing_metrics.py && ./local/bin/py/missing_metrics.py -k $(get_secret 'dd-demo-api-key') -p $(get_secret 'dd-demo-app-key') -a $(get_secret 'dd-api-key') -b $(get_secret 'dd-app-key')
  rules:
    - if: $CI_COMMIT_REF_NAME == "master"
      when: manual

# Uploads git ignored source (EN) files to Transifex as part of the nightly build.
# The translation syncing pipeline is triggered by github webhook, but these files are not in git.
# This allows some dynamically built content (such as integrations) to be localized automatically.
manage_ignored_translations:on-schedule:
  <<: *base_template
  <<: *live_schedule_rules
  stage: post-deploy
  cache: {}
  environment: "live"
  script:
    - echo "Downloading ignored translation source files..."
    - manage_ignored_translation_files

test_missing_tms_live:
  <<: *base_template
  <<: *live_rules
  stage: post-deploy
  cache: {}
  environment: "live"
  dependencies:
    - build_live
  script:
    - check_missing_tms

test_pa11y:
  <<: *base_template
  <<: *live_schedule_rules
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "live"
  dependencies:
    - build_live
  script:
    - yarn install --immutable
    - node ./local/bin/js/pa11y.js --env live --dd_api_key $(get_secret 'dd-api-key')

tag_successful_live_pipeline:
  <<: *base_template
  <<: *live_rules
  stage: cleanup
  cache: {}
  environment: "live"
  when: on_success
  script:
    - tag_successful_pipeline

post_success_event_to_dd_live:
  <<: *base_template
  <<: *live_rules
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  script:
    - dog --config "$HOME/.dogrc" event post "documentation deploy ${CI_COMMIT_REF_NAME} succeeded" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
    - dog --config "$HOME/.dogrc" metric post "documentation.pipeline.completed" "1" --tags="${DEFAULT_TAGS},step_status:success"

post_failure_event_to_dd_live:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  script:
    - dog --config "$HOME/.dogrc" event post "documentation deploy ${CI_COMMIT_REF_NAME} failed" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "error" --tags="${DEFAULT_TAGS},step_status:failure"
    - dog --config "$HOME/.dogrc" metric post "documentation.pipeline.completed" "0" --tags="${DEFAULT_TAGS},step_status:failure"
  rules:
    - if: $CI_COMMIT_REF_NAME == "master"
      when: on_failure

set_redirect_metadata_preview:
  <<: *base_template
  <<: *preview_rules
  variables:
    BUCKET: "datadog-docs-preview"
    SET_REDIRECTS_PREVIEW: "true"
  stage: post-deploy
  cache: []
  environment: "preview"
  dependencies:
    - build_preview
  script:
    - in-isolation run_update_redirect_metadata
  interruptible: true
  allow_failure: true

set_redirect_metadata_live:
  <<: *base_template
  <<: *live_rules
  variables:
    BUCKET: "datadog-docs-live-hugo"
  stage: post-deploy
  cache: []
  environment: "live"
  dependencies:
    - build_live
  script:
    - in-isolation run_update_redirect_metadata
  interruptible: true

synthetic_tests_preview:
  <<: *base_template
  <<: *preview_rules
  stage: post-deploy
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
  allow_failure: false
  cache: {}
  dependencies:
    - build_preview
  script:
    - run_preview_synthetic_tests "./datadog-ci.preview.json"
  interruptible: true