diff --git a/content/docs/draft/cbe-005_gitlab_ci/_ref/Flockademic/.gitlab-ci.yml b/content/docs/draft/cbe-005_gitlab_ci/_ref/Flockademic/.gitlab-ci.yml deleted file mode 100644 index d0eaf801f1..0000000000 --- a/content/docs/draft/cbe-005_gitlab_ci/_ref/Flockademic/.gitlab-ci.yml +++ /dev/null @@ -1,456 +0,0 @@ -image: node:8.10 - -stages: - - prepare - - test - - build-backend - - deploy-backend - - provision-backend - - build-frontend - - deploy-frontend - - confidence-check - -prepare: - stage: prepare - script: - - wipCommits=`git log --grep=^WIP`; if [ -n "$wipCommits" ]; then echo "WIP commits detected; aborting."; exit 1; fi - - yarn install --ignore-scripts --frozen-lockfile - artifacts: - expire_in: 1 month - paths: - - node_modules/ -prepare_frontend: - stage: prepare - script: - - wipCommits=`git log --grep=^WIP`; if [ -n "$wipCommits" ]; then echo "WIP commits detected; aborting."; exit 1; fi - - cd stacks/frontend - - yarn install --frozen-lockfile - artifacts: - expire_in: 1 month - paths: - - stacks/frontend/node_modules/ -prepare_server: - stage: prepare - script: - - wipCommits=`git log --grep=^WIP`; if [ -n "$wipCommits" ]; then echo "WIP commits detected; aborting."; exit 1; fi - - cd server - - yarn install --frozen-lockfile - artifacts: - expire_in: 1 month - paths: - - server/node_modules/ -prepare_accounts: - stage: prepare - script: - - wipCommits=`git log --grep=^WIP`; if [ -n "$wipCommits" ]; then echo "WIP commits detected; aborting."; exit 1; fi - - cd stacks/accounts - - yarn install --frozen-lockfile - artifacts: - expire_in: 1 month - paths: - - stacks/accounts/node_modules/ -prepare_periodicals: - stage: prepare - script: - - wipCommits=`git log --grep=^WIP`; if [ -n "$wipCommits" ]; then echo "WIP commits detected; aborting."; exit 1; fi - - cd stacks/periodicals - - yarn install --frozen-lockfile - artifacts: - expire_in: 1 month - paths: - - stacks/periodicals/node_modules/ - -test: - stage: test - dependencies: - - prepare - script: - - yarn run test -test_frontend: - stage: test - dependencies: - - prepare - - prepare_frontend - script: - - cd stacks/frontend - - yarn run test --ci - artifacts: - expire_in: 1 week - paths: - - stacks/frontend/__tests__/__coverage__/ - when: always -test_accounts: - stage: test - dependencies: - - prepare - - prepare_accounts - script: - - cd stacks/accounts - - yarn run test --ci - artifacts: - expire_in: 1 week - paths: - - stacks/accounts/__tests__/__coverage__/ - when: always -test_periodicals: - stage: test - dependencies: - - prepare - - prepare_periodicals - script: - - cd stacks/periodicals - - yarn run test --ci - artifacts: - expire_in: 1 week - paths: - - stacks/periodicals/__tests__/__coverage__/ - when: always - -build_terraform: - stage: build-backend - dependencies: - # No previous assets needed for this job - script: echo "Dummy job to preserve artifacts for the undeploy job" - artifacts: - expire_in: 1 month - paths: - # The below files are needed for the undeploy jobs, - # because when undeploying the branch (and thus these files) is no longer available - - stacks.tf - - stacks/*.tf - - stacks/*/*.tf - - stacks/*/*_lambda_role_policy.json - - stacks/*/*_lambda_policy.json -build_server: - stage: build-backend - dependencies: - - prepare_server - script: - - cd server - - yarn run build - artifacts: - expire_in: 1 month - paths: - - server/dist/ -build_accounts: - stage: build-backend - dependencies: - - prepare - - prepare_accounts - script: - - cd stacks/accounts - - yarn run build - artifacts: - expire_in: 1 month - paths: - - stacks/accounts/dist/ -build_periodicals: - stage: build-backend - dependencies: - - prepare - - prepare_periodicals - script: - - cd stacks/periodicals - - yarn run build - artifacts: - expire_in: 1 month - paths: - - stacks/periodicals/dist/ - -deploy_terraform: - stage: deploy-backend - image: - name: hashicorp/terraform:0.11.2 - entrypoint: - # These values provided by https://github.com/hashicorp/docker-hub-images/issues/37#issuecomment-356330221 - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - dependencies: - - build_terraform - - build_server - - build_accounts - - build_periodicals - script: - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export TF_VAR_aws_access_key_id=$AWS_ACCESS_KEY_ID_PRODUCTION; else export TF_VAR_aws_access_key_id=$AWS_ACCESS_KEY_ID; fi; - - export TF_VAR_aws_account_id=$AWS_ACCOUNT_ID - - export TF_VAR_aws_region=$AWS_REGION - - export TF_VAR_heroku_email=$HEROKU_EMAIL - - export TF_VAR_heroku_api_key=$HEROKU_API_KEY - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export TF_VAR_aws_secret_access_key=$AWS_SECRET_ACCESS_KEY_PRODUCTION; else export TF_VAR_aws_secret_access_key=$AWS_SECRET_ACCESS_KEY; fi; - - export TF_VAR_jwt_secret=$(echo "$JWT_SECRET$CI_COMMIT_REF_NAME" | sha512sum) - - export TF_VAR_frontend_url="https://$CI_COMMIT_REF_SLUG.flockademic.com" - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export TF_VAR_orcid_base_path="$ORCID_BASE_PATH_PRODUCTION"; export TF_VAR_orcid_client_id="$ORCID_CLIENT_ID_PRODUCTION"; export TF_VAR_orcid_client_secret="$ORCID_CLIENT_SECRET_PRODUCTION"; else export TF_VAR_orcid_base_path="$ORCID_BASE_PATH"; export TF_VAR_orcid_client_id="$ORCID_CLIENT_ID"; export TF_VAR_orcid_client_secret="$ORCID_CLIENT_SECRET"; fi; - # Use the branch/tag name as the TerraForm Workspace, - # limited to 26 chars (30 is the max length of Heroku app names, which we prepend with `fl9-`), - # and with a potential trailing dash stripped - # (Note that I'd rather name this $TF_WORKSPACE, but: - # https://github.com/hashicorp/terraform/issues/15874 ) - - export WORKSPACE_NAME=`echo $CI_COMMIT_REF_SLUG | cut -c 1-26 | sed -e 's/-$//'` - # Deploy everything - - terraform version - - terraform init -backend-config="region=$TF_VAR_aws_region" -input=false - - terraform workspace select $WORKSPACE_NAME || terraform workspace new $WORKSPACE_NAME - - terraform get - - terraform graph - - terraform plan -out .tfplan -input=false | sed -e "s/$TF_VAR_aws_access_key_id/***/g" | sed -e "s/$TF_VAR_aws_secret_access_key/***/g" - - terraform apply -input=false .tfplan | sed -e "s/$TF_VAR_aws_access_key_id/***/g" | sed -e "s/$TF_VAR_aws_secret_access_key/***/g" | sed -e "s/$TF_VAR_orcid_client_secret/***/g" - # Store the API endpoint to a file so that we can pass it on to other jobs (without a trailing slash) - - terraform output heroku_web_url | sed s'/\/$//' > api_url - - terraform output heroku_git_url > heroku_git_url - # Copy everything to a dir to push to Heroku - - cd server - - mkdir -p dist/stacks/accounts - - mkdir -p dist/stacks/periodicals - - cp package.json yarn.lock dist - # Make sure that library dependencies are available to all stacks: - - cp ../package.json ../yarn.lock dist/stacks - - cp -r ../stacks/accounts/dist ../stacks/accounts/package.json ../stacks/accounts/yarn.lock dist/stacks/accounts - - cp -r ../stacks/periodicals/dist ../stacks/periodicals/package.json ../stacks/periodicals/yarn.lock dist/stacks/periodicals - - cd dist - - git init . - - git config user.email "ci@flockademic.com" - - git config user.name "CI deployment" - - git add . - - git commit -m "Create deployment package for Heroku" - - git remote add heroku `cat ../../heroku_git_url` - - echo "machine git.heroku.com login \"\" password $HEROKU_API_KEY" > ~/.netrc - - chmod 600 ~/.netrc - - git push --force heroku master - environment: - name: review_api/$CI_COMMIT_REF_SLUG - url: https://api.$CI_COMMIT_REF_SLUG.flockademic.com - on_stop: stop_terraform - artifacts: - # This is just needed for subsequent jobs, so 1 hour should be enough - expire_in: 1 hour - paths: - - api_url -stop_terraform: - stage: deploy-backend - image: - name: hashicorp/terraform:0.11.2 - entrypoint: - # These values provided by https://github.com/hashicorp/docker-hub-images/issues/37#issuecomment-356330221 - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - variables: - GIT_STRATEGY: none - dependencies: - - build_terraform - script: - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export TF_VAR_aws_access_key_id=$AWS_ACCESS_KEY_ID_PRODUCTION; else export TF_VAR_aws_access_key_id=$AWS_ACCESS_KEY_ID; fi; - - export TF_VAR_aws_account_id=$AWS_ACCOUNT_ID - - export TF_VAR_aws_region=$AWS_REGION - - export TF_VAR_heroku_email=$HEROKU_EMAIL - - export TF_VAR_heroku_api_key=$HEROKU_API_KEY - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export TF_VAR_aws_secret_access_key=$AWS_SECRET_ACCESS_KEY_PRODUCTION; else export TF_VAR_aws_secret_access_key=$AWS_SECRET_ACCESS_KEY; fi; - - export TF_VAR_jwt_secret="irrelevant_for_undeployment" - - export TF_VAR_orcid_base_path="irrelevant_for_undeployment" - - export TF_VAR_orcid_client_id="irrelevant_for_undeployment" - - export TF_VAR_orcid_client_secret="irrelevant_for_undeployment" - - export TF_VAR_frontend_url="irrelevant_for_undeployment" - # Use the branch/tag name as the TerraForm Workspace, - # limited to 26 chars (30 is the max length of Heroku app names, which we prepend with `fl9-`), - # and with a potential trailing dash stripped - # (Note that I'd rather name this $TF_WORKSPACE, but: - # https://github.com/hashicorp/terraform/issues/15874 ) - - export WORKSPACE_NAME=`echo $CI_COMMIT_REF_SLUG | cut -c 1-26 | sed -e 's/-$//'` - - terraform init -backend-config="region=$TF_VAR_aws_region" -input=false - - terraform workspace select $WORKSPACE_NAME - - terraform get - - terraform plan -destroy -input=false - - terraform destroy -input=false -force - when: manual - environment: - name: review_api/$CI_COMMIT_REF_SLUG - action: stop - -provision_databases: - stage: provision-backend - dependencies: - - deploy_terraform - image: alpine:3.6 - script: - - apk add --no-cache curl jq postgresql-client - # Use the branch/tag name as the TerraForm Workspace, - # limited to 26 chars (30 is the max length of Heroku app names, which we prepend with `fl9-`), - # and with a potential trailing dash stripped - # (Note that I'd rather name this $TF_WORKSPACE, but: - # https://github.com/hashicorp/terraform/issues/15874 ) - - export WORKSPACE_NAME=`echo $CI_COMMIT_REF_SLUG | cut -c 1-26 | sed -e 's/-$//'` - - "DATABASE_URL=`curl -X GET \"https://api.heroku.com/apps/fl9-${WORKSPACE_NAME}/config-vars\" -H \"Accept: application/vnd.heroku+json; version=3\" -H \"Authorization: Bearer ${HEROKU_API_KEY}\" | jq -r \".DATABASE_URL\"`" - # $DATABASE_URL is of the form `postgres://:@:/ - - export PGHOST=`echo ${DATABASE_URL:11} | cut -d '@' -f 2 | cut -d ':' -f 1` - - export PGPORT=`echo ${DATABASE_URL:11} | cut -d '@' -f 2 | cut -d ':' -f 2 | cut -d '/' -f 1` - - export PGUSER=`echo ${DATABASE_URL:11} | cut -d '@' -f 1 | cut -d ':' -f 1` - - export PGDATABASE=`echo ${DATABASE_URL:11} | cut -d '@' -f 2 | cut -d ':' -f 2 | cut -d '/' -f 2` - - echo "$PGHOST:$PGPORT:$PGDATABASE:$PGUSER:`echo ${DATABASE_URL:11} | cut -d '@' -f 1 | cut -d ':' -f 2`" > .pgpass.conf - - chmod 0600 .pgpass.conf - - export PGPASSFILE=.pgpass.conf - # Create the databases and provision them: - - sh stacks/provision_databases.sh stacks - -build_frontend: - stage: build-frontend - dependencies: - - prepare - - prepare_frontend - - deploy_terraform - script: - - cd stacks/frontend - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export ORCID_BASE_PATH="$ORCID_BASE_PATH_PRODUCTION"; export ORCID_CLIENT_ID="$ORCID_CLIENT_ID_PRODUCTION"; fi; - - export API_URL=`cat ../../api_url` - - export CODE_BRANCH="$CI_COMMIT_REF_SLUG" - - if [[ $CI_COMMIT_REF_NAME == $BRANCH_IN_MAINTENANCE ]]; then echo 'Building in maintenance mode.'; export MAINTENANCE_MODE=true; fi; - - yarn run build - artifacts: - expire_in: 1 month - paths: - - stacks/frontend/dist/ - # The below files are needed for the undeploy jobs, - # because when undeploying the branch (and thus these files) is no longer available - - stacks/frontend/package.json - - stacks/frontend/build/ - -deploy_frontend: - stage: deploy-frontend - dependencies: - - prepare_frontend - - build_frontend - script: - - cd stacks/frontend - - yarn run deploy - - yarn run awaitDeployTasks - environment: - name: review/$CI_COMMIT_REF_SLUG - url: https://$CI_COMMIT_REF_SLUG.flockademic.com - on_stop: stop_frontend - except: - - master -deploy_frontend_prod: - stage: deploy-frontend - dependencies: - - prepare_frontend - - build_frontend - script: - - cd stacks/frontend - - export AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID_PRODUCTION" - - export AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY_PRODUCTION" - - yarn run deploy - - yarn run awaitDeployTasks - environment: - # Since the deploy_terraform environment cannot have different environment names for production and review apps, - # we're not doing that here either in order to be able to match on the name and make certain variables only available to production. - name: review/$CI_COMMIT_REF_SLUG - url: https://flockademic.com - only: - - master -stop_frontend: - stage: deploy-frontend - variables: - GIT_STRATEGY: none - dependencies: - - prepare_frontend - - build_frontend - script: - - cd stacks/frontend - - yarn run undeploy - when: manual - environment: - name: review/$CI_COMMIT_REF_SLUG - action: stop - -e2e:firefox: - stage: confidence-check - services: - - selenium/standalone-firefox:3.13 - dependencies: - - prepare - script: - - if [[ $CI_COMMIT_REF_NAME == $BRANCH_IN_MAINTENANCE ]]; then echo 'Running checks in maintenance mode.'; export MAINTENANCE_MODE=true; fi; - - yarn run confidence-check --host=selenium__standalone-firefox - artifacts: - expire_in: 1 week - paths: - - __e2e__/results/ - - __e2e__/errorShots/ - when: always - # GUI tests loading a full browser are notoriously unstable. - # See https://gitlab.com/Flockademic/Flockademic/issues/366 - retry: 2 -e2e:chrome: - stage: confidence-check - services: - - selenium/standalone-chrome:3.13 - dependencies: - - prepare - script: - - if [[ $CI_COMMIT_REF_NAME == $BRANCH_IN_MAINTENANCE ]]; then echo 'Running checks in maintenance mode.'; export MAINTENANCE_MODE=true; fi; - - yarn run confidence-check --host=selenium__standalone-chrome - artifacts: - expire_in: 1 week - paths: - - __e2e__/results/ - - __e2e__/errorShots/ - when: always - # GUI tests loading a full browser are notoriously unstable. - # See https://gitlab.com/Flockademic/Flockademic/issues/366 - retry: 2 -# See https://docs.gitlab.com/ce/ci/examples/sast.html -sast: - stage: confidence-check - image: registry.gitlab.com/gitlab-org/gl-sast:latest - dependencies: - - prepare - - prepare_frontend - - prepare_accounts - - prepare_periodicals - script: - - /app/bin/run . - artifacts: - paths: - - gl-sast-report.json -# See https://docs.gitlab.com/ee/ci/examples/dast.html -dast: - stage: confidence-check - image: owasp/zap2docker-weekly - dependencies: - script: - - mkdir /zap/wrk/ - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export SITE_URL=https://flockademic.com; else export SITE_URL=https://$CI_COMMIT_REF_SLUG.flockademic.com; fi; - - /zap/zap-baseline.py -J gl-dast-report.json -t $SITE_URL || true - - cp /zap/wrk/gl-dast-report.json . - artifacts: - paths: [gl-dast-report.json] -# See https://docs.gitlab.com/ee/ci/examples/browser_performance.html -performance: - stage: confidence-check - image: docker:git - dependencies: - services: - - docker:stable-dind - script: - - mkdir gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - - mkdir sitespeed-results - - if [[ $CI_COMMIT_REF_NAME == "master" ]]; then export SITE_URL=https://flockademic.com; else export SITE_URL=https://$CI_COMMIT_REF_SLUG.flockademic.com; fi; - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $SITE_URL - - mv sitespeed-results/data/performance.json performance.json - artifacts: - paths: - - performance.json -license_management: - stage: confidence-check - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - script: - - export LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run - --volume "$PWD:/code" - "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code - artifacts: - paths: [gl-license-management-report.json] diff --git a/content/docs/draft/cbe-005_gitlab_ci/_ref/gitlab/.gitlab-ci.yml b/content/docs/draft/cbe-005_gitlab_ci/_ref/gitlab/.gitlab-ci.yml new file mode 100644 index 0000000000..cc52aec04e --- /dev/null +++ b/content/docs/draft/cbe-005_gitlab_ci/_ref/gitlab/.gitlab-ci.yml @@ -0,0 +1,290 @@ +stages: + - sync + - preflight + - prepare + - build-images + - fixtures + - lint + - test-frontend + - test + - post-test + - review + - qa + - post-qa + - pre-merge + - pages + - notify + - release-environments + - benchmark + +# always use `gitlab-org` runners, however +# in cases where jobs require Docker-in-Docker, the job +# definition must be extended with `.use-docker-in-docker` +default: + image: $DEFAULT_CI_IMAGE + tags: + - $DEFAULT_JOB_TAG + # All jobs are interruptible by default + interruptible: true + # Default job timeout doesn't work: https://gitlab.com/gitlab-org/gitlab/-/issues/387528 + timeout: 90m + +.default-ruby-variables: &default-ruby-variables + RUBY_VERSION: "${RUBY_VERSION_DEFAULT}" + OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY3_1" + +.next-ruby-variables: &next-ruby-variables + RUBY_VERSION: "${RUBY_VERSION_NEXT}" + OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY3_2" + +.default-branch-pipeline-failure-variables: &default-branch-pipeline-failure-variables + CREATE_RAILS_FLAKY_TEST_ISSUES: "true" + CREATE_RAILS_SLOW_TEST_ISSUES: "true" + CREATE_RAILS_TEST_FAILURE_ISSUES: "true" + +.default-merge-request-variables: &default-merge-request-variables + NO_SOURCEMAPS: "true" + ADD_SLOW_TEST_NOTE_TO_MERGE_REQUEST: "true" + FF_NETWORK_PER_BUILD: "true" + FF_TIMESTAMPS: "true" + +.if-merge-request-security-canonical-sync: &if-merge-request-security-canonical-sync + if: '$CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $CI_DEFAULT_BRANCH && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' + +.if-not-security-canonical-sync: &if-not-security-canonical-sync + if: '$CI_MERGE_REQUEST_SOURCE_PROJECT_PATH != "gitlab-org/security/gitlab" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME != $CI_DEFAULT_BRANCH' + +.if-merge-request-labels-run-with-rails-next: &if-merge-request-labels-run-with-rails-next + if: '($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_EVENT_TYPE != "merge_train") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-with-rails-next/' + +workflow: + name: '$PIPELINE_NAME' + rules: + - if: '$CI_PIPELINE_SOURCE == "pipeline" && $GITALY_TEST' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Gitaly Rails Test Pipeline' + # If `$FORCE_GITLAB_CI` is set, create a pipeline. + - if: '$FORCE_GITLAB_CI' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION forced' + - if: '$START_AS_IF_FOSS' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION as-if-foss' + # As part of the process of creating RCs automatically, we update stable + # branches with the changes of the most recent production deployment. The + # merge requests used for this merge a branch release-tools/X into a stable + # branch. For these merge requests we don't want to run any pipelines, as + # they serve no purpose and will run anyway when the changes are merged. + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^release-tools\/\d+\.\d+\.\d+-rc\d+$/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable(-ee)?$/ && $CI_PROJECT_PATH == "gitlab-org/gitlab"' + when: never + - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_1/ || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable(-ee)?$/' + variables: + <<: [*default-ruby-variables, *default-merge-request-variables] + PIPELINE_NAME: 'Ruby $RUBY_VERSION MR' + - if: '$CI_MERGE_REQUEST_LABELS =~ /Community contribution/' + variables: + <<: [*next-ruby-variables, *default-merge-request-variables] + GITLAB_DEPENDENCY_PROXY_ADDRESS: "" + PIPELINE_NAME: 'Ruby $RUBY_VERSION MR (community contribution)' + # This work around https://gitlab.com/gitlab-org/gitlab/-/issues/332411 which prevents usage of dependency proxy + # when pipeline is triggered by a project access token. + # Example of project bot usernames (the format changed over time): + # - project_278964_bot2 + # - project_278964_bot_7fb4d1cca8242cb399a0b8f483783120 + - if: '$CI_MERGE_REQUEST_IID && $GITLAB_USER_LOGIN =~ /project_\d+_bot/' + variables: + <<: [*next-ruby-variables, *default-merge-request-variables] + GITLAB_DEPENDENCY_PROXY_ADDRESS: "" + PIPELINE_NAME: 'Ruby $RUBY_VERSION MR (triggered by a project token)' + - <<: *if-merge-request-security-canonical-sync + variables: + <<: [*next-ruby-variables, *default-merge-request-variables] + PIPELINE_NAME: '$CI_DEFAULT_BRANCH security->canonical sync' + SKIP_MESSAGE: 'MR only contains changes from the security mirror, which have already been reviewed, tested and deployed.' + - <<: *if-merge-request-labels-run-with-rails-next + variables: + <<: [*next-ruby-variables, *default-merge-request-variables] + BUNDLE_GEMFILE: Gemfile.next + # For (detached) merge request pipelines. + - if: '$CI_MERGE_REQUEST_IID' + variables: + <<: [*next-ruby-variables, *default-merge-request-variables] + PIPELINE_NAME: 'Ruby $RUBY_VERSION MR' + # For the scheduled pipelines, we set specific variables. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $BUILD_WITH_NEXT_RUBY_VERSION == "true"' + variables: + <<: *next-ruby-variables + PIPELINE_NAME: 'Scheduled Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule"' + variables: + <<: [*default-ruby-variables, *default-branch-pipeline-failure-variables] + CRYSTALBALL: "true" + PIPELINE_NAME: 'Scheduled Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + - if: '$CI_COMMIT_BRANCH == "ruby3_2" && $CI_PIPELINE_SOURCE == "schedule"' + variables: + <<: *next-ruby-variables + PIPELINE_NAME: 'Scheduled Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + # This work around https://gitlab.com/gitlab-org/gitlab/-/issues/332411 which prevents usage of dependency proxy + # when pipeline is triggered by a project access token. + # Example of project bot usernames (the format changed over time): + # - project_278964_bot2 + # - project_278964_bot_7fb4d1cca8242cb399a0b8f483783120 + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot/' + variables: + <<: [*default-ruby-variables, *default-branch-pipeline-failure-variables] + GITLAB_DEPENDENCY_PROXY_ADDRESS: "" + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch (triggered by a project token)' + # For `$CI_DEFAULT_BRANCH` from wider community contributors, we don't want to run any pipelines on pushes, + # because normally we want to run merge request pipelines and scheduled pipelines, not for repository synchronization. + # This can avoid accidentally using up pipeline minutes quota while synchronizing the repository for wider community contributors. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push" && $CI_PROJECT_NAMESPACE !~ /^gitlab(-org|-cn)?($|\/)/' + when: never + # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + variables: + <<: [*default-ruby-variables, *default-branch-pipeline-failure-variables] + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + # For tags, create a pipeline. + - if: '$CI_COMMIT_TAG' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_TAG tag' + # If `$GITLAB_INTERNAL` isn't set, don't create a pipeline. + - if: '$GITLAB_INTERNAL == null' + when: never + # For stable, auto-deploy, and security branches, create a pipeline. + - if: '$CI_COMMIT_BRANCH =~ /^[\d-]+-stable(-ee)?$/' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + - if: '$CI_COMMIT_BRANCH =~ /^\d+-\d+-auto-deploy-\d+$/' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + - if: '$CI_COMMIT_BRANCH =~ /^security\//' + variables: + <<: *default-ruby-variables + PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_COMMIT_BRANCH branch' + +variables: + PG_VERSION: "14" + DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-${NODE_VERSION}-postgresql-${PG_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-yarn-${YARN_VERSION}-graphicsmagick-${GRAPHICSMAGICK_VERSION}" + DEFAULT_JOB_TAG: "gitlab-org" + GITLAB_LARGE_RUNNER_OPTIONAL: "gitlab-org" # overridden just in gitlab-org/gitlab + DEFAULT_RSPEC_PREDICTIVE_JOB_TAGS: "${DEFAULT_JOB_TAG}" # Separated by commas, overridden in JiHu + # We set $GITLAB_DEPENDENCY_PROXY to another variable (since it's set at the group level and has higher precedence than .gitlab-ci.yml) + # so that we can override $GITLAB_DEPENDENCY_PROXY_ADDRESS in workflow rules. + GITLAB_DEPENDENCY_PROXY_ADDRESS: "${GITLAB_DEPENDENCY_PROXY}" + RAILS_ENV: "test" + NODE_ENV: "test" + BUNDLE_WITHOUT: "production:development" + BUNDLE_INSTALL_FLAGS: "--jobs=$(nproc) --retry=3" + BUNDLE_FROZEN: "true" + BUNDLE_GEMFILE: Gemfile + # we override the max_old_space_size to prevent OOM errors + NODE_OPTIONS: --max-old-space-size=10240 + GIT_DEPTH: "20" + # 'GIT_STRATEGY: clone' optimizes the pack-objects cache hit ratio + GIT_STRATEGY: "clone" + GIT_SUBMODULE_STRATEGY: "none" + GET_SOURCES_ATTEMPTS: "3" + # CI_FETCH_REPO_GIT_STRATEGY: "none" is from artifacts. "clone" is from cloning + CI_FETCH_REPO_GIT_STRATEGY: "none" + FORCE_COLOR: 1 + CLICOLOR_FORCE: 1 + + FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json + FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json + GITLAB_WORKHORSE_FOLDER: "gitlab-workhorse" + JOB_METRICS_FILE_PATH: "${CI_PROJECT_DIR}/tmp/job-metrics.json" + KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json + RSPEC_CHANGED_FILES_PATH: rspec/changed_files.txt + RSPEC_FAIL_FAST_THRESHOLD: 20 + RSPEC_FAST_QUARANTINE_PATH: rspec/fast_quarantine-gitlab.txt + RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt + RSPEC_MATCHING_JS_FILES_PATH: rspec/js_matching_files.txt + RSPEC_MATCHING_TESTS_EE_PATH: rspec/matching_tests-ee.txt + RSPEC_MATCHING_TESTS_FOSS_PATH: rspec/matching_tests-foss.txt + RSPEC_MATCHING_TESTS_PATH: rspec/matching_tests.txt + RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json + RSPEC_PACKED_TESTS_MAPPING_ALT_PATH: crystalball/packed-mapping-alt.json + RSPEC_PREDICTIVE_PIPELINE_TEMPLATE_YML: .gitlab/ci/rails/rspec-predictive.gitlab-ci.yml.erb + RSPEC_PROFILING_FOLDER_PATH: rspec/profiling + RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json + RSPEC_TESTS_MAPPING_ALT_PATH: crystalball/mapping-alt.json + RSPEC_VIEWS_INCLUDING_PARTIALS_PATH: rspec/views_including_partials.txt + RSPEC_AUTO_EXPLAIN_LOG_PATH: auto_explain/auto_explain.ndjson.gz + TMP_TEST_FOLDER: "${CI_PROJECT_DIR}/tmp/tests" + TMP_TEST_GITLAB_WORKHORSE_PATH: "${TMP_TEST_FOLDER}/${GITLAB_WORKHORSE_FOLDER}" + + ES_JAVA_OPTS: "-Xms256m -Xmx256m" + ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200" + BUNDLER_CHECKSUM_VERIFICATION_OPT_IN: "1" + CACHE_CLASSES: "true" + CHECK_PRECOMPILED_ASSETS: "true" + RETRY_FAILED_TESTS_IN_NEW_PROCESS: "true" + # Run with decomposed databases by default + DECOMPOSED_DB: "true" + SEC_DECOMPOSED_DB: "false" + + DOCS_REVIEW_APPS_DOMAIN: "docs.gitlab-review.app" + DOCS_GITLAB_REPO_SUFFIX: "ee" + + REVIEW_APPS_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:gcloud-${GCLOUD_VERSION}-kubectl-1.30-helm-${HELM_VERSION}" + REVIEW_APPS_DOMAIN: "gitlab-review.app" + REVIEW_APPS_GCP_PROJECT: "gitlab-review-apps" + REVIEW_APPS_GCP_REGION: "us-central1" + + REGISTRY_HOST: "registry.gitlab.com" + REGISTRY_GROUP: "gitlab-org" + + # Disable useless network connections when installing some NPM packages. + # See https://gitlab.com/gitlab-com/gl-security/engineering-and-research/inventory/-/issues/827#note_1203181407 + DISABLE_OPENCOLLECTIVE: "true" + + # This is set at the gitlab-org level, but we set it here for forks + DANGER_DO_NOT_POST_INVALID_DANGERFILE_ERROR: "1" + + # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/390313. This can be dropped whenever + # https://github.com/ruby/ruby/pull/7663 lands in the Ruby interpreter. + NOKOGIRI_LIBXML_MEMORY_MANAGEMENT: default + + # CI jobs behavior can be changed by changing the value of these variables in the project's CI/CD variables + AVERAGE_KNAPSACK_REPORT: "true" + ENABLE_DEPSCORE: "true" + CACHE_ASSETS_AS_PACKAGE: "true" + REUSE_FRONTEND_FIXTURES_ENABLED: "true" + BUILD_ASSETS_IMAGE: "true" # Set it to "false" to disable assets image building, used in `build-assets-image` + SIMPLECOV: "true" + +include: + - local: .gitlab/ci/_skip.yml + rules: + - <<: *if-merge-request-security-canonical-sync + - local: .gitlab/ci/version.yml + - local: .gitlab/ci/*.gitlab-ci.yml + rules: + - <<: *if-not-security-canonical-sync + - remote: 'https://gitlab.com/gitlab-org/frontend/untamper-my-lockfile/-/raw/main/templates/merge_request_pipelines.yml' + rules: + - <<: *if-not-security-canonical-sync + - local: .gitlab/ci/includes/gitlab-com/*.gitlab-ci.yml + rules: + - if: '$CI_SERVER_HOST == "gitlab.com"' + - if: '$CI_SERVER_HOST == "jihulab.com"' + - local: .gitlab/ci/includes/as-if-jh.gitlab-ci.yml + rules: + # Only run as-if-jh triggerred pipelines for gitlab.com/gitlab-org/gitlab MRs that don't target stable branches + # and that don't have the quarantine or pipeline::expedited labels. + - if: '$CI_PROJECT_URL != "https://gitlab.com/gitlab-org/gitlab"' + when: never + - if: '$CI_MERGE_REQUEST_ID == null' + when: never + - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable(-ee|-jh)?$/' + when: never + - if: '$CI_MERGE_REQUEST_LABELS =~ /quarantine/ || $CI_MERGE_REQUEST_LABELS =~ /pipeline::expedited/ || $CI_MERGE_REQUEST_LABELS =~ /pipeline:expedite/' + when: never + - when: always diff --git a/content/docs/draft/cbe-005_gitlab_ci/en.md b/content/docs/draft/cbe-005_gitlab_ci/en.md index 52c789897b..9657aea254 100644 --- a/content/docs/draft/cbe-005_gitlab_ci/en.md +++ b/content/docs/draft/cbe-005_gitlab_ci/en.md @@ -15,7 +15,7 @@ Source: https://raw.githubusercontent.com/cue-labs/cue-by-example/refs/heads/mai export PATH=/cues/$CUELANG_CUE_PRERELEASE:$PATH # Set up example content as a git repo. -cd Flockademic +cd gitlab #ellipsis 0 git init . git config user.email cuelang.org@cue.example @@ -42,16 +42,18 @@ know you're managing your pipelines with CUE. ## Prerequisites -- You have [`cue` installed](https://cuelang.org/docs/install/). - You have a GitLab pipeline file. - - The example shown throughout this guide uses the state of a specific commit - from the - [Flockademic repository](https://gitlab.com/Flockademic/Flockademic/-/blob/8efcea927b10c2773790fe78bb858905a75cf3ef/.gitlab-ci.yml) - on gitlab.com, as linked from - [GitLab's documentation pages](https://docs.gitlab.com/ee/ci/examples/end_to_end_testing_webdriverio/index.html), - but you don't need to use that repository in any way.\ - It is used here as it represents a reasonably complex example of a GitLab + - The example shown throughout this guide uses the pipeline file from a + specific commit in the + [`gitlab-org/gitlab` repository](https://gitlab.com/gitlab-org/gitlab/-/blob/3308936efcd70839cc61e0545dcb780756e4ec28/.gitlab-ci.yml) + on gitlab.com, as linked from GitLab's + [CI documentation pages](https://docs.gitlab.com/ee/ci/yaml/), + but **you don't need to use that repository in any way**. It's used as the + example in this guide only because it's a reasonably complex GitLab pipeline file. +- You have [`cue` installed](https://cuelang.org/docs/install/). + - You must have version `v0.11.0-alpha.4` or later installed. Using an + earlier version will cause certain commands in this guide to fail. - You have [`git` installed](https://git-scm.com/downloads). - You have [`curl` installed](https://curl.se/dlwiz/), or can fetch a remote file some other way. @@ -68,8 +70,8 @@ no modified files. For example: :computer: `terminal` {{{with script "en" "1"}}} -cd Flockademic # our example repository -git status # should report "working tree clean" +cd gitlab # our example repository +git status # should report "working tree clean" {{{end}}} #### :arrow_right: Initialise a CUE module @@ -79,7 +81,7 @@ working with, but containing only lowercase letters and numbers. For example: :computer: `terminal` {{{with script "en" "2"}}} -cue mod init gitlab.com/flockademic/flockademic +cue mod init gitlab.com/gitlab-org/gitlab {{{end}}} #### :arrow_right: Import YAML pipeline @@ -133,12 +135,12 @@ The output should reflect your pipeline. In our example: package gitlab pipelines: ".gitlab-ci": { - image: "node:8.10" stages: [ + "sync", + "preflight", "prepare", - "test", - "build-backend", - "deploy-backend", + "build-images", + "fixtures", {{{end}}} {{{with _script_ "en" "5-check"}}} @@ -175,7 +177,7 @@ place it in the `internal/ci/gitlab` directory: :computer: `terminal` {{{with script "en" "8"}}} -curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/7aa6170c4c81a98f372d7c52f3918858c4b69cca/app/assets/javascripts/editor/schema/ci.json +curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/277c9f6b643c92d00101aca0f2b4b874a144f7c5/app/assets/javascripts/editor/schema/ci.json {{{end}}} We use a specific commit from the upstream repository to make sure that this @@ -207,13 +209,27 @@ Create the file in the `internal/ci/gitlab/` directory and add this CUE: :floppy_disk: `internal/ci/gitlab/pipelines.cue` {{{with upload "en" "10"}}} --- Flockademic/internal/ci/gitlab/pipelines.cue -- +-- gitlab/internal/ci/gitlab/pipelines.cue -- package gitlab // each member of the pipelines struct must be a valid #Pipeline -pipeline: [_]: #Pipeline +pipelines: [_]: #Pipeline {{{end}}} +#### :arrow_right: Validate your pipelines + +:computer: `terminal` +{{{with script "en" "10.5"}}} +cue vet ./internal/ci/gitlab +{{{end}}} + +If this command fails and produces any output, then CUE believes that at least +one of your pipelines isn't valid. You'll need to resolve this before +continuing, by updating your pipelines inside your new CUE files. If you're +having difficulty fixing them, please come and ask for help in the friendly CUE +[Slack workspace](https://cuelang.org/s/slack) or +[Discord server](https://cuelang.org/s/discord)! + ### Generate YAML from CUE #### :arrow_right: Create a CUE workflow command @@ -223,7 +239,7 @@ Adapt the element commented with `TODO`: :floppy_disk: `internal/ci/gitlab/ci_tool.cue` {{{with upload "en" "11"}}} --- Flockademic/internal/ci/gitlab/ci_tool.cue -- +-- gitlab/internal/ci/gitlab/ci_tool.cue -- package gitlab import ( @@ -276,16 +292,13 @@ example: cd $(git rev-parse --show-toplevel) # make sure we're sitting at the repository root # Actual command in CUE-By-Example guide: # cue help cmd regenerate ./internal/ci/gitlab # the "./" prefix is required -cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.expected.txt -# Sometimes the above command's regeneration of the YAML file doesn't get -# sync'd to disk before we git-diff it, below. Make sure that it does. -sync +cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.actual.txt {{{end}}} The output of the `cue help` command **must** begin with the following: {{{with upload "en" "12"}}} --- 12.actual.txt -- +-- 12.expected.txt -- Regenerate pipeline files Usage: @@ -317,12 +330,16 @@ original: :computer: `terminal` {{{with script "en" "14"}}} +# 2 commands not present in CUE-By-Example guide, added as an attempt to work +# around cue-lang/cue#3492. DELETE THESE COMMANDS! +sleep 1 +sync # Actual command in CUE-By-Example guide: # git diff .gitlab-ci.yml # For some unknown reason the trailing '>../...' redirection *only* works when # the diff command is given a '--' separator. I'm utterly stumped, but let's # just give it what it wants! -git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\.\.[0-9a-f]{7}' | head -10 >../14.actual.txt +git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\.\.[0-9a-f]{7}' | head -9 >../14.actual.txt {{{end}}} Your output should look similar to the following example: @@ -332,18 +349,16 @@ Your output should look similar to the following example: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml -@@ -1,5 +1,6 @@ --image: node:8.10 +@@ -1,3 +1,5 @@ +# Code generated by internal/ci/gitlab/ci_tool.cue; DO NOT EDIT. - -+image: node:8.10 ++ stages: - - prepare + - sync + - preflight {{{end}}} {{{with _script_ "en" "14-check"}}} -# TODO(jcm): This is unstable (cue-lang/cue#3492). Disabled temporarily. -# diff --side ../14.expected.txt ../14.actual.txt +diff --side ../14.expected.txt ../14.actual.txt {{{end}}} The main change in each YAML file is the addition of a header that warns the diff --git a/content/docs/draft/cbe-005_gitlab_ci/gen_cache.cue b/content/docs/draft/cbe-005_gitlab_ci/gen_cache.cue index fbd3d822d2..af3427f9de 100644 --- a/content/docs/draft/cbe-005_gitlab_ci/gen_cache.cue +++ b/content/docs/draft/cbe-005_gitlab_ci/gen_cache.cue @@ -8,15 +8,15 @@ package site cache: { upload: { "4": "qlFgYLWiExD4s+LL0HpI0+zi2ar6W8GYQ9+kS/d8WbQ=" - "5": "si8+M/I9Hm1HzzLf/o/TH3dyqsLLLNIrySWCZfwjiao=" - "10": "NhZ6xrCVQsIebEqXHZ0lFS4Y9Ms4Ujj8NHMyinVj5o4=" - "11": "qZjvSiXel+VPIBcPlIIcGOdEYdAcSIN9syB35a9hixM=" - "12": "16yZUhoE0tRlEYse56HnQlU6xolOGlRhVYvjDt6OUyU=" - "14": "U3CbYQ0viVRaklzhlEknDxcUj4/DtH1Ugwo7WxdCzl8=" + "5": "A/4TLoi0XnDkfYEVIp8t/jPyAlp0aIL+FLBATEzqRsY=" + "10": "En3G8+vF3Miolfz0uvQEXV7FRWtxjg9D3shPU68yAV4=" + "11": "KRTLuuzhmOFWvgwwVrmpyHXZ8Lbfb1sNG8jwmU9rObM=" + "12": "v4JudRwTrtvlNSc3vYxnk7QfpSbwXZ5BsjYvk7h0BtY=" + "14": "8pAEzSv4YBV5tvsNaJ88V8+Pe8fZZHSJEIHAVqS3yXc=" } multi_step: { - hash: "1IDAPD6OR11PR9O62REVB4P8R0U7PJ4IVBL79K13OCLNCVCD7P3G====" - scriptHash: "IVS5FP8VISFOJ2PF93OMKF2470LMT73GR58B5A29UT9N93GFK4OG====" + hash: "0PTMV30A2P4E02JD0F3CAKJ934EEUIUG6QUCMSP3JUQCUD2EIQ7G====" + scriptHash: "VO0U61BNH9F5CE9RKCSOQD1MBTBDJPBBF5TTGSV9HRTSPIBS5D00====" steps: [{ doc: """ # Required because v0.10.0 suffers from https://cuelang.org/issue/3462, which @@ -27,7 +27,7 @@ package site output: "" }, { doc: "# Set up example content as a git repo." - cmd: "cd Flockademic" + cmd: "cd gitlab" exitCode: 0 output: "" }, { @@ -68,7 +68,7 @@ package site output: "" }, { doc: "" - cmd: "cd Flockademic # our example repository" + cmd: "cd gitlab # our example repository" exitCode: 0 output: "" }, { @@ -82,7 +82,7 @@ package site """ }, { doc: "" - cmd: "cue mod init gitlab.com/flockademic/flockademic" + cmd: "cue mod init gitlab.com/gitlab-org/gitlab" exitCode: 0 output: "" }, { @@ -119,12 +119,12 @@ package site package gitlab\t\t\t\t\t\t\tpackage gitlab pipelines: ".gitlab-ci": {\t\t\t\t\tpipelines: ".gitlab-ci": { - \timage: "node:8.10"\t\t\t\t\t\timage: "node:8.10" \tstages: [\t\t\t\t\t\t\tstages: [ + \t\t"sync",\t\t\t\t\t\t\t\t"sync", + \t\t"preflight",\t\t\t\t\t\t\t"preflight", \t\t"prepare",\t\t\t\t\t\t\t"prepare", - \t\t"test",\t\t\t\t\t\t\t\t"test", - \t\t"build-backend",\t\t\t\t\t\t"build-backend", - \t\t"deploy-backend",\t\t\t\t\t\t"deploy-backend", + \t\t"build-images",\t\t\t\t\t\t\t"build-images", + \t\t"fixtures",\t\t\t\t\t\t\t"fixtures", """ }, { @@ -139,7 +139,7 @@ package site output: "" }, { doc: "" - cmd: "curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/7aa6170c4c81a98f372d7c52f3918858c4b69cca/app/assets/javascripts/editor/schema/ci.json" + cmd: "curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/277c9f6b643c92d00101aca0f2b4b874a144f7c5/app/assets/javascripts/editor/schema/ci.json" exitCode: 0 output: "" }, { @@ -147,6 +147,11 @@ package site cmd: "cue import -p gitlab -l '#Pipeline:' internal/ci/gitlab/gitlab.cicd.pipeline.schema.json" exitCode: 0 output: "" + }, { + doc: "" + cmd: "cue vet ./internal/ci/gitlab" + exitCode: 0 + output: "" }, { doc: "" cmd: "cd $(git rev-parse --show-toplevel) # make sure we're sitting at the repository root" @@ -157,25 +162,30 @@ package site # Actual command in CUE-By-Example guide: # cue help cmd regenerate ./internal/ci/gitlab # the "./" prefix is required """ - cmd: "cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.expected.txt" + cmd: "cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.actual.txt" exitCode: 0 output: "" }, { - doc: """ - # Sometimes the above command's regeneration of the YAML file doesn't get - # sync'd to disk before we git-diff it, below. Make sure that it does. - """ - cmd: "sync" + doc: "" + cmd: "diff ../12.expected.txt ../12.actual.txt" exitCode: 0 output: "" }, { doc: "" - cmd: "diff ../12.expected.txt ../12.actual.txt" + cmd: "cue cmd regenerate ./internal/ci/gitlab # the \"./\" prefix is required" + exitCode: 0 + output: "" + }, { + doc: """ + # 2 commands not present in CUE-By-Example guide, added as an attempt to work + # around cue-lang/cue#3492. DELETE THESE COMMANDS! + """ + cmd: "sleep 1" exitCode: 0 output: "" }, { doc: "" - cmd: "cue cmd regenerate ./internal/ci/gitlab # the \"./\" prefix is required" + cmd: "sync" exitCode: 0 output: "" }, { @@ -186,9 +196,25 @@ package site # the diff command is given a '--' separator. I'm utterly stumped, but let's # just give it what it wants! """ - cmd: "git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\\.\\.[0-9a-f]{7}' | head -10 >../14.actual.txt" + cmd: "git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\\.\\.[0-9a-f]{7}' | head -9 >../14.actual.txt" exitCode: 0 output: "" + }, { + doc: "" + cmd: "diff --side ../14.expected.txt ../14.actual.txt" + exitCode: 0 + output: """ + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml\t\t\tdiff --git a/.gitlab-ci.yml b/.gitlab-ci.yml + --- a/.gitlab-ci.yml\t\t\t\t\t\t--- a/.gitlab-ci.yml + +++ b/.gitlab-ci.yml\t\t\t\t\t\t+++ b/.gitlab-ci.yml + @@ -1,3 +1,5 @@\t\t\t\t\t\t\t@@ -1,3 +1,5 @@ + +# Code generated by internal/ci/gitlab/ci_tool.cue; DO NOT E\t+# Code generated by internal/ci/gitlab/ci_tool.cue; DO NOT E + +\t\t\t\t\t\t\t\t+ + stages:\t\t\t\t\t\t\t stages: + - sync\t\t\t\t\t\t\t - sync + - preflight\t\t\t\t\t\t\t - preflight + + """ }, { doc: "" cmd: "git add .gitlab-ci.yml internal/ci/gitlab/ cue.mod/module.cue" diff --git a/hugo/content/en/docs/draft/cbe-005_gitlab_ci/index.md b/hugo/content/en/docs/draft/cbe-005_gitlab_ci/index.md index 14be1b206b..71ae4aaae0 100644 --- a/hugo/content/en/docs/draft/cbe-005_gitlab_ci/index.md +++ b/hugo/content/en/docs/draft/cbe-005_gitlab_ci/index.md @@ -24,16 +24,18 @@ know you're managing your pipelines with CUE. ## Prerequisites -- You have [`cue` installed](https://cuelang.org/docs/install/). - You have a GitLab pipeline file. - - The example shown throughout this guide uses the state of a specific commit - from the - [Flockademic repository](https://gitlab.com/Flockademic/Flockademic/-/blob/8efcea927b10c2773790fe78bb858905a75cf3ef/.gitlab-ci.yml) - on gitlab.com, as linked from - [GitLab's documentation pages](https://docs.gitlab.com/ee/ci/examples/end_to_end_testing_webdriverio/index.html), - but you don't need to use that repository in any way.\ - It is used here as it represents a reasonably complex example of a GitLab + - The example shown throughout this guide uses the pipeline file from a + specific commit in the + [`gitlab-org/gitlab` repository](https://gitlab.com/gitlab-org/gitlab/-/blob/3308936efcd70839cc61e0545dcb780756e4ec28/.gitlab-ci.yml) + on gitlab.com, as linked from GitLab's + [CI documentation pages](https://docs.gitlab.com/ee/ci/yaml/), + but **you don't need to use that repository in any way**. It's used as the + example in this guide only because it's a reasonably complex GitLab pipeline file. +- You have [`cue` installed](https://cuelang.org/docs/install/). + - You must have version `v0.11.0-alpha.4` or later installed. Using an + earlier version will cause certain commands in this guide to fail. - You have [`git` installed](https://git-scm.com/downloads). - You have [`curl` installed](https://curl.se/dlwiz/), or can fetch a remote file some other way. @@ -49,8 +51,8 @@ pipeline file, and ensure you start this process with a clean git state, with no modified files. For example: :computer: `terminal` -```text { title="TERMINAL" type="terminal" codeToCopy="Y2QgRmxvY2thZGVtaWMgIyBvdXIgZXhhbXBsZSByZXBvc2l0b3J5CmdpdCBzdGF0dXMgIyBzaG91bGQgcmVwb3J0ICJ3b3JraW5nIHRyZWUgY2xlYW4i" } -$ cd Flockademic # our example repository +```text { title="TERMINAL" type="terminal" codeToCopy="Y2QgZ2l0bGFiICMgb3VyIGV4YW1wbGUgcmVwb3NpdG9yeQpnaXQgc3RhdHVzICMgc2hvdWxkIHJlcG9ydCAid29ya2luZyB0cmVlIGNsZWFuIg==" } +$ cd gitlab # our example repository $ git status # should report "working tree clean" On branch master nothing to commit, working tree clean @@ -62,8 +64,8 @@ Initialise a CUE module named after the organisation and repository you're working with, but containing only lowercase letters and numbers. For example: :computer: `terminal` -```text { title="TERMINAL" type="terminal" codeToCopy="Y3VlIG1vZCBpbml0IGdpdGxhYi5jb20vZmxvY2thZGVtaWMvZmxvY2thZGVtaWM=" } -$ cue mod init gitlab.com/flockademic/flockademic +```text { title="TERMINAL" type="terminal" codeToCopy="Y3VlIG1vZCBpbml0IGdpdGxhYi5jb20vZ2l0bGFiLW9yZy9naXRsYWI=" } +$ cue mod init gitlab.com/gitlab-org/gitlab ``` #### :arrow_right: Import YAML pipeline @@ -111,12 +113,12 @@ The output should reflect your pipeline. In our example: package gitlab pipelines: ".gitlab-ci": { - image: "node:8.10" stages: [ + "sync", + "preflight", "prepare", - "test", - "build-backend", - "deploy-backend", + "build-images", + "fixtures", {{< /code-tab >}}{{< /code-tabs >}} #### :arrow_right: Store CUE pipelines in a dedicated directory @@ -147,8 +149,8 @@ Fetch a schema for GitLab pipelines, as defined by the GitLab project, and place it in the `internal/ci/gitlab` directory: :computer: `terminal` -```text { title="TERMINAL" type="terminal" codeToCopy="Y3VybCAtc1NvIGludGVybmFsL2NpL2dpdGxhYi9naXRsYWIuY2ljZC5waXBlbGluZS5zY2hlbWEuanNvbiBodHRwczovL2dpdGxhYi5jb20vZ2l0bGFiLW9yZy9naXRsYWIvLS9yYXcvN2FhNjE3MGM0YzgxYTk4ZjM3MmQ3YzUyZjM5MTg4NThjNGI2OWNjYS9hcHAvYXNzZXRzL2phdmFzY3JpcHRzL2VkaXRvci9zY2hlbWEvY2kuanNvbg==" } -$ curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/7aa6170c4c81a98f372d7c52f3918858c4b69cca/app/assets/javascripts/editor/schema/ci.json +```text { title="TERMINAL" type="terminal" codeToCopy="Y3VybCAtc1NvIGludGVybmFsL2NpL2dpdGxhYi9naXRsYWIuY2ljZC5waXBlbGluZS5zY2hlbWEuanNvbiBodHRwczovL2dpdGxhYi5jb20vZ2l0bGFiLW9yZy9naXRsYWIvLS9yYXcvMjc3YzlmNmI2NDNjOTJkMDAxMDFhY2EwZjJiNGI4NzRhMTQ0ZjdjNS9hcHAvYXNzZXRzL2phdmFzY3JpcHRzL2VkaXRvci9zY2hlbWEvY2kuanNvbg==" } +$ curl -sSo internal/ci/gitlab/gitlab.cicd.pipeline.schema.json https://gitlab.com/gitlab-org/gitlab/-/raw/277c9f6b643c92d00101aca0f2b4b874a144f7c5/app/assets/javascripts/editor/schema/ci.json ``` We use a specific commit from the upstream repository to make sure that this @@ -179,13 +181,27 @@ Create the file in the `internal/ci/gitlab/` directory and add this CUE: :floppy_disk: `internal/ci/gitlab/pipelines.cue` {{< code-tabs >}} -{{< code-tab name="Flockademic/internal/ci/gitlab/pipelines.cue" language="cue" area="top-left" >}} +{{< code-tab name="gitlab/internal/ci/gitlab/pipelines.cue" language="cue" area="top-left" >}} package gitlab // each member of the pipelines struct must be a valid #Pipeline -pipeline: [_]: #Pipeline +pipelines: [_]: #Pipeline {{< /code-tab >}}{{< /code-tabs >}} +#### :arrow_right: Validate your pipelines + +:computer: `terminal` +```text { title="TERMINAL" type="terminal" codeToCopy="Y3VlIHZldCAuL2ludGVybmFsL2NpL2dpdGxhYg==" } +$ cue vet ./internal/ci/gitlab +``` + +If this command fails and produces any output, then CUE believes that at least +one of your pipelines isn't valid. You'll need to resolve this before +continuing, by updating your pipelines inside your new CUE files. If you're +having difficulty fixing them, please come and ask for help in the friendly CUE +[Slack workspace](https://cuelang.org/s/slack) or +[Discord server](https://cuelang.org/s/discord)! + ### Generate YAML from CUE #### :arrow_right: Create a CUE workflow command @@ -195,7 +211,7 @@ Adapt the element commented with `TODO`: :floppy_disk: `internal/ci/gitlab/ci_tool.cue` {{< code-tabs >}} -{{< code-tab name="Flockademic/internal/ci/gitlab/ci_tool.cue" language="cue" area="top-left" >}} +{{< code-tab name="gitlab/internal/ci/gitlab/ci_tool.cue" language="cue" area="top-left" >}} package gitlab import ( @@ -244,22 +260,18 @@ workflow command is available **from a shell sitting at the repository root**. F example: :computer: `terminal` -```text { title="TERMINAL" type="terminal" codeToCopy="Y2QgJChnaXQgcmV2LXBhcnNlIC0tc2hvdy10b3BsZXZlbCkgIyBtYWtlIHN1cmUgd2UncmUgc2l0dGluZyBhdCB0aGUgcmVwb3NpdG9yeSByb290CmN1ZSBoZWxwIGNtZCByZWdlbmVyYXRlIC4vaW50ZXJuYWwvY2kvZ2l0bGFiIHwgaGVhZCAtNCA+Li4vMTIuZXhwZWN0ZWQudHh0CnN5bmM=" } +```text { title="TERMINAL" type="terminal" codeToCopy="Y2QgJChnaXQgcmV2LXBhcnNlIC0tc2hvdy10b3BsZXZlbCkgIyBtYWtlIHN1cmUgd2UncmUgc2l0dGluZyBhdCB0aGUgcmVwb3NpdG9yeSByb290CmN1ZSBoZWxwIGNtZCByZWdlbmVyYXRlIC4vaW50ZXJuYWwvY2kvZ2l0bGFiIHwgaGVhZCAtNCA+Li4vMTIuYWN0dWFsLnR4dA==" } $ cd $(git rev-parse --show-toplevel) # make sure we're sitting at the repository root # Actual command in CUE-By-Example guide: # cue help cmd regenerate ./internal/ci/gitlab # the "./" prefix is required -$ cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.expected.txt - -# Sometimes the above command's regeneration of the YAML file doesn't get -# sync'd to disk before we git-diff it, below. Make sure that it does. -$ sync +$ cue help cmd regenerate ./internal/ci/gitlab | head -4 >../12.actual.txt ``` The output of the `cue help` command **must** begin with the following: {{< code-tabs >}} -{{< code-tab name="12.actual.txt" language="txt" area="top-left" >}} +{{< code-tab name="12.expected.txt" language="txt" area="top-left" >}} Regenerate pipeline files Usage: @@ -285,13 +297,18 @@ Check that your YAML pipeline file has a single *material* change from the original: :computer: `terminal` -```text { title="TERMINAL" type="terminal" codeToCopy="Z2l0IGRpZmYgLS0gLmdpdGxhYi1jaS55bWwgfCBncmVwIC12RSAnXmluZGV4IFswLTlhLWZdezd9XC5cLlswLTlhLWZdezd9JyB8IGhlYWQgLTEwID4uLi8xNC5hY3R1YWwudHh0" } +```text { title="TERMINAL" type="terminal" codeToCopy="c2xlZXAgMQpzeW5jCmdpdCBkaWZmIC0tIC5naXRsYWItY2kueW1sIHwgZ3JlcCAtdkUgJ15pbmRleCBbMC05YS1mXXs3fVwuXC5bMC05YS1mXXs3fScgfCBoZWFkIC05ID4uLi8xNC5hY3R1YWwudHh0" } +# 2 commands not present in CUE-By-Example guide, added as an attempt to work +# around cue-lang/cue#3492. DELETE THESE COMMANDS! +$ sleep 1 +$ sync + # Actual command in CUE-By-Example guide: # git diff .gitlab-ci.yml # For some unknown reason the trailing '>../...' redirection *only* works when # the diff command is given a '--' separator. I'm utterly stumped, but let's # just give it what it wants! -$ git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\.\.[0-9a-f]{7}' | head -10 >../14.actual.txt +$ git diff -- .gitlab-ci.yml | grep -vE '^index [0-9a-f]{7}\.\.[0-9a-f]{7}' | head -9 >../14.actual.txt ``` Your output should look similar to the following example: @@ -301,13 +318,12 @@ Your output should look similar to the following example: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml -@@ -1,5 +1,6 @@ --image: node:8.10 +@@ -1,3 +1,5 @@ +# Code generated by internal/ci/gitlab/ci_tool.cue; DO NOT EDIT. - -+image: node:8.10 ++ stages: - - prepare + - sync + - preflight {{< /code-tab >}}{{< /code-tabs >}} The main change in each YAML file is the addition of a header that warns the reader not to edit the file directly.