-
Notifications
You must be signed in to change notification settings - Fork 135
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
moving infra files fixes: https://jira-dc.paloaltonetworks.com/browse/CIAC-7077
- Loading branch information
Showing
15 changed files
with
3,811 additions
and
130 deletions.
There are no files selected for viewing
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,4 +7,5 @@ _site | |
| artifacts | ||
| dockerfiles-trust | ||
| .python-version | ||
| .venv | ||
| .venv | ||
| /report.xml | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,7 @@ | ||
| variables: | ||
| GH_BRANCH: $CI_COMMIT_BRANCH | ||
| GH_COMMIT: $CI_COMMIT_SHA | ||
| trigger_job: | ||
| trigger: | ||
| strategy: depend | ||
| project: ${CI_PROJECT_NAMESPACE}/dockerfiles-cicd | ||
| CURRENT_BRANCH_NAME: $INFRA_BRANCH | ||
|
|
||
| include: | ||
| - file: "/.gitlab/ci/dockerfiles/.gitlab-ci.yml" | ||
| ref: $INFRA_BRANCH | ||
| project: "${CI_PROJECT_NAMESPACE}/infra" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| #!/bin/bash | ||
|
|
||
| CYAN="\e[0;36m" | ||
| CLEAR="\e[0m" | ||
| RED='\033[0;31m' | ||
| GREEN='\033[0;32m' | ||
| BLUE='\033[0;34m' | ||
| NC='\033[0m' | ||
|
|
||
| SECTION_START="\e[0Ksection_start:the_time:section_id\r\e[0K${CYAN}section_header${CLEAR}" | ||
| SECTION_END="\e[0Ksection_end:the_time:section_id\r\e[0K" | ||
|
|
||
| section_start() { | ||
| local section_header section_id start | ||
| start="$SECTION_START" | ||
| if [[ "$#" -eq 1 ]]; then | ||
| section_header="$1" | ||
| section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')" | ||
| elif [[ "$#" -eq 2 ]]; then | ||
| if [[ "$2" =~ -{0,2}collapsed ]]; then | ||
| start="${start/section_id/section_id[collapsed=true]}" | ||
| section_header="$1" | ||
| section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')" | ||
| else | ||
| section_header="$2" | ||
| section_id="$1" | ||
| fi | ||
| elif [[ "$#" -eq 3 && "$3" =~ /^-{0,2}collapsed$/ ]]; then | ||
| start="${start/section_id/section_id[collapsed=true]}" | ||
| section_header="$2" | ||
| section_id="$1" | ||
| else | ||
| echo "section_start should be called with 1-3 args but it was called with $# args" | ||
| echo "acceptable usages:" | ||
| echo " 1. section_start \"<section-header>\"" | ||
| echo " 2. section_start \"<section-start-id>\" \"<section-header>\"" | ||
| echo " 3. section_start \"<section-header>\" --collapse" | ||
| echo " 4. section_start \"<section-start-id>\" \"<section-header>\" --collapse" | ||
| echo "where <section-start-id> is only alphanumeric characters and underscore and" | ||
| echo "--collapse indicates that you would like those log steps to be collapsed in the job log output by default" | ||
| exit 9 | ||
| fi | ||
| start_time=$(date +%s) | ||
| start="$(echo "$start" | sed -e "s/the_time/$start_time/" -e "s/section_id/$section_id/" -e "s/section_header/$section_header/")" | ||
| echo -e "$start" | ||
| date +"[%Y-%m-%dT%H:%M:%S.%3N] section start" | ||
| } | ||
|
|
||
| section_end() { | ||
| local section_id end | ||
| date +"[%Y-%m-%dT%H:%M:%S.%3N] section end" | ||
| end="$SECTION_END" | ||
| if [[ "$#" -eq 1 ]]; then | ||
| section_id="$(echo "$1" | tr -c '[:alnum:]\n\r' '_')" | ||
| else | ||
| echo "section_end should be called with 1 arg but it was called with $# args" | ||
| echo "acceptable usage:" | ||
| echo " 1. section_end \"<section-start-id>\"" | ||
| echo " 2. section_end \"<section-header>\"" | ||
| echo "where <section-start-id> or <section-header> is the id of the section this marks the end of" | ||
| exit 9 | ||
| fi | ||
| end_time=$(date +%s) | ||
| end="$(echo "$end" | sed -e "s/the_time/$end_time/" -e "s/section_id/$section_id/")" | ||
| echo -e "$end" | ||
| } | ||
|
|
||
| job-done() { | ||
| mkdir -p "${PIPELINE_JOBS_FOLDER}" | ||
| echo "creating file ${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt" | ||
| echo "done" > "${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt" | ||
| echo "finished writing to file ${PIPELINE_JOBS_FOLDER}/${CI_JOB_NAME}.txt" | ||
| } | ||
|
|
||
| sleep-with-progress() { | ||
| local sleep_time=${1:-10} | ||
| local sleep_interval=${2:-1} | ||
| local sleep_message=${3:-"Sleeping... "} | ||
| local columns=${4:-$(tput cols)} | ||
| local sleep_step=$((sleep_time / sleep_interval)) | ||
| for ((i=0; i< sleep_step;i++)); do echo "${sleep_interval}";sleep "${sleep_interval}"; done | poetry run tqdm --total ${sleep_time} --unit seconds --leave --update --colour green -ncols ${columns} --desc "${sleep_message}" 1> /dev/null | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # | ||
| # Bootstraps a development environment. | ||
| # | ||
| # This includes: | ||
| # * install pre-commit hooks | ||
| # * setup infra dependencies with poetry | ||
|
|
||
| function exit_on_error { | ||
| if [ "${1}" -ne 0 ]; then | ||
| echo "ERROR: ${2}, exiting with code ${1}" 1>&2 | ||
| exit "${1}" | ||
| fi | ||
| } | ||
|
|
||
| # poetry is installed in ~/.local/bin | ||
| PATH=~/.local/bin:$PATH | ||
|
|
||
| if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then | ||
| cat << __EOF__ | ||
| Setup development environment (run with no arguments): | ||
| * install pre-commit hooks (set NO_HOOKS=1 to skip) | ||
| __EOF__ | ||
| exit 0 | ||
| fi | ||
|
|
||
| if [ ! "${PWD}" == "$(git rev-parse --show-toplevel)" ]; then | ||
| cat >&2 <<__EOF__ | ||
| ERROR: this script must be run at the root of the source tree | ||
| __EOF__ | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "=======================" | ||
| if [ -z "${INSTALL_POETRY}" ]; then | ||
| if ! command -v poetry >/dev/null 2>&1; then | ||
| echo "ERROR: poetry is missing. Please run the following command to install it: | ||
| curl -sSL https://install.python-poetry.org | python3 -" 1>&2 | ||
| exit 1 | ||
| fi | ||
| else | ||
| should_install_poetry="yes" | ||
| if command -v poetry >/dev/null 2>&1; then | ||
| if [[ "$(poetry --version)" == "Poetry (version ${POETRY_VERSION})" ]]; then | ||
| echo "Poetry is already installed with the correct version: $(poetry --version)" | ||
| should_install_poetry="no" | ||
| else | ||
| echo "Poetry is already installed with a different version: $(poetry --version), required version: ${POETRY_VERSION}" | ||
| fi | ||
| else | ||
| echo "Poetry isn't installed" | ||
| fi | ||
| if [[ "${should_install_poetry}" == "yes" ]]; then | ||
| echo "Installing Poetry version:${POETRY_VERSION}" | ||
| curl -sSL https://install.python-poetry.org | python3 - --version "${POETRY_VERSION}" | ||
| error_code=$? | ||
| if ! command -v poetry >/dev/null 2>&1; then | ||
| exit_on_error $? "Poetry isn't installed" | ||
| fi | ||
| if [[ "$(poetry --version)" == "Poetry (version ${POETRY_VERSION})" ]]; then | ||
| echo "Poetry version ${POETRY_VERSION} installed successfully" | ||
| else | ||
| exit_on_error 1 "Poetry version $(poetry --version) doesn't match the required version: ${POETRY_VERSION}" | ||
| fi | ||
| if [ -n "${ARTIFACTS_FOLDER}" ] && [ "${error_code}" -ne 0 ]; then | ||
| cp "${PWD}"/poetry-installer-error-*.log "${ARTIFACTS_FOLDER}" | ||
| fi | ||
| exit_on_error $error_code "Failed to install Poetry version:${POETRY_VERSION}" | ||
| fi | ||
| fi | ||
|
|
||
| if [ -n "${NO_HOOKS}" ]; then | ||
| echo "Skipping hooks setup as environment variable NO_HOOKS is set" | ||
| else | ||
| GIT_HOOKS_DIR="${PWD}/.git/hooks" | ||
| if [ ! -e "${GIT_HOOKS_DIR}/pre-commit" ]; then | ||
| echo "Installing 'pre-commit' hooks" | ||
| poetry run pre-commit install | ||
| exit_on_error $? "Failed to install pre-commit hook" | ||
| else | ||
| echo "Skipping install of pre-commit hook as it already exists." | ||
| echo "If you want to re-install: 'rm ${GIT_HOOKS_DIR}/pre-commit' and then run this script again." | ||
| exit 1 | ||
| fi | ||
| fi | ||
|
|
||
| if [ -n "${CI}" ]; then | ||
| echo "Detected CI environment" | ||
| echo "Checking whether poetry files are valid" | ||
| poetry check --no-interaction | ||
| exit_on_error $? "Failed to check poetry files" | ||
| echo "Installing dependencies..." | ||
| poetry install --no-interaction | ||
| exit_on_error $? "Failed to install dependencies" | ||
| else | ||
| echo "Detected local environment" | ||
| echo "Check if poetry files are valid" | ||
| poetry check | ||
| exit_on_error $? "Failed to check poetry files" | ||
| echo "Installing dependencies..." | ||
| poetry install | ||
| exit_on_error $? "Failed to install dependencies" | ||
| fi | ||
|
|
||
| echo "==========================" | ||
| echo "Done setting up virtualenv with poetry" | ||
| echo "Activate the virtualenv by running: poetry shell" | ||
| echo "Deactivate by running: deactivate" | ||
| echo "=======================" | ||
|
|
||
| echo "Finished setting up the environment." | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| repos: | ||
| - repo: https://github.com/charliermarsh/ruff-pre-commit | ||
| rev: v0.5.4 | ||
| hooks: | ||
| - id: ruff | ||
| args: | ||
| - --fix | ||
| - id: ruff-format | ||
| - repo: https://github.com/pappasam/toml-sort | ||
| rev: v0.23.1 | ||
| hooks: | ||
| - id: toml-sort-fix | ||
| exclude: poetry.lock | ||
| - repo: https://github.com/pre-commit/pre-commit-hooks | ||
| rev: v4.6.0 | ||
| hooks: | ||
| - id: check-json | ||
| - id: check-yaml | ||
| exclude: .gitlab | ||
| - id: check-ast | ||
| - id: check-merge-conflict | ||
| - id: name-tests-test | ||
| files: .+_test.py$ | ||
| - id: trailing-whitespace | ||
| - id: check-toml | ||
| - id: check-xml | ||
| - repo: https://github.com/python-poetry/poetry | ||
| rev: 1.8.3 | ||
| hooks: | ||
| - id: poetry-check | ||
| args: | ||
| - --lock | ||
| files: ^pyproject.toml$ | ||
| - repo: https://github.com/hadialqattan/pycln | ||
| rev: v2.4.0 | ||
| hooks: | ||
| - id: pycln | ||
| args: | ||
| - --all | ||
| - repo: https://github.com/pre-commit/mirrors-mypy | ||
| rev: v1.10.1 | ||
| hooks: | ||
| - id: mypy | ||
| entry: mypy | ||
| args: | ||
| - --show-error-codes | ||
| additional_dependencies: | ||
| - types-requests | ||
| - types-paramiko | ||
| - types-PyYAML | ||
| - types-setuptools | ||
| - types-ujson | ||
| - types-decorator | ||
| - types-retry | ||
| - types-pytz | ||
| - types-python-dateutil | ||
| - types-tabulate | ||
| - types-dateparser | ||
| types: [python] | ||
| #duplicate here and in pyproject.toml because https://github.com/python/mypy/issues/13916 | ||
| exclude: artifacts*|.*_test|test_.*|test_data|tests_data|.venv | ||
| - repo: https://github.com/koalaman/shellcheck-precommit | ||
| rev: v0.10.0 | ||
| hooks: | ||
| - id: shellcheck | ||
| args: ["--severity=error"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.