Skip to content

Commit 46752a0

Browse files
patriciacatandipatriciacatandi
committed
merge master
2 parents 1b81097 + 883cd93 commit 46752a0

File tree

156 files changed

+13244
-3463
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

156 files changed

+13244
-3463
lines changed

‎.github/workflows/build_docker.yaml

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Build Docker image
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
paths:
8+
- ".github/workflows/cd.yaml"
9+
- "pipelines/**/*"
10+
- "pyproject.toml"
11+
- "Dockerfile"
12+
pull_request:
13+
branches:
14+
- master
15+
paths:
16+
- ".github/workflows/cd_staging.yaml"
17+
- "pipelines/**/*"
18+
- "pyproject.toml"
19+
- "Dockerfile"
20+
21+
env:
22+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
23+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
24+
GH_PAT: ${{ secrets.GH_PAT }}
25+
IMAGE_NAME: ghcr.io/prefeitura-rio/prefect-flows
26+
27+
jobs:
28+
build-container:
29+
name: Build Docker image
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v2
33+
34+
- name: Setup Google Cloud CLI
35+
uses: google-github-actions/[email protected]
36+
with:
37+
service_account_key: ${{ secrets.GCP_SA_KEY }}
38+
project_id: ${{ secrets.GCP_PROJECT_ID }}
39+
export_default_credentials: true
40+
41+
- name: Build and publish Prefect Agent image
42+
run: |
43+
docker build -t $IMAGE_NAME:$GITHUB_SHA .
44+
docker build -t $IMAGE_NAME:latest .
45+
echo $GH_PAT | docker login ghcr.io -u gabriel-milan --password-stdin
46+
docker push $IMAGE_NAME:$GITHUB_SHA
47+
docker push $IMAGE_NAME:latest

‎.github/workflows/cd.yaml

+29-24
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: CD
1+
name: Register flows (production)
22

33
on:
44
push:
@@ -17,14 +17,15 @@ env:
1717
GKE_CLUSTER: ${{ secrets.GKE_CLUSTER_NAME }}
1818
GKE_ZONE: ${{ secrets.GKE_ZONE }}
1919
IMAGE_NAME: ghcr.io/prefeitura-rio/prefect-flows
20-
PREFECT__BACKEND: ${{ secrets.PREFECT__BACKEND }}
21-
PREFECT__SERVER__HOST: ${{ secrets.PREFECT__SERVER__HOST }}
22-
PREFECT__SERVER__PORT: ${{ secrets.PREFECT__SERVER__PORT }}
20+
PREFECT__BACKEND: cloud
21+
PREFECT__CLOUD__API: https://prefect.dados.rio/api
22+
PREFECT__CLOUD__PORT: 443
2323
PREFECT__SERVER__PROJECT: ${{ secrets.PREFECT__SERVER__PROJECT }}
24+
PREFECT_AUTH_TOML: ${{ secrets.PREFECT_AUTH_TOML }}
2425

2526
jobs:
2627
build-container:
27-
name: Build, publish and register
28+
name: Register flows (production)
2829
runs-on: ubuntu-latest
2930
steps:
3031
- uses: actions/checkout@v2
@@ -53,30 +54,34 @@ jobs:
5354
run: |-
5455
pip install .
5556
56-
- name: Get GKE credentials
57-
uses: google-github-actions/[email protected]
58-
with:
59-
cluster_name: ${{ env.GKE_CLUSTER }}
60-
location: ${{ env.GKE_ZONE }}
61-
credentials: ${{ secrets.GCP_SA_KEY }}
62-
63-
- name: Build and publish Prefect Agent image
64-
run: |
65-
docker build -t $IMAGE_NAME:$GITHUB_SHA .
66-
docker build -t $IMAGE_NAME:latest .
67-
echo $GH_PAT | docker login ghcr.io -u gabriel-milan --password-stdin
68-
docker push $IMAGE_NAME:$GITHUB_SHA
69-
docker push $IMAGE_NAME:latest
70-
7157
- name: Install GDAL for osgeo deps
7258
run: |
73-
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://sourceforge.net/projects/gdal-wheels-for-linux/files/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl/download
59+
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://prefeitura-rio.github.io/storage/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
7460
python -m pip install --no-cache-dir ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
7561
76-
- name: Forward port to Apollo
62+
- name: Get changed files for code tree analysis
63+
id: files
64+
uses: Ana06/[email protected]
65+
66+
- name: Perform code tree analysis
67+
id: code-tree-analysis
68+
continue-on-error: true
69+
run: |
70+
python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}" --write-to-file
71+
72+
- name: Write auth.toml
7773
run: |-
78-
bash -c "kubectl port-forward -n prefect deploy/prefect-apollo 4200:4200 &" && sleep 5
74+
mkdir -p $HOME/.prefect
75+
echo $PREFECT_AUTH_TOML | base64 --decode > $HOME/.prefect/auth.toml
76+
77+
- name: Wait for Docker image to be available
78+
uses: lewagon/[email protected]
79+
with:
80+
ref: ${{ github.ref }}
81+
check-name: 'Build Docker image'
82+
repo-token: ${{ secrets.GITHUB_TOKEN }}
83+
verbose: true
7984

8085
- name: Register Prefect flows
8186
run: |-
82-
python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/
87+
python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --schedule --filter-affected-flows

‎.github/workflows/cd_staging.yaml

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Register flows (staging)
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- master
7+
paths:
8+
- ".github/workflows/cd_staging.yaml"
9+
- "pipelines/**/*"
10+
- "pyproject.toml"
11+
- "Dockerfile"
12+
13+
env:
14+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
15+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
16+
GH_PAT: ${{ secrets.GH_PAT }}
17+
GKE_CLUSTER: ${{ secrets.GKE_CLUSTER_NAME }}
18+
GKE_ZONE: ${{ secrets.GKE_ZONE }}
19+
IMAGE_NAME: ghcr.io/prefeitura-rio/prefect-flows
20+
PREFECT__BACKEND: cloud
21+
PREFECT__CLOUD__API: https://prefect.dados.rio/api
22+
PREFECT__CLOUD__PORT: 443
23+
PREFECT__SERVER__PROJECT: ${{ secrets.PREFECT__SERVER__PROJECT__STAGING }}
24+
PREFECT_AUTH_TOML: ${{ secrets.PREFECT_AUTH_TOML }}
25+
26+
jobs:
27+
build-container:
28+
if: startsWith(github.head_ref, 'staging/')
29+
name: Register flows (staging)
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@v2
33+
34+
- name: Setup Python version
35+
uses: actions/setup-python@v2
36+
with:
37+
python-version: "3.9"
38+
39+
- name: Setup Google Cloud CLI
40+
uses: google-github-actions/[email protected]
41+
with:
42+
service_account_key: ${{ secrets.GCP_SA_KEY }}
43+
project_id: ${{ secrets.GCP_PROJECT_ID }}
44+
export_default_credentials: true
45+
46+
- name: Install Python dependencies for deploying
47+
run: |-
48+
pip install -r requirements-deploy.txt
49+
50+
- name: Update image tag in constants
51+
run: |-
52+
python .github/workflows/scripts/replace_docker_tag.py $IMAGE_NAME $GITHUB_SHA
53+
54+
- name: Install package locally
55+
run: |-
56+
pip install .
57+
58+
- name: Install GDAL for osgeo deps
59+
run: |
60+
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://prefeitura-rio.github.io/storage/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
61+
python -m pip install --no-cache-dir ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
62+
63+
- name: Get changed files for code tree analysis
64+
id: files
65+
uses: Ana06/[email protected]
66+
67+
- name: Perform code tree analysis
68+
id: code-tree-analysis
69+
continue-on-error: true
70+
run: |
71+
python .github/workflows/scripts/code_tree_analysis.py "${{ steps.files.outputs.all }}" --write-to-file
72+
73+
- name: Write auth.toml
74+
run: |-
75+
mkdir -p $HOME/.prefect
76+
echo $PREFECT_AUTH_TOML | base64 --decode > $HOME/.prefect/auth.toml
77+
78+
- name: Wait for Docker image to be available
79+
uses: lewagon/[email protected]
80+
with:
81+
ref: ${{ github.event.pull_request.head.sha || github.sha }}
82+
check-name: 'Build Docker image'
83+
repo-token: ${{ secrets.GITHUB_TOKEN }}
84+
verbose: true
85+
86+
- name: Register Prefect flows
87+
run: |-
88+
python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --no-schedule --filter-affected-flows

‎.github/workflows/ci.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
pip install --prefer-binary -r requirements-test.txt
2828
- name: Install GDAL for osgeo deps
2929
run: |
30-
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://sourceforge.net/projects/gdal-wheels-for-linux/files/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl/download
30+
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://prefeitura-rio.github.io/storage/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
3131
python -m pip install --no-cache-dir ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
3232
- name: Install flows
3333
run: |

‎.github/workflows/codeql-analysis.yml

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# For most projects, this workflow file will not need changing; you simply need
2+
# to commit it to your repository.
3+
#
4+
# You may wish to alter this file to override the set of languages analyzed,
5+
# or to provide custom queries or build logic.
6+
#
7+
# ******** NOTE ********
8+
# We have attempted to detect the languages in your repository. Please check
9+
# the `language` matrix defined below to confirm you have the correct set of
10+
# supported CodeQL languages.
11+
#
12+
name: "CodeQL"
13+
14+
on:
15+
push:
16+
branches: [ "master" ]
17+
pull_request:
18+
# The branches below must be a subset of the branches above
19+
branches: [ "master" ]
20+
schedule:
21+
- cron: '0 3 * * 6'
22+
23+
jobs:
24+
analyze:
25+
name: Analyze
26+
runs-on: ubuntu-latest
27+
permissions:
28+
actions: read
29+
contents: read
30+
security-events: write
31+
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
language: [ 'python' ]
36+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
37+
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
38+
39+
steps:
40+
- name: Checkout repository
41+
uses: actions/checkout@v3
42+
43+
# Initializes the CodeQL tools for scanning.
44+
- name: Initialize CodeQL
45+
uses: github/codeql-action/init@v2
46+
with:
47+
languages: ${{ matrix.language }}
48+
# If you wish to specify custom queries, you can do so here or in a config file.
49+
# By default, queries listed here will override any specified in a config file.
50+
# Prefix the list here with "+" to use these queries and those in the config file.
51+
52+
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
53+
# queries: security-extended,security-and-quality
54+
55+
56+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
57+
# If this step fails, then you should remove it and run the build manually (see below)
58+
- name: Autobuild
59+
uses: github/codeql-action/autobuild@v2
60+
61+
# ℹ️ Command-line programs to run using the OS shell.
62+
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
63+
64+
# If the Autobuild fails above, remove it and uncomment the following three lines.
65+
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
66+
67+
# - run: |
68+
# echo "Run, Build Application using script"
69+
# ./location_of_script_within_repo/buildscript.sh
70+
71+
- name: Perform CodeQL Analysis
72+
uses: github/codeql-action/analyze@v2

‎.github/workflows/docs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
2525
- name: Install GDAL for osgeo deps
2626
run: |
27-
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://sourceforge.net/projects/gdal-wheels-for-linux/files/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl/download
27+
curl -sSLo ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl https://prefeitura-rio.github.io/storage/GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
2828
python -m pip install --no-cache-dir ./GDAL-3.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl
2929
3030
- name: Install package and doc dependencies

‎.github/workflows/scripts/code_tree_analysis.py

+15-3
Original file line numberDiff line numberDiff line change
@@ -446,8 +446,11 @@ def identify_code_owners(files: List[str]) -> List[str]:
446446
if __name__ == "__main__":
447447

448448
# Assert arguments.
449-
if len(sys.argv) != 2:
450-
print(f"Usage: python {sys.argv[0]} <changed_files>")
449+
if len(sys.argv) not in [2, 3]:
450+
print(f"Usage: python {sys.argv[0]} <changed_files> [--write-to-file]")
451+
452+
# Write to file?
453+
write_to_file = "--write-to-file" in sys.argv
451454

452455
# Get modified files
453456
changed_files: List[str] = sys.argv[1].split(" ")
@@ -484,10 +487,18 @@ def identify_code_owners(files: List[str]) -> List[str]:
484487
dependent_files.update(graph.successors(declaration))
485488
if "pipelines/flows.py" in dependent_files:
486489
dependent_files.remove("pipelines/flows.py")
490+
for file_ in changed_files:
491+
dependent_files.add(file_)
487492
print("These files depend on the exported declarations:")
488493
for file_ in dependent_files:
489494
print(f"\t- {file_}")
490495

496+
# Write dependent file list to file.
497+
if write_to_file:
498+
with open("dependent_files.txt", "w") as f:
499+
for file_ in dependent_files:
500+
f.write(f"{file_}\n")
501+
491502
# Start a PR message
492503
message = "### Análise da árvore de código\n\n"
493504

@@ -524,7 +535,8 @@ def identify_code_owners(files: List[str]) -> List[str]:
524535
if len(dependent_files) == 0 and len(conflicts) == 0:
525536
message += "*Nenhum problema encontrado!*"
526537

527-
log(message)
538+
if not write_to_file:
539+
log(message)
528540

529541
# Raise if there are conflicts
530542
if len(conflicts) > 0:

0 commit comments

Comments
 (0)