Skip to content

Commit 40977cf

Browse files
polomarcuspolomarcus
committed
merge
2 parents 38668f0 + e7eed78 commit 40977cf

33 files changed

+7316
-3779
lines changed

‎.github/workflows/deploy-main.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111

1212

1313
env:
14-
PYTHON_VERSION: '3.11'
14+
PYTHON_VERSION: '3.12'
1515
POETRY_VERSION: '1.8.3'
1616

1717
jobs:
@@ -61,7 +61,7 @@ jobs:
6161
run: docker push ${{ secrets.CONTAINER_REGISTRY_ENDPOINT }}/ingest_to_db
6262

6363
- name: update scaleway job definition with version mediatree_import
64-
uses: jawher/action-scw@v2.32.1
64+
uses: jawher/action-scw@v2.34.0
6565
env:
6666
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
6767
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}

‎.github/workflows/scaleway-down.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: ubuntu-latest
1212
steps:
1313
- name: Use CLI
14-
uses: jawher/action-scw@v2.32.1
14+
uses: jawher/action-scw@v2.34.0
1515
env:
1616
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
1717
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
@@ -25,7 +25,7 @@ jobs:
2525

2626

2727
- name: 0 instances
28-
uses: jawher/action-scw@v2.32.1
28+
uses: jawher/action-scw@v2.34.0
2929
env:
3030
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
3131
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}

‎.github/workflows/scaleway-start-import-job-update.yml

+23-3
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,35 @@ jobs:
88
start-job-image:
99
strategy:
1010
matrix:
11-
start_date: ["2023-04-01", "2023-05-01","2023-06-01","2023-07-01"]
11+
dates: [
12+
{start_date: "2023-04-01", end_date: "2023-05-01"}
13+
,{start_date: "2023-05-01", end_date: "2023-06-01"}
14+
,{start_date: "2023-06-01", end_date: "2023-07-01"}
15+
,{start_date: "2023-07-01", end_date: "2023-08-01"}
16+
,{start_date: "2023-08-01", end_date: "2023-09-01"}
17+
,{start_date: "2023-09-01", end_date: "2023-10-01"}
18+
,{start_date: "2023-10-01", end_date: "2023-11-01"}
19+
,{start_date: "2023-11-01", end_date: "2023-12-01"}
20+
,{start_date: "2023-12-01", end_date: "2024-01-01"}
21+
,{start_date: "2024-01-01", end_date: "2024-02-01"}
22+
,{start_date: "2024-02-01", end_date: "2024-03-01"}
23+
,{start_date: "2024-03-01", end_date: "2024-04-01"}
24+
,{start_date: "2024-04-01", end_date: "2024-05-01"}
25+
,{start_date: "2024-05-01", end_date: "2024-06-01"}
26+
,{start_date: "2024-06-01", end_date: "2024-07-01"}
27+
,{start_date: "2024-07-01", end_date: "2024-08-01"}
28+
,{start_date: "2024-08-01", end_date: "2024-09-01"}
29+
,{start_date: "2024-09-01", end_date: "2024-10-01"}
30+
,{start_date: "2024-10-01", end_date: "2024-11-01"}
31+
]
1232
runs-on: ubuntu-latest
1333
steps:
1434
- name: start import job to reapply logic to all elements start_date matrix
15-
uses: jawher/action-scw@v2.32.1
35+
uses: jawher/action-scw@v2.34.0
1636
env:
1737
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
1838
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
1939
SCW_ORGANIZATION_ID: ${{ secrets.SCW_ORGANIZATION_ID }}
2040
SCW_ZONE: ${{ secrets.SCW_ZONE }}
2141
with:
22-
args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_DATE_UPDATE=${{ matrix.start_date }}
42+
args: jobs definition start ${{ secrets.SCALEWAY_JOB_IMPORT_ID }} environment-variables.UPDATE=true environment-variables.START_DATE_UPDATE=${{ matrix.dates.start_date }} environment-variables.END_DATE=${{ matrix.dates.end_date }}

‎.github/workflows/scaleway-up.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: ubuntu-latest
1212
steps:
1313
- name: Use CLI
14-
uses: jawher/action-scw@v2.32.1
14+
uses: jawher/action-scw@v2.34.0
1515
env:
1616
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
1717
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}
@@ -24,7 +24,7 @@ jobs:
2424
run: echo "CONTAINER_ID=$(cat "${GITHUB_WORKSPACE}/scw.output" | jq -r '.[0].id')" >> $GITHUB_ENV
2525

2626
- name: start 1 instances
27-
uses: jawher/action-scw@v2.32.1
27+
uses: jawher/action-scw@v2.34.0
2828
env:
2929
SCW_ACCESS_KEY: ${{ secrets.SCW_ACCESS_KEY }}
3030
SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }}

‎.github/workflows/test.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
push:
55

66
env:
7-
PYTHON_VERSION: '3.11'
7+
PYTHON_VERSION: '3.12'
88
POETRY_VERSION: '1.8.3'
99

1010
jobs:

‎Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#from https://medium.com/@albertazzir/blazing-fast-python-docker-builds-with-poetry-a78a66f5aed0
2-
FROM python:3.11.9 as builder
2+
FROM python:3.12.7 as builder
33

44
ENV VIRTUAL_ENV=/app/.venv
55

@@ -17,7 +17,7 @@ RUN pip install poetry==1.8.3
1717
RUN poetry install
1818

1919
# The runtime image, used to just run the code provided its virtual environment
20-
FROM python:3.11.9-slim as runtime
20+
FROM python:3.12.7-slim as runtime
2121

2222
WORKDIR /app
2323

‎Dockerfile_api_import

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#from https://medium.com/@albertazzir/blazing-fast-python-docker-builds-with-poetry-a78a66f5aed0
2-
FROM python:3.11.9 as builder
2+
FROM python:3.12.7 as builder
33

44
ENV VIRTUAL_ENV=/app/.venv
55

@@ -17,7 +17,7 @@ RUN pip install poetry==1.8.3
1717
RUN poetry install
1818

1919
# The runtime image, used to just run the code provided its virtual environment
20-
FROM python:3.11.9-slim as runtime
20+
FROM python:3.12.7-slim as runtime
2121

2222
WORKDIR /app
2323

‎README.md

+13-4
Original file line numberDiff line numberDiff line change
@@ -313,10 +313,14 @@ We can adjust batch update with these env variables (as in the docker-compose.ym
313313
```
314314
BATCH_SIZE: 50000 # number of records to update in one batch
315315
```
316+
### Update only one channel
317+
Use env variable `CHANNEL` like in docker compose (string: tf1) with `UPDATE` to true
316318

317319
### Batch program data
318320
`UPDATE_PROGRAM_ONLY` to true will only update program metadata, otherwise, it will update program metadata and all theme/keywords calculations.
319321

322+
`UPDATE_PROGRAM_CHANNEL_EMPTY_ONLY` to true will only update program metadata with empty value : "".
323+
320324
### Batch update from an offset
321325
With +1 millions rows, we can update from an offset to fix a custom logic by using `START_DATE_UPDATE` (YYYY-MM-DD), the default will use the end of the month otherwise you can specify`END_DATE` (optional) (YYYY-MM-DD) to batch update PG from a date range.
322326

@@ -335,10 +339,10 @@ Using [Alembic](https://alembic.sqlalchemy.org/en/latest/autogenerate.html) Auto
335339
# If changes have already been applied (on your feature vranch) and you have to recreate your alembic file by doing :
336340
# 1. change to your main branch
337341
git switch main
338-
# 2. start test container and run "pytest -vv -k api" to rebuild the state of the DB (or drop table the table you want)
342+
# 2. start test container (docker compose up testconsole -d / docker compose exec testconsole bash) and run "pytest -vv -k api" to rebuild the state of the DB (or drop table the table you want) - just let it run a few seconds.
339343
# 3. rechange to your WIP branch
340344
git switch -
341-
# 4. connect to the test container : docker compose up test -d / docker compose exec test bash
345+
# 4. connect to the test container : docker compose up testconsole -d / docker compose exec testconsole bash
342346
# 5. reapply the latest saved state :
343347
poetry run alembic stamp head
344348
# 6. Save the new columns
@@ -366,14 +370,18 @@ poetry run python3 quotaclimat/transform_excel_to_json.py
366370
```
367371

368372
## Program Metadata table
369-
The media perimeter is defined here : "quotaclimat/data_processing/mediatree/channel_program.json"
373+
The media perimeter is defined here : "quotaclimat/data_processing/mediatree/channel_program_data.py"
374+
375+
To evolve the media perimeter, we use `program_grid_start` and `program_grid_end` columns to version all evolutions.
370376

371-
To calculate the right total duration for each channel, after updating "quotaclimat/data_processing/mediatree/channel_program.json" you need to execute this command to update `postgres/program_metadata.json`
377+
To calculate the right total duration for each channel, after updating "quotaclimat/data_processing/mediatree/channel_program_data.py" you need to execute this command to update `postgres/program_metadata.json`
372378
```
373379
poetry run python3 transform_program.py
374380
```
375381
The SQL queries are based on this file that generate the Program Metadata table.
376382

383+
Program data will not be updated to avoid lock concurrent issues when using `UPDATE=true` for keywords logic. Note: The default case will update them.
384+
377385
**With the docker-entrypoint.sh this command is done automatically, so for production uses, you will not have to run this command.**
378386

379387
## Production monitoring
@@ -391,3 +399,4 @@ There is a debt regarding the cleanest of the code right now. Let's just not mak
391399

392400
## Thanks
393401
* [Eleven-Strategy](https://www.welcometothejungle.com/fr/companies/eleven-strategy)
402+
* [Kevin Tessier](https://kevintessier.fr)

‎alembic/versions/30abfd828007_program_metadata.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
"""program metadata
2+
3+
Revision ID: 30abfd828007
4+
Revises: 43103d5b49c9
5+
Create Date: 2024-10-03 14:18:09.874225
6+
7+
"""
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
import sqlalchemy as sa
12+
13+
14+
# revision identifiers, used by Alembic.
15+
revision: str = '30abfd828007'
16+
down_revision: Union[str, None] = '43103d5b49c9'
17+
branch_labels: Union[str, Sequence[str], None] = None
18+
depends_on: Union[str, Sequence[str], None] = None
19+
20+
21+
def upgrade() -> None:
22+
# ### commands auto generated by Alembic - please adjust! ###
23+
op.add_column('program_metadata', sa.Column('program_grid_start', sa.DateTime(), nullable=True))
24+
op.add_column('program_metadata', sa.Column('program_grid_end', sa.DateTime(), nullable=True))
25+
# ### end Alembic commands ###
26+
27+
28+
def downgrade() -> None:
29+
# ### commands auto generated by Alembic - please adjust! ###
30+
op.drop_column('program_metadata', 'program_grid_end')
31+
op.drop_column('program_metadata', 'program_grid_start')
32+
# ### end Alembic commands ###

‎alembic/versions/43103d5b49c9_program_add_start_end_date_for_grid_.py

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
"""program: add start/end date for grid evolution
2+
3+
Revision ID: 43103d5b49c9
4+
Revises: af956a85658f
5+
Create Date: 2024-10-02 13:18:56.251135
6+
7+
"""
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
import sqlalchemy as sa
12+
13+
14+
# revision identifiers, used by Alembic.
15+
revision: str = '43103d5b49c9'
16+
down_revision: Union[str, None] = 'af956a85658f'
17+
branch_labels: Union[str, Sequence[str], None] = None
18+
depends_on: Union[str, Sequence[str], None] = None
19+
20+
21+
def upgrade() -> None:
22+
# ### commands auto generated by Alembic - please adjust! ###
23+
pass
24+
# ### end Alembic commands ###
25+
26+
27+
def downgrade() -> None:
28+
# ### commands auto generated by Alembic - please adjust! ###
29+
pass
30+
# ### end Alembic commands ###

‎docker-compose.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,8 @@ services:
152152
#START_DATE_UPDATE: "2024-02-01" # to batch update PG from a date
153153
#END_DATE: "2024-02-29" # optional - otherwise end of the month
154154
BATCH_SIZE: 100 # number of records to update in one batch
155-
# START_DATE: 1717227223 # to test batch import
156-
CHANNEL : france-info # to reimport only one channel
155+
# START_DATE: 1727610071 # to test batch import
156+
CHANNEL : fr3-idf # to reimport only one channel
157157
MEDIATREE_USER : /run/secrets/username_api
158158
MEDIATREE_PASSWORD: /run/secrets/pwd_api
159159
MEDIATREE_AUTH_URL: https://keywords.mediatree.fr/api/auth/token/

‎docker-entrypoint.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ poetry run alembic upgrade head
66

77

88
echo "update program metadata file"
9-
python transform_program.py
9+
poetry run python3 transform_program.py
1010
if [[ $? -eq 0 ]]; then
1111
echo "Command succeeded"
1212
else

0 commit comments

Comments
 (0)