Skip to content

Commit

Permalink
refacto: remove directes from column names
Browse files Browse the repository at this point in the history
  • Loading branch information
@polomarcus
polomarcus committed Nov 25, 2024
1 parent bb1f7f7 commit af704ff
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 95 deletions.
10 changes: 5 additions & 5 deletions postgres/schemas/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,16 @@ class Keywords(Base):
number_of_keywords_biodiversite = Column(Integer) # sum of all biodiversite counters without duplicate
number_of_keywords_ressources = Column(Integer) # sum of all ressources counters without duplicate
number_of_changement_climatique_constat_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_constat integer;
number_of_changement_climatique_causes_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_causes_directes integer;
number_of_changement_climatique_causes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_causes_directes integer;
number_of_changement_climatique_consequences_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_changement_climatique_consequences integer;
number_of_attenuation_climatique_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_attenuation_climatique_solutions_directes integer;
number_of_adaptation_climatique_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_adaptation_climatique_solutions_directes integer;
number_of_attenuation_climatique_solutions_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_attenuation_climatique_solutions_directes integer;
number_of_adaptation_climatique_solutions_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_adaptation_climatique_solutions_directes integer;
number_of_ressources_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_ressources_naturelles_concepts_generaux integer;
number_of_ressources_solutions_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_ressources_solutions integer;
number_of_biodiversite_concepts_generaux_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_concepts_generaux integer;
number_of_biodiversite_causes_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_causes_directes integer;
number_of_biodiversite_causes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_causes_directes integer;
number_of_biodiversite_consequences_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_consequences integer;
number_of_biodiversite_solutions_directes_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_solutions_directes integer;
number_of_biodiversite_solutions_no_hrfp= Column(Integer) # ALTER TABLE keywords ADD number_of_biodiversite_solutions_directes integer;

class Channel_Metadata(Base):
__tablename__ = channel_metadata_table
Expand Down
40 changes: 20 additions & 20 deletions quotaclimat/data_processing/mediatree/update_pg_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,16 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
,new_number_of_keywords_biodiversite \
,new_number_of_keywords_ressources \
,number_of_changement_climatique_constat_no_hrfp \
,number_of_changement_climatique_causes_directes_no_hrfp \
,number_of_changement_climatique_causes_no_hrfp \
,number_of_changement_climatique_consequences_no_hrfp \
,number_of_attenuation_climatique_solutions_directes_no_hrfp \
,number_of_adaptation_climatique_solutions_directes_no_hrfp \
,number_of_attenuation_climatique_solutions_no_hrfp \
,number_of_adaptation_climatique_solutions_no_hrfp \
,number_of_ressources_no_hrfp \
,number_of_ressources_solutions_no_hrfp \
,number_of_biodiversite_concepts_generaux_no_hrfp \
,number_of_biodiversite_causes_directes_no_hrfp \
,number_of_biodiversite_causes_no_hrfp \
,number_of_biodiversite_consequences_no_hrfp \
,number_of_biodiversite_solutions_directes_no_hrfp = get_themes_keywords_duration(plaintext, srt, start)
,number_of_biodiversite_solutions_no_hrfp = get_themes_keywords_duration(plaintext, srt, start)
except Exception as err:
logging.error(f"continuing loop but met error : {err}")
continue
Expand Down Expand Up @@ -99,16 +99,16 @@ def update_keywords(session: Session, batch_size: int = 50000, start_date : str
,new_number_of_keywords_biodiversite
,new_number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp
,number_of_changement_climatique_causes_directes_no_hrfp
,number_of_changement_climatique_causes_no_hrfp
,number_of_changement_climatique_consequences_no_hrfp
,number_of_attenuation_climatique_solutions_directes_no_hrfp
,number_of_adaptation_climatique_solutions_directes_no_hrfp
,number_of_attenuation_climatique_solutions_no_hrfp
,number_of_adaptation_climatique_solutions_no_hrfp
,number_of_ressources_no_hrfp
,number_of_ressources_solutions_no_hrfp
,number_of_biodiversite_concepts_generaux_no_hrfp
,number_of_biodiversite_causes_directes_no_hrfp
,number_of_biodiversite_causes_no_hrfp
,number_of_biodiversite_consequences_no_hrfp
,number_of_biodiversite_solutions_directes_no_hrfp
,number_of_biodiversite_solutions_no_hrfp
)
else: # Program only mode
logging.info(f"Updating program for keyword {keyword_id} - {channel_name} - original tz : {start}")
Expand Down Expand Up @@ -199,16 +199,16 @@ def update_keyword_row(session: Session,
,number_of_keywords_biodiversite: int
,number_of_keywords_ressources: int
,number_of_changement_climatique_constat_no_hrfp: int,
number_of_changement_climatique_causes_directes_no_hrfp: int,
number_of_changement_climatique_causes_no_hrfp: int,
number_of_changement_climatique_consequences_no_hrfp: int,
number_of_attenuation_climatique_solutions_directes_no_hrfp: int,
number_of_adaptation_climatique_solutions_directes_no_hrfp: int,
number_of_attenuation_climatique_solutions_no_hrfp: int,
number_of_adaptation_climatique_solutions_no_hrfp: int,
number_of_ressources_no_hrfp: int,
number_of_ressources_solutions_no_hrfp: int,
number_of_biodiversite_concepts_generaux_no_hrfp: int,
number_of_biodiversite_causes_directes_no_hrfp: int,
number_of_biodiversite_causes_no_hrfp: int,
number_of_biodiversite_consequences_no_hrfp: int,
number_of_biodiversite_solutions_directes_no_hrfp: int
number_of_biodiversite_solutions_no_hrfp: int
):
if matching_themes is not None:
session.query(Keywords).filter(Keywords.id == keyword_id).update(
Expand All @@ -232,16 +232,16 @@ def update_keyword_row(session: Session,
,Keywords.number_of_keywords_biodiversite: number_of_keywords_biodiversite
,Keywords.number_of_keywords_ressources: number_of_keywords_ressources
,Keywords.number_of_changement_climatique_constat_no_hrfp:number_of_changement_climatique_constat_no_hrfp ,
Keywords.number_of_changement_climatique_causes_directes_no_hrfp:number_of_changement_climatique_causes_directes_no_hrfp ,
Keywords.number_of_changement_climatique_causes_no_hrfp:number_of_changement_climatique_causes_no_hrfp ,
Keywords.number_of_changement_climatique_consequences_no_hrfp:number_of_changement_climatique_consequences_no_hrfp ,
Keywords.number_of_attenuation_climatique_solutions_directes_no_hrfp:number_of_attenuation_climatique_solutions_directes_no_hrfp ,
Keywords.number_of_adaptation_climatique_solutions_directes_no_hrfp:number_of_adaptation_climatique_solutions_directes_no_hrfp ,
Keywords.number_of_attenuation_climatique_solutions_no_hrfp:number_of_attenuation_climatique_solutions_no_hrfp ,
Keywords.number_of_adaptation_climatique_solutions_no_hrfp:number_of_adaptation_climatique_solutions_no_hrfp ,
Keywords.number_of_ressources_no_hrfp:number_of_ressources_no_hrfp,
Keywords.number_of_ressources_solutions_no_hrfp:number_of_ressources_solutions_no_hrfp ,
Keywords.number_of_biodiversite_concepts_generaux_no_hrfp:number_of_biodiversite_concepts_generaux_no_hrfp ,
Keywords.number_of_biodiversite_causes_directes_no_hrfp:number_of_biodiversite_causes_directes_no_hrfp ,
Keywords.number_of_biodiversite_causes_no_hrfp:number_of_biodiversite_causes_no_hrfp ,
Keywords.number_of_biodiversite_consequences_no_hrfp:number_of_biodiversite_consequences_no_hrfp ,
Keywords.number_of_biodiversite_solutions_directes_no_hrfp:number_of_biodiversite_solutions_directes_no_hrfp,
Keywords.number_of_biodiversite_solutions_no_hrfp:number_of_biodiversite_solutions_no_hrfp,
},
synchronize_session=False
)
Expand Down
40 changes: 20 additions & 20 deletions test/sitemap/test_detect_keywords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,16 +111,16 @@ def test_one_theme_get_themes_keywords_duration():
number_of_keywords_biodiversite,
number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp,
number_of_changement_climatique_causes_directes_no_hrfp,
number_of_changement_climatique_causes_no_hrfp,
number_of_changement_climatique_consequences_no_hrfp,
number_of_attenuation_climatique_solutions_directes_no_hrfp,
number_of_adaptation_climatique_solutions_directes_no_hrfp,
number_of_attenuation_climatique_solutions_no_hrfp,
number_of_adaptation_climatique_solutions_no_hrfp,
number_of_ressources_no_hrfp,
number_of_ressources_solutions_no_hrfp,
number_of_biodiversite_concepts_generaux_no_hrfp,
number_of_biodiversite_causes_directes_no_hrfp,
number_of_biodiversite_causes_no_hrfp,
number_of_biodiversite_consequences_no_hrfp,
number_of_biodiversite_solutions_directes_no_hrfp) = get_themes_keywords_duration(plaintext_climat, subtitles, start)
number_of_biodiversite_solutions_no_hrfp) = get_themes_keywords_duration(plaintext_climat, subtitles, start)
assert set(themes_output) == set(themes)
# assert compare_unordered_lists_of_dicts(keywords_output, keywords)

Expand Down Expand Up @@ -235,16 +235,16 @@ def test_long_sentence_theme_get_themes_keywords_duration():
number_of_keywords_biodiversite,
number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp,
number_of_changement_climatique_causes_directes_no_hrfp,
number_of_changement_climatique_causes_no_hrfp,
number_of_changement_climatique_consequences_no_hrfp,
number_of_attenuation_climatique_solutions_directes_no_hrfp,
number_of_adaptation_climatique_solutions_directes_no_hrfp,
number_of_attenuation_climatique_solutions_no_hrfp,
number_of_adaptation_climatique_solutions_no_hrfp,
number_of_ressources_no_hrfp,
number_of_ressources_solutions_no_hrfp,
number_of_biodiversite_concepts_generaux_no_hrfp,
number_of_biodiversite_causes_directes_no_hrfp,
number_of_biodiversite_causes_no_hrfp,
number_of_biodiversite_consequences_no_hrfp,
number_of_biodiversite_solutions_directes_no_hrfp) = get_themes_keywords_duration(plaintext_climat, subtitles, start)
number_of_biodiversite_solutions_no_hrfp) = get_themes_keywords_duration(plaintext_climat, subtitles, start)

assert set(themes_output) == set(themes)
# assert compare_unordered_lists_of_dicts(keywords_output, keywords)
Expand Down Expand Up @@ -307,16 +307,16 @@ def test_three_get_themes_keywords_duration():
number_of_keywords_biodiversite,
number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp,
number_of_changement_climatique_causes_directes_no_hrfp,
number_of_changement_climatique_causes_no_hrfp,
number_of_changement_climatique_consequences_no_hrfp,
number_of_attenuation_climatique_solutions_directes_no_hrfp,
number_of_adaptation_climatique_solutions_directes_no_hrfp,
number_of_attenuation_climatique_solutions_no_hrfp,
number_of_adaptation_climatique_solutions_no_hrfp,
number_of_ressources_no_hrfp,
number_of_ressources_solutions_no_hrfp,
number_of_biodiversite_concepts_generaux_no_hrfp,
number_of_biodiversite_causes_directes_no_hrfp,
number_of_biodiversite_causes_no_hrfp,
number_of_biodiversite_consequences_no_hrfp,
number_of_biodiversite_solutions_directes_no_hrfp) = get_themes_keywords_duration("record de température pizza adaptation au dérèglement climatique", subtitles, start)
number_of_biodiversite_solutions_no_hrfp) = get_themes_keywords_duration("record de température pizza adaptation au dérèglement climatique", subtitles, start)

assert set(themes_output)== themes
#assert keywords_output == keywords
Expand Down Expand Up @@ -377,16 +377,16 @@ def test_long_get_themes_keywords_duration():
number_of_keywords_biodiversite,
number_of_keywords_ressources
,number_of_changement_climatique_constat_no_hrfp,
number_of_changement_climatique_causes_directes_no_hrfp,
number_of_changement_climatique_causes_no_hrfp,
number_of_changement_climatique_consequences_no_hrfp,
number_of_attenuation_climatique_solutions_directes_no_hrfp,
number_of_adaptation_climatique_solutions_directes_no_hrfp,
number_of_attenuation_climatique_solutions_no_hrfp,
number_of_adaptation_climatique_solutions_no_hrfp,
number_of_ressources_no_hrfp,
number_of_ressources_solutions_no_hrfp,
number_of_biodiversite_concepts_generaux_no_hrfp,
number_of_biodiversite_causes_directes_no_hrfp,
number_of_biodiversite_causes_no_hrfp,
number_of_biodiversite_consequences_no_hrfp,
number_of_biodiversite_solutions_directes_no_hrfp) = get_themes_keywords_duration("il rencontre aussi une crise majeure de la pénurie de l' offre laetitia jaoude des barrages sauvages", subtitles, start)
number_of_biodiversite_solutions_no_hrfp) = get_themes_keywords_duration("il rencontre aussi une crise majeure de la pénurie de l' offre laetitia jaoude des barrages sauvages", subtitles, start)
assert set(themes_output) == set(themes)
# too hard to maintain
#assert compare_unordered_lists_of_dicts(keywords_output, keywords)
Expand Down
Loading

1 comment on commit af704ff

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
postgres
   insert_data.py43784%36–38, 56–58, 63
   insert_existing_data_example.py19384%25–27
postgres/schemas
   models.py1681193%137–144, 157, 159–160, 225–226, 240–241
quotaclimat/data_ingestion
   scrap_sitemap.py1341787%27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
   ingest_sitemap_in_db.py553733%21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
   scrap_description_article.py36392%19–20, 32
quotaclimat/data_processing/mediatree
   api_import.py21313338%44–48, 53–74, 78–81, 87, 90–132, 138–153, 158, 171–183, 187–193, 206–218, 221–225, 231, 269–270, 273–304, 307–309
   channel_program.py1625765%21–23, 34–36, 53–54, 57–59, 98–99, 108, 124, 175–216
   config.py15287%7, 16
   detect_keywords.py2481694%111–118, 126–127, 260, 330–337, 375
   update_pg_keywords.py674927%15–130, 154, 157, 164–179, 213–250, 257
   utils.py792568%29–53, 56, 65, 86–87, 117–120
quotaclimat/utils
   healthcheck_config.py291452%22–24, 27–38
   logger.py241154%22–24, 28–37
   sentry.py11282%22–23
TOTAL133038771% 

Tests Skipped Failures Errors Time
97 0 💤 0 ❌ 0 🔥 8m 24s ⏱️

Please sign in to comment.