From 1590cdc5d8b674531b521ffc10e841530627dac3 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 8 Nov 2023 12:08:32 -0300 Subject: [PATCH 1/8] feat: add educacao_basica_frequencia --- .../dump_db_educacao_basica/schedules.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index 6edff8e17..ebb6e773d 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -229,8 +229,40 @@ "materialization_mode": "prod", "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", }, + "turma": { + "dump_mode": "overwrite", + "materialize_after_dump": True, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", + }, + "aca_turno": { + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + trn_id, + ent_id, + ttn_id, + trn_descricao, + trn_padrao, + trn_situacao, + trn_dataCriacao, + trn_dataAlteracao, + trn_controleTempo, + trn_horaInicio, + trn_horaFim + FROM GestaoEscolar.dbo.ACA_Turno + """, + "dataset_id": "educacao_basica_frequencia", + }, } + sme_clocks = generate_dump_db_schedules( interval=timedelta(days=1), start_date=datetime(2022, 1, 1, 2, 10, tzinfo=pytz.timezone("America/Sao_Paulo")), From a230f0dcd0b387a75bdf3d2d3a0815a196665e3a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:09:12 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../dump_db_educacao_basica/schedules.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index ebb6e773d..e93702144 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -244,17 +244,17 @@ "dump_to_gcs": False, "materialization_mode": "prod", "execute_query": """ - SELECT - trn_id, - ent_id, - ttn_id, - trn_descricao, - trn_padrao, - trn_situacao, - trn_dataCriacao, - trn_dataAlteracao, - trn_controleTempo, - trn_horaInicio, + SELECT + trn_id, + ent_id, + ttn_id, + trn_descricao, + trn_padrao, + trn_situacao, + trn_dataCriacao, + trn_dataAlteracao, + trn_controleTempo, + trn_horaInicio, trn_horaFim FROM GestaoEscolar.dbo.ACA_Turno """, From fd4eca6d4bdec73b0697676c8b9e012c6813fddd Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 8 Nov 2023 12:09:29 -0300 Subject: [PATCH 3/8] feat: add educacao_basica_frequencia --- pipelines/rj_sme/dump_db_educacao_basica/schedules.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index ebb6e773d..16bd32050 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -229,14 +229,6 @@ "materialization_mode": "prod", "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", }, - "turma": { - "dump_mode": "overwrite", - "materialize_after_dump": True, - "materialize_to_datario": False, - "dump_to_gcs": False, - "materialization_mode": "prod", - "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", - }, "aca_turno": { "dump_mode": "overwrite", "materialize_after_dump": False, From 653a0f870382b2341fef120094938bc2a48c5f90 Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 8 Nov 2023 16:43:43 -0300 Subject: [PATCH 4/8] feat: add educacao_basica_frequencia --- pipelines/rj_sme/dump_db_educacao_basica/schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index 730859f02..b89fe09f2 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -229,7 +229,7 @@ "materialization_mode": "prod", "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", }, - "aca_turno": { + "turno": { "dump_mode": "overwrite", "materialize_after_dump": False, "materialize_to_datario": False, From 0bdf324dd39ce1fc4c32bd68cb383e22ef64a4cd Mon Sep 17 00:00:00 2001 From: d116626 Date: Wed, 8 Nov 2023 16:48:34 -0300 Subject: [PATCH 5/8] feat: add educacao_basica_frequencia --- pipelines/rj_sme/dump_db_educacao_basica/schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index b89fe09f2..bc808764b 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -222,6 +222,7 @@ "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Movimentacao_lgpd", }, "turma": { + "dataset_id": "educacao_basica_frequencia", "dump_mode": "overwrite", "materialize_after_dump": True, "materialize_to_datario": False, @@ -250,7 +251,6 @@ trn_horaFim FROM GestaoEscolar.dbo.ACA_Turno """, - "dataset_id": "educacao_basica_frequencia", }, } From 884ab8a78f1df8af9694462345131e38190b8511 Mon Sep 17 00:00:00 2001 From: Karina Passos Date: Thu, 9 Nov 2023 16:59:44 +0000 Subject: [PATCH 6/8] adicionando queries --- .../dump_db_educacao_basica/schedules.py | 202 +++++++++++++++++- 1 file changed, 201 insertions(+), 1 deletion(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index bc808764b..8a3b8d3e0 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -230,7 +230,8 @@ "materialization_mode": "prod", "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", }, - "turno": { + "g": { + "dataset_id": "educacao_basica_frequencia", "dump_mode": "overwrite", "materialize_after_dump": False, "materialize_to_datario": False, @@ -252,6 +253,205 @@ FROM GestaoEscolar.dbo.ACA_Turno """, }, + "curriculo_periodo": { + "dataset_id": "educacao_basica_frequencia", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + cur_id, + crr_id, + crp_id, + mep_id, + crp_ordem, + crp_descricao, + crp_idadeIdealAnoInicio, + crp_idadeIdealMesInicio, + crp_idadeIdealAnoFim, + crp_idadeIdealMesFim, + crp_situacao, + crp_dataCriacao, + crp_dataAlteracao, + crp_controleTempo, + crp_qtdeDiasSemana, + crp_qtdeTemposSemana, + crp_qtdeHorasDia, + crp_qtdeMinutosDia, + crp_qtdeEletivasAlunos, + crp_ciclo, + crp_turmaAvaliacao, + crp_nomeAvaliacao, + crp_qtdeTemposDia, + crp_concluiNivelEnsino, + tci_id, + crp_fundoFrente, + tcp_id, + clg_id + FROM GestaoEscolar.dbo.ACA_CurriculoPeriodo + """, + }, + "tipo_turno": { + "dataset_id": "educacao_basica_frequencia", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + ttn_id, + ttn_nome, + ttn_situacao, + ttn_dataCriacao, + ttn_dataAlteracao, + ttn_tipo, + ttn_classificacao + FROM GestaoEscolar.dbo.ACA_TipoTurno + """, + }, + "turma_disciplina_rel": { + "dataset_id": "educacao_basica_frequencia", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + tur_id, + tud_id + FROM GestaoEscolar.dbo.TUR_TurmaRelTurmaDisciplina + """, + }, + "turma_curriculo": { + "dataset_id": "educacao_basica_frequencia", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + tur_id, + cur_id, + crr_id, + crp_id, + tcr_prioridade, + tcr_situacao, + tcr_dataCriacao, + tcr_dataAlteracao + FROM GestaoEscolar.dbo.TUR_TurmaCurriculo + """, + }, + "turma_aula": { + "dataset_id": "educacao_basica_frequencia", + "partition_columns": "tau_dataAlteracao", + "partition_date_format": "%Y-%m-%d", + "lower_bound_date": "current_month", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + tud_id, + tau_id, + tpc_id, + tau_sequencia, + tau_data, + tau_numeroAulas, + tau_planoAula, + tau_diarioClasse, + tau_situacao, + tau_dataCriacao, + tau_dataAlteracao, + tau_conteudo, + tau_efetivado, + tau_atividadeCasa, + tdt_posicao, + pro_id, + tau_sintese, + tau_reposicao, + usu_id, + usu_idDocenteAlteracao, + tau_statusFrequencia, + tau_statusAtividadeAvaliativa, + tau_statusAnotacoes, + tau_statusPlanoAula, + tau_recursosUtilizados + FROM GestaoEscolar.dbo.CLS_TurmaAula + """, + }, + "turma_aula_aluno": { + "dataset_id": "educacao_basica_frequencia", + "partition_columns": "tau_dataAlteracao", + "partition_date_format": "%Y-%m-%d", + "lower_bound_date": "current_month", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + tud_id, + tau_id, + alu_id, + mtu_id, + mtd_id, + taa_frequencia, + taa_situacao, + taa_dataCriacao, + taa_dataAlteracao, + taa_anotacao, + taa_frequenciaBitMap, + usu_idDocenteAlteracao + FROM GestaoEscolar.dbo.CLS_TurmaAulaAluno + """, + }, + "turma_disciplina": { + "dataset_id": "educacao_basica_frequencia", + "partition_columns": "tud_dataAlteracao", + "partition_date_format": "%Y-%m-%d", + "lower_bound_date": "current_month", + "dump_mode": "overwrite", + "materialize_after_dump": False, + "materialize_to_datario": False, + "dump_to_gcs": False, + "materialization_mode": "prod", + "execute_query": """ + SELECT + tud_id, + tud_codigo, + tud_nome, + tud_multiseriado, + tud_vagas, + tud_minimoMatriculados, + tud_duracao, + tud_modo, + tud_tipo, + tud_dataInicio, + tud_dataFim, + tud_situacao, + tud_dataCriacao, + tud_dataAlteracao, + tud_cargaHorariaSemanal, + tud_aulaForaPeriodoNormal, + tud_global, + tud_disciplinaEspecial, + tud_naoLancarNota, + tud_naoLancarFrequencia, + tud_naoExibirNota, + tud_naoExibirFrequencia, + tud_semProfessor, + tud_naoExibirBoletim + FROM GestaoEscolar.dbo.TUR_TurmaDisciplina + """, + }, } From 29fe00bcb613acd3b36165be55adacb8a44c6e52 Mon Sep 17 00:00:00 2001 From: Karina Passos Date: Fri, 10 Nov 2023 17:44:45 +0000 Subject: [PATCH 7/8] corrigindo erro de digitacao --- pipelines/rj_sme/dump_db_educacao_basica/schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index 8a3b8d3e0..6604ded07 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -230,7 +230,7 @@ "materialization_mode": "prod", "execute_query": "SELECT * FROM GestaoEscolar.dbo.VW_BI_Turma", }, - "g": { + "turno": { "dataset_id": "educacao_basica_frequencia", "dump_mode": "overwrite", "materialize_after_dump": False, From a3ee2d01a0ce989c8d582f675e92e25aac916719 Mon Sep 17 00:00:00 2001 From: Karina Passos Date: Tue, 14 Nov 2023 18:42:57 +0000 Subject: [PATCH 8/8] corrigindo erro de digitacao --- pipelines/rj_sme/dump_db_educacao_basica/schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py index 6604ded07..960bc4934 100644 --- a/pipelines/rj_sme/dump_db_educacao_basica/schedules.py +++ b/pipelines/rj_sme/dump_db_educacao_basica/schedules.py @@ -388,7 +388,7 @@ }, "turma_aula_aluno": { "dataset_id": "educacao_basica_frequencia", - "partition_columns": "tau_dataAlteracao", + "partition_columns": "taa_dataAlteracao", "partition_date_format": "%Y-%m-%d", "lower_bound_date": "current_month", "dump_mode": "overwrite",