From ac4a66b92c6fe0d211303fba0df640c00775fe76 Mon Sep 17 00:00:00 2001 From: Faizi52996 Date: Fri, 31 May 2024 10:30:53 +0100 Subject: [PATCH] updated scenario name --- .gitignore | 1 + data-pipeline/tests/e2e/.env.local | 8 +++++++- .../expected/academies.csv | 0 .../expected/academy_ar.csv | 0 .../expected/cdc.csv | 0 .../expected/census.csv | 0 .../expected/ks2.csv | 0 .../expected/ks4.csv | 0 .../expected/schools.csv | 0 .../expected/sen.csv | 0 .../expected/trust_ar.csv | 0 .../raw/academy_ar.xlsx | Bin .../raw/academy_master_list.xlsx | Bin .../raw/cdc.csv | 0 .../raw/census_pupils.csv | 0 .../raw/census_workforce.xlsx | Bin .../raw/gias.csv | 0 .../raw/gias_all_links.csv | 0 .../raw/gias_links.csv | 0 .../raw/ks2.xlsx | Bin .../raw/ks4.xlsx | Bin .../raw/maintained_schools_master_list.csv | 0 .../raw/sen.csv | 0 data-pipeline/tests/e2e/synthetic_data_test.py | 2 +- 24 files changed, 9 insertions(+), 2 deletions(-) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/academies.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/academy_ar.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/cdc.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/census.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/ks2.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/ks4.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/schools.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/sen.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/expected/trust_ar.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/academy_ar.xlsx (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/academy_master_list.xlsx (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/cdc.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/census_pupils.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/census_workforce.xlsx (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/gias.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/gias_all_links.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/gias_links.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/ks2.xlsx (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/ks4.xlsx (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/maintained_schools_master_list.csv (100%) rename data-pipeline/tests/e2e/scenarios/{scenario2 => WhenLoadingSingleAcademyTrusts}/raw/sen.csv (100%) diff --git a/.gitignore b/.gitignore index 64f5c5fd4..bb8ecae43 100644 --- a/.gitignore +++ b/.gitignore @@ -424,3 +424,4 @@ local.settings.json front-end-components/vite.config-dev.js.timestamp-* documentation/build documentation/**/work +data-pipeline/tests/e2e/.env.local diff --git a/data-pipeline/tests/e2e/.env.local b/data-pipeline/tests/e2e/.env.local index 5f9d360ec..faeec2158 100644 --- a/data-pipeline/tests/e2e/.env.local +++ b/data-pipeline/tests/e2e/.env.local @@ -1,4 +1,10 @@ STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1; WORKER_QUEUE_NAME=data-pipeline-job-start COMPLETE_QUEUE_NAME=data-pipeline-job-finished -RAW_DATA_CONTAINER=raw \ No newline at end of file +RAW_DATA_CONTAINER=raw +DB_NAME=data +DB_PORT=1433 +DB_ARGS=Encrypt=no;TrustServerCertificate=no;Connection Timeout=30 +DB_HOST=mssql +DB_USER=sa +DB_PWD=mystrong!Pa55word \ No newline at end of file diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/academies.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/academies.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/academies.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/academies.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/academy_ar.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/academy_ar.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/academy_ar.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/academy_ar.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/cdc.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/cdc.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/cdc.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/cdc.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/census.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/census.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/census.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/census.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/ks2.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/ks2.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/ks2.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/ks2.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/ks4.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/ks4.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/ks4.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/ks4.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/schools.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/schools.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/schools.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/schools.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/sen.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/sen.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/sen.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/sen.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/expected/trust_ar.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/trust_ar.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/expected/trust_ar.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/expected/trust_ar.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/academy_ar.xlsx b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/academy_ar.xlsx similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/academy_ar.xlsx rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/academy_ar.xlsx diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/academy_master_list.xlsx b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/academy_master_list.xlsx similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/academy_master_list.xlsx rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/academy_master_list.xlsx diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/cdc.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/cdc.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/cdc.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/cdc.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/census_pupils.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/census_pupils.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/census_pupils.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/census_pupils.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/census_workforce.xlsx b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/census_workforce.xlsx similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/census_workforce.xlsx rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/census_workforce.xlsx diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/gias.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/gias.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/gias_all_links.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias_all_links.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/gias_all_links.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias_all_links.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/gias_links.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias_links.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/gias_links.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/gias_links.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/ks2.xlsx b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/ks2.xlsx similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/ks2.xlsx rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/ks2.xlsx diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/ks4.xlsx b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/ks4.xlsx similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/ks4.xlsx rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/ks4.xlsx diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/maintained_schools_master_list.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/maintained_schools_master_list.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/maintained_schools_master_list.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/maintained_schools_master_list.csv diff --git a/data-pipeline/tests/e2e/scenarios/scenario2/raw/sen.csv b/data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/sen.csv similarity index 100% rename from data-pipeline/tests/e2e/scenarios/scenario2/raw/sen.csv rename to data-pipeline/tests/e2e/scenarios/WhenLoadingSingleAcademyTrusts/raw/sen.csv diff --git a/data-pipeline/tests/e2e/synthetic_data_test.py b/data-pipeline/tests/e2e/synthetic_data_test.py index 4543c1f92..edeb62a69 100644 --- a/data-pipeline/tests/e2e/synthetic_data_test.py +++ b/data-pipeline/tests/e2e/synthetic_data_test.py @@ -101,7 +101,7 @@ def test_synthetic_data_env_is_set(): # To add a new scenario just add a folder with the name of the scenario # and add the scenario to the list below. -@pytest.mark.parametrize("scenario_name", ["scenario1"]) +@pytest.mark.parametrize("scenario_name", ["WhenLoadingSingleAcademyTrusts"]) def test_run_scenarios(scenario_name): copy_raw_files_to_raw_blob_storage(scenario_name) trigger_pipeline(scenario_name)