Feat/pipeline e2 e tests

.gitignore

@@ -424,3 +424,4 @@ local.settings.json
 front-end-components/vite.config-dev.js.timestamp-*
 documentation/build
 documentation/**/work
+data-pipeline/tests/e2e/.env.local

data-pipeline/tests/e2e/.env.local

@@ -1,4 +1,4 @@
 STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
 WORKER_QUEUE_NAME=data-pipeline-job-start
 COMPLETE_QUEUE_NAME=data-pipeline-job-finished
-RAW_DATA_CONTAINER=raw
+RAW_DATA_CONTAINER=raw

data-pipeline/tests/e2e/scenarios/WhenLoadingMultiAcademyTrusts/raw/academy_master_list.csv

@@ -0,0 +1,50 @@
+Company Registration Number,Incorporation Date,Academy Trust UPIN,UKPRN,Academy Trust Name,Academy Name,Academy UPIN,Trust Type,LA Establishment Number,Date Opened,LA Name,Type of Provision - Phase,Regional School Commissioner,Valid From,Valid to,Date joined or opened if in period,Date left or closed if in period,Territory,Academy Status,Academy Trust Status,Number of Academies in Trust,Number of Pupils,GAG Value,Lead RSC,Year
+2236171,00:00.0,134858,10058148,Endeavour Academies Trust,Macmillan Academy,118397,Multi Academy Trust (MAT),806-6907,00:00.0,Middlesbrough Council,Secondary,North,00:00.0,,,,North,Member for whole period,Existing,2,1592,,North,202122
+2236171,00:00.0,134858,10058148,Endeavour Academies Trust,Archibald Primary School,140487,Multi Academy Trust (MAT),806-2325,00:00.0,Middlesbrough Council,Primary,North,00:00.0,,,,North,Member for whole period,Existing,2,462,,North,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons City Academy,118388,Multi Academy Trust (MAT),380-6905,00:00.0,Bradford Metropolitan District Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,885,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Allerton Academy,118885,Multi Academy Trust (MAT),380-6908,00:00.0,Bradford Metropolitan District Council,All through,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,1932,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Kings Academy,120465,Multi Academy Trust (MAT),380-4004,00:00.0,Bradford Metropolitan District Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,835,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Music Primary,121977,Multi Academy Trust (MAT),380-2008,00:00.0,Bradford Metropolitan District Council,Primary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,420,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Trinity Academy,122017,Multi Academy Trust (MAT),380-4010,00:00.0,Bradford Metropolitan District Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,623,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Marchbank Primary,127700,Multi Academy Trust (MAT),380-2018,00:00.0,Bradford Metropolitan District Council,Primary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,455,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons McMillan Academy,130495,Multi Academy Trust (MAT),380-4024,00:00.0,Bradford Metropolitan District Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,621,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Manningham Academy,131394,Multi Academy Trust (MAT),380-2025,00:00.0,Bradford Metropolitan District Council,Primary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,410,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Trinity Chapeltown,139847,Multi Academy Trust (MAT),383-4075,00:00.0,Leeds City Council,All through,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,709,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Cottingley Academy,141162,Multi Academy Trust (MAT),380-4073,00:00.0,Bradford Metropolitan District Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,818,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Unity Academy,141373,Multi Academy Trust (MAT),383-4078,00:00.0,Leeds City Council,Secondary,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,834,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons free Sixth Form,152005,Multi Academy Trust (MAT),380-4048,00:00.0,Bradford Metropolitan District Council,16 Plus,Lancashire & West Yorkshire,00:00.0,,,,North,Member for whole period,Existing,15,960,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Broadgreen Academy,163199,Multi Academy Trust (MAT),341-4013,00:00.0,Liverpool City Council,Secondary,Lancashire & West Yorkshire,00:00.0,,00:00.0,,North,Newly converted member,Existing,15,1287,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Fazakerley Academy,163203,Multi Academy Trust (MAT),341-4012,00:00.0,Liverpool City Council,Special,Lancashire & West Yorkshire,00:00.0,,00:00.0,,North,Newly converted member,Existing,15,830,,Lancashire & West Yorkshire,202122
+2303464,00:00.0,134876,10058150,Dixons Academies Trust,Dixons Brooklands Academy,163753,Multi Academy Trust (MAT),352-4017,00:00.0,Manchester City Council,Secondary,Lancashire & West Yorkshire,00:00.0,,00:00.0,,North,in period transfer,Existing,15,993,,Lancashire & West Yorkshire,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Longfield Academy,118625,Multi Academy Trust (MAT),886-6914,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1026,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,The Leigh Academy,118686,Multi Academy Trust (MAT),886-6910,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1379,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Strood Academy,119100,Multi Academy Trust (MAT),887-6905,00:00.0,Medway Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1324,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Hayesbrook Academy,119302,Multi Academy Trust (MAT),886-5455,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,00:00.0,,South and South West,in period transfer,Existing,32,464,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Wilmington Academy,119330,Multi Academy Trust (MAT),886-6920,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1355,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Sir Joseph Williamson's Mathematical School,119657,Multi Academy Trust (MAT),887-4530,00:00.0,Medway Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1464,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Mascalls Academy,119847,Multi Academy Trust (MAT),886-5439,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1291,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,The Hundred of Hoo Academy,120119,Multi Academy Trust (MAT),887-4000,00:00.0,Medway Council,All through,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1676,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Milestone Academy,120666,Multi Academy Trust (MAT),886-7066,00:00.0,Kent County Council,Special,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,362,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Tree Tops Primary Academy,120925,Multi Academy Trust (MAT),886-2004,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,284,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,High Weald Academy,120934,Multi Academy Trust (MAT),886-4003,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,00:00.0,,South and South West,in period transfer,Existing,32,98,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,The Ebbsfleet Academy,120981,Multi Academy Trust (MAT),886-4001,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,00:00.0,,South and South West,in period transfer,Existing,32,684,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,High Halstow Primary Academy,121054,Multi Academy Trust (MAT),887-2421,00:00.0,Medway Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,210,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Molehill Primary Academy,121174,Multi Academy Trust (MAT),886-2007,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,295,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Oaks Primary Academy - Maidstone,122237,Multi Academy Trust (MAT),886-2003,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,201,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Hartley Primary Academy - Longfield,122281,Multi Academy Trust (MAT),886-2511,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,439,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Allhallows Primary Academy,124701,Multi Academy Trust (MAT),887-2005,00:00.0,Medway Council,Primary,South London & South East,00:00.0,00:00.0,,00:00.0,South and South West,closure in period,Existing,32,0,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Stationers' Crown Woods Academy,130341,Multi Academy Trust (MAT),203-4271,00:00.0,Greenwich London Borough Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,1613,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,The Leigh UTC - Dartford,130560,Multi Academy Trust (MAT),886-4012,00:00.0,Kent County Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,730,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Peninsula East Primary Academy,130714,Multi Academy Trust (MAT),887-2194,00:00.0,Medway Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,244,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Dartford Primary Academy,131069,Multi Academy Trust (MAT),886-2069,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,668,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Langley Park Primary Academy - Maidstone,132730,Multi Academy Trust (MAT),886-2073,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,415,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Eastcote Primary Academy,133453,Multi Academy Trust (MAT),303-2003,00:00.0,Bexley London Borough Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,212,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Cherry Orchard Primary Academy,133619,Multi Academy Trust (MAT),886-2076,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,437,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,The Halley Academy,140063,Multi Academy Trust (MAT),203-4007,00:00.0,Greenwich London Borough Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,895,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Leigh Academy Blackheath,140963,Multi Academy Trust (MAT),203-4008,00:00.0,Greenwich London Borough Council,Secondary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,718,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Horsmonden Primary Academy,152020,Multi Academy Trust (MAT),886-2135,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,154,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Paddock Wood Primary Academy,162188,Multi Academy Trust (MAT),886-2127,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,599,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Bearsted Primary Academy,162796,Multi Academy Trust (MAT),886-2131,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,177,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Snowfields Academy,162826,Multi Academy Trust (MAT),886-7006,00:00.0,Kent County Council,Special,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,85,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Marden Primary Academy,162894,Multi Academy Trust (MAT),886-2183,00:00.0,Kent County Council,Primary,South London & South East,00:00.0,,,,South and South West,Member for whole period,Existing,32,285,,South London & South East,202122
+2336587,00:00.0,134878,10058151,Leigh Academies Trust,Leigh Academy Rainham,163432,Multi Academy Trust (MAT),887-4004,00:00.0,Medway Council,Primary,South London & South East,00:00.0,,00:00.0,,South and South West,Newly converted member,Existing,32,239,,South London & South East,202122