Merge pull request #3308 from DFE-Digital/dedicated_redis_postgres_re…

…view_app

Dedicated redis and postgresql for review app

Makefile

@@ -93,7 +93,7 @@ composed-variables:
 	$(eval STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}tfstate${CONFIG_SHORT}sa)
 
 bin/terrafile: ## Install terrafile to manage terraform modules
-	curl -sL https://github.com/coretech/terrafile/releases/download/v${TERRAFILE_VERSION}/terrafile_${TERRAFILE_VERSION}_$$(uname)_x86_64.tar.gz \
+	curl -sL https://github.com/coretech/terrafile/releases/download/v${TERRAFILE_VERSION}/terrafile_${TERRAFILE_VERSION}_$$(uname)_$$(uname -m).tar.gz \
 		| tar xz -C ./bin terrafile
 
 terraform-init: composed-variables bin/terrafile set-azure-account

app/models/bookings/data/gias_data_file.rb

@@ -4,6 +4,7 @@ class GiasDataFile
       EXPECTED_HEADER = '"URN","LA (code)","LA (name)","EstablishmentNumber","EstablishmentName"'.freeze
       EXPECTED_FIRST_ROW = %r(\A\d{3}\d+,)
       TEMP_PATH = Rails.root.join('tmp', 'gias').freeze
+      SAMPLE_PATH = "db/data/example_schools.csv".freeze
       attr_reader :today
 
       def initialize

app/models/bookings/school_sync.rb

@@ -25,6 +25,10 @@ def import_all
     end
   end
 
+  def import_sample
+    Bookings::Data::SchoolMassImporter.new(data_sample, email_override).import
+  end
+
   # update any school records that differ from edubase source
   def update_all
     data_in_batches do |batch|
@@ -48,6 +52,10 @@ def gias_data_file
     Bookings::Data::GiasDataFile.new.path
   end
 
+  def gias_data_sample_file
+    Bookings::Data::GiasDataFile::SAMPLE_PATH
+  end
+
   def data_in_batches
     rows = []
     CSV.foreach(gias_data_file, headers: true, encoding: "ISO-8859-1:UTF-8") do |row|
@@ -63,4 +71,12 @@ def data_in_batches
 
     true
   end
+
+  def data_sample
+    rows = []
+    CSV.foreach(gias_data_sample_file, headers: true, encoding: "ISO-8859-1:UTF-8") do |row|
+      rows << row
+    end
+    rows
+  end
 end

config/database.yml

@@ -17,6 +17,9 @@
 default: &default
   adapter: postgis
   encoding: unicode
+  # AKS DATABASE_URL will start with `postgres://`, which overrides the postgis adapter
+  # so we need to replace `postgres` if it occurs at the beginning of DATABASE_URL
+  url: <%= ENV.fetch('DATABASE_URL', 'postgis://postgres@localhost:5432').sub(/^postgres/, "postgis") %>
   # For details on connection pooling, see Rails configuration guide
   # http://guides.rubyonrails.org/configuring.html#database-pooling
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
@@ -98,10 +101,6 @@ test: &test
 #
 production: &production
   <<: *default
-  host: <%= ENV['DB_HOST'] %>
-  database: <%= ENV['DB_DATABASE'] %>
-  username: <%= ENV['DB_USERNAME'] %>
-  password: <%= ENV['DB_PASSWORD'] %>
   keepalives: 1
   keepalives_idle: 60
   keepalives_interval: 20

db/data/example_schools.csv

@@ -0,0 +1,3 @@
+"URN","LA (code)","LA (name)","EstablishmentNumber","EstablishmentName","TypeOfEstablishment (code)","TypeOfEstablishment (name)","EstablishmentTypeGroup (code)","EstablishmentTypeGroup (name)","EstablishmentStatus (code)","EstablishmentStatus (name)","ReasonEstablishmentOpened (code)","ReasonEstablishmentOpened (name)","OpenDate","ReasonEstablishmentClosed (code)","ReasonEstablishmentClosed (name)","CloseDate","PhaseOfEducation (code)","PhaseOfEducation (name)","StatutoryLowAge","StatutoryHighAge","Boarders (code)","Boarders (name)","NurseryProvision (name)","OfficialSixthForm (code)","OfficialSixthForm (name)","Gender (code)","Gender (name)","ReligiousCharacter (code)","ReligiousCharacter (name)","ReligiousEthos (name)","Diocese (code)","Diocese (name)","AdmissionsPolicy (code)","AdmissionsPolicy (name)","SchoolCapacity","SpecialClasses (code)","SpecialClasses (name)","CensusDate","NumberOfPupils","NumberOfBoys","NumberOfGirls","PercentageFSM","TrustSchoolFlag (code)","TrustSchoolFlag (name)","Trusts (code)","Trusts (name)","SchoolSponsorFlag (name)","SchoolSponsors (name)","FederationFlag (name)","Federations (code)","Federations (name)","UKPRN","FEHEIdentifier","FurtherEducationType (name)","OfstedLastInsp","OfstedSpecialMeasures (code)","OfstedSpecialMeasures (name)","LastChangedDate","Street","Locality","Address3","Town","County (name)","Postcode","SchoolWebsite","TelephoneNum","HeadTitle (name)","HeadFirstName","HeadLastName","HeadPreferredJobTitle","BSOInspectorateName (name)","InspectorateReport","DateOfLastInspectionVisit","NextInspectionVisit","TeenMoth (name)","TeenMothPlaces","CCF (name)","SENPRU (name)","EBD (name)","PlacesPRU","FTProv (name)","EdByOther (name)","Section41Approved (name)","SEN1 (name)","SEN2 (name)","SEN3 (name)","SEN4 (name)","SEN5 (name)","SEN6 (name)","SEN7 (name)","SEN8 (name)","SEN9 (name)","SEN10 (name)","SEN11 (name)","SEN12 (name)","SEN13 (name)","TypeOfResourcedProvision (name)","ResourcedProvisionOnRoll","ResourcedProvisionCapacity","SenUnitOnRoll","SenUnitCapacity","GOR (code)","GOR (name)","DistrictAdministrative (code)","DistrictAdministrative (name)","AdministrativeWard (code)","AdministrativeWard (name)","ParliamentaryConstituency (code)","ParliamentaryConstituency (name)","UrbanRural (code)","UrbanRural (name)","GSSLACode (name)","Easting","Northing","MSOA (name)","LSOA (name)","InspectorateName (name)","SENStat","SENNoStat","BoardingEstablishment (name)","PropsName","PreviousLA (code)","PreviousLA (name)","PreviousEstablishmentNumber","OfstedRating (name)","Country (name)","UPRN","SiteName","QABName (code)","QABName (name)","EstablishmentAccredited (code)","EstablishmentAccredited (name)","QABReport","CHNumber","MSOA (code)","LSOA (code)","FSM","AccreditationExpiryDate"
+138130,"355","Salford","7025","Example Academy","44","Academy special converter","10","Academies","1","Open","10","Academy Converter","01-01-2000","99","","","0","Not applicable",9,19,"1","No boarders","No Nursery Classes","1","Has a sixth form","3","Mixed","00","Does not apply","Does not apply","0000","Not applicable","9","",284,"1","Has Special Classes","18-01-2024","315","221","94","51.80","3","Supported by a multi-academy trust","5711","EXAMPLE ACADEMY TRUST","Linked to a sponsor","Example Academy Trust","Not applicable","","",10037232,"","Not applicable","24-01-2024","0","Not applicable","18-07-2024","Example Road","Ellesmere Park","","Eccles","Lancashire","M30 9DY","www.example.co.uk","01234567890","Mrs","Example","Example","Executive Principal","Not applicable","","","","Not applicable","","Not applicable","Not applicable","Not applicable","","","Not applicable","Not applicable","SpLD - Specific Learning Difficulty","VI - Visual Impairment","SLCN - Speech, language and Communication","ASD - Autistic Spectrum Disorder","SEMH - Social, Emotional and Mental Health","PD - Physical Disability","MLD - Moderate Learning Difficulty","","","","","","","","","","","","B","North West","E08000006","Salford","E05013024","Eccles","E14001598","Worsley and Eccles","A1","(England/Wales) Urban major conurbation","E08000006",377777,399838,"Salford 020","Salford 020C","","","","","","999","","","Outstanding","","10091474414","","0","Not applicable","0","Not applicable","","","E02001176","E01005633","130",""
+140134,"207","Kensington and Chelsea","4320","Example School","34","Academy converter","10","Academies","2","Closed","10","Academy Converter","01-01-2000","04","Fresh Start","31-12-2022","4","Secondary",11,18,"1","No boarders","No Nursery Classes","1","Has a sixth form","3","Mixed","00","Does not apply","Does not apply","0000","Not applicable","4","Non-selective",1430,"2","No Special Classes","20-01-2022","1391","732","659","24.0","2","Not supported by a trust","","","-","","Not applicable","","",10043056,"","Not applicable","21-04-2022","0","Not applicable","31-12-2022","Example Gardens","Example Road","","London","","W8 7AF","","","","Example","Example","Headteacher","Not applicable","","","","Not applicable","","Not applicable","Not applicable","Not applicable","","","Not applicable","Not applicable","","","","","","","","","","","","","","","","","","","H","London","E09000020","Kensington and Chelsea","E05009390","Campden","E14000768","Kensington","A1","(England/Wales) Urban major conurbation","E09000020",524948,179954,"Kensington and Chelsea 008","Kensington and Chelsea 008B","","","","","","999","","","Serious Weaknesses","","217126493","","0","Not applicable","0","Not applicable","","","E02000584","E01002825","283",""

db/migrate/20190924080833_add_dfe_signin_organisation_uuid_to_bookings_schools.rb

@@ -1,4 +1,4 @@
-class AddDfeSigninOrganisationUuidToBookingsSchools < ActiveRecord::Migration[5.2]
+class AddDfESigninOrganisationUuidToBookingsSchools < ActiveRecord::Migration[5.2]
   def change
     add_column :bookings_schools, :dfe_signin_organisation_uuid, :uuid, null: true
   end

db/seeds.rb

@@ -13,6 +13,8 @@
   Bookings::SchoolType.create(name: name, edubase_id: edubase_id)
 end
 
+# Import a sample of 1000 schools
 # rubocop:disable Rails/Output
-puts "\nYou can import all 47000 schools using 'bundle exec rails data:schools:mass_import'"
+puts "\nImporting a small sample of schools. You can import all 47000 schools using 'bundle exec rails data:schools:mass_import'"
+Bookings::SchoolSync.new(email_override: nil).import_sample
 # rubocop:enable Rails/Output

docs/documents/release-process.md

@@ -4,4 +4,28 @@
 
    * When deploying a review application a script is run to find the first available static route (review-school-experience[1-20] ) and attach that to the review, via terraform. This enables the DFE Sign In process to work.
 
-1. Merging the PR will deploy the changes to the staging environment and, if successful, the production environment
+2. Merging the PR will deploy the changes to the staging environment and, if successful, the production environment
+
+## Configuring a school profile in a review app or local deployment for testing
+
+When starting from a freshly seeded database, you must manually set up a school profile before you can start making bookings for the school. 
+
+1. Ensure you have a login to DfE Sign In (pre-prod) with permission to manage any of the schools listed in `db/data/example_schools.csv`. [See this page](https://technical-guidance.education.gov.uk/infrastructure/support/#dfe-sign-in) for support and access to DfE Sign In. 
+2. In a School Experience app running locally or as review app, click on [Manage school experience](/schools) on the right-hand menu
+3. Click on "Start now >" and you will be directed to login to DfE Sign In
+4. Login to DfE Sign In
+5. After signing in, you will be re-directed back to the School Experience website
+6. Click on "Complete your school profile"
+7. Fill in each of the sections:
+   1. School experience subjects and education phase
+   2. Candidate requirements and school experience details
+   3. Safeguarding and fees
+   4. Disability and access
+   5. Admin contact details
+   6. Click on check and submit your profile
+   7. Accept the conditions and click on "Accept and set up profile"
+8. Add some placement dates to the school profile
+9. Ensure the profile is set to "on"
+
+You should now be able to find the school and apply for placements by searching for it from the [candidates search page](/candidates/school_searches/new)
+

lib/tasks/data/manage_schools.rake

@@ -7,6 +7,13 @@ namespace :data do
       Bookings::SchoolSync.new(email_override: args[:email_override]).import_all
     end
 
+    desc "Import a sample of  GiaS (EduBase) data from local file"
+    task :sample_import, %i[email_override] => :environment do |_t, args|
+      args.with_defaults(email_override: nil)
+
+      Bookings::SchoolSync.new(email_override: args[:email_override]).import_sample
+    end
+
     desc "Update schools"
     task update: :environment do |_t, _args|
       Bookings::SchoolSync.new.update_all

script/docker-entrypoint.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -eu
+
 MODE=NOTSET
 MODE_ERROR="Mode must be one of (  -s (sidekiq), -c (brakeman), -f (frontend), -r (rubocop) , -x (database) , -y (cucumber) , -z (rspec), -s (shell) )"
 
@@ -12,6 +14,11 @@ while [[ $# -gt 0 ]]; do
           bundle exec rails db:migrate
 	  shift
       ;;
+    -e|prepare)
+	  echo Preparing database
+          bundle exec rails db:prepare
+	  shift
+      ;;
     -p|profile)
 	  echo Setting Profile ${2}
           export PROFILE=${2}

terraform/aks/application.tf

@@ -16,11 +16,8 @@ module "application_configuration" {
     DFE_SIGNIN_BASE_URL = "https://${var.dsi_hostname}"
   }
   secret_variables = {
-    DB_HOST     = var.deploy_postgres ? module.postgres[0].host : "${data.azurerm_key_vault_secret.db_host[0].value}"
-    DB_USERNAME = var.deploy_postgres ? module.postgres[0].username : "${data.azurerm_key_vault_secret.db_username[0].value}"
-    DB_PASSWORD = var.deploy_postgres ? module.postgres[0].password : "${data.azurerm_key_vault_secret.db_password[0].value}"
-    DB_DATABASE = var.deploy_postgres ? module.postgres[0].name : "${data.azurerm_key_vault_secret.db_name[0].value}"
-    REDIS_URL   = var.deploy_redis ? module.redis-cache[0].url : "${data.azurerm_key_vault_secret.redis_url[0].value}"
+    DATABASE_URL = module.postgres[0].url
+    REDIS_URL    = module.redis-cache[0].url
   }
 }
 
@@ -38,12 +35,12 @@ module "web_application" {
   kubernetes_secret_name     = module.application_configuration.kubernetes_secret_name
 
   docker_image           = var.docker_image
-  command                = ["/app/docker-entrypoint.sh", "-m", "-f"]
+  command                = var.webapp_command
   probe_path             = "/check"
   web_external_hostnames = var.create_dsi_ingress ? [var.dsi_hostname] : []
-  enable_logit               = var.enable_logit
+  enable_logit           = var.enable_logit
 
-  enable_prometheus_monitoring  = var.enable_prometheus_monitoring
+  enable_prometheus_monitoring = var.enable_prometheus_monitoring
 }
 
 module "worker_application" {
@@ -62,5 +59,5 @@ module "worker_application" {
   replicas                   = var.sidekiq_replicas
   enable_logit               = var.enable_logit
 
-  enable_prometheus_monitoring  = var.enable_prometheus_monitoring
+  enable_prometheus_monitoring = var.enable_prometheus_monitoring
 }

terraform/aks/config/review.tfvars.json

@@ -2,20 +2,15 @@
     "cluster": "test",
     "namespace": "git-development",
     "azure_enable_backup_storage": false,
+    "deploy_azure_backing_services": false,
     "enable_monitoring": false,
-    "deploy_redis": false,
-    "deploy_postgres": false,
-    "key_vault_name": "s189t01-gse-rv-app-kv",
     "infra_key_vault_name": "s189t01-gse-rv-inf-kv",
     "key_vault_resource_group":  "s189t01-gse-rv-rg",
-    "review_db_dbname": "review-db-name",
-    "review_db_hostname": "review-db-host",
-    "review_db_username": "review-db-username",
-    "review_db_password": "review-db-password",
-    "review_url_redis_name": "review-redis-url",
     "statuscake_password_name": "SC-PASSWORD",
     "sidekiq_replicas" : 1,
     "sidekiq_memory_max" : "1Gi",
     "create_dsi_ingress": true,
-    "enable_logit": true
+    "enable_logit": true,
+    "webapp_command": ["/app/docker-entrypoint.sh", "-e", "-f"],
+    "create_database": false
 }

terraform/aks/data.tf

@@ -1,38 +1,3 @@
-data "azurerm_key_vault" "app_secret_vault" {
-  count               = var.deploy_postgres ? 0 : 1
-  name                = var.key_vault_name
-  resource_group_name = var.key_vault_resource_group
-}
-data "azurerm_key_vault_secret" "db_name" {
-  count        = length(data.azurerm_key_vault.app_secret_vault) == 0 ? 0 : 1
-  name         = var.review_db_dbname
-  key_vault_id = data.azurerm_key_vault.app_secret_vault[0].id
-}
-
-data "azurerm_key_vault_secret" "db_password" {
-  count        = length(data.azurerm_key_vault.app_secret_vault) == 0 ? 0 : 1
-  name         = var.review_db_password
-  key_vault_id = data.azurerm_key_vault.app_secret_vault[0].id
-}
-
-data "azurerm_key_vault_secret" "db_username" {
-  count        = length(data.azurerm_key_vault.app_secret_vault) == 0 ? 0 : 1
-  name         = var.review_db_username
-  key_vault_id = data.azurerm_key_vault.app_secret_vault[0].id
-}
-
-data "azurerm_key_vault_secret" "db_host" {
-  count        = length(data.azurerm_key_vault.app_secret_vault) == 0 ? 0 : 1
-  name         = var.review_db_hostname
-  key_vault_id = data.azurerm_key_vault.app_secret_vault[0].id
-}
-
-data "azurerm_key_vault_secret" "redis_url" {
-  count        = length(data.azurerm_key_vault.app_secret_vault) == 0 ? 0 : 1
-  name         = var.review_url_redis_name
-  key_vault_id = data.azurerm_key_vault.app_secret_vault[0].id
-}
-
 data "azurerm_key_vault" "infra_secret_vault" {
   name                = var.infra_key_vault_name
   resource_group_name = var.key_vault_resource_group