diff --git a/functions/README.md b/functions/README.md index baacd4e9..5948b35a 100644 --- a/functions/README.md +++ b/functions/README.md @@ -18,14 +18,14 @@ Then install the dependencies ### Configuring AWS credentials -First we need a bucket to write to, e.g. `concordqa-report-data`. We need to set this as an environment variable +First we need a bucket to write to, e.g. `concord-staging-report-data`. We need to set this as an environment variable for the auto_update function: -`firebase functions:config:set aws.s3_bucket=concordqa-report-data` +`firebase functions:config:set aws.s3_bucket=concord-staging-report-data` In order to write to S3, a user needs to be created in AWS IAM with permission to write to the S3 bucket. For instance there exists a user, `report-service-qa`, within the AminConcordQA role, with permission to write -to the `concordqa-report-data` S3 bucket. +to the `concord-staging-report-data` S3 bucket. We can download the AWS key and Secret Key for this user (check 1Password), and then push it up to Firebase with diff --git a/query-creator/README.md b/query-creator/README.md index ae03f693..f5fc7a31 100644 --- a/query-creator/README.md +++ b/query-creator/README.md @@ -219,7 +219,7 @@ You can change the behavior of the query-creator using the following query param ## AWS Glue/Athena Setup 1. Create a `report-service` database in AWS Glue. -2. If running on production, replace all instances of `concordqa-report-data` below with `concord-report-data`, or +2. If running on production, replace all instances of `concord-staging-report-data` below with `concord-report-data`, or any other location as appropriate 2. In Athena with the `report-service` database selected run the following: @@ -233,12 +233,12 @@ You can change the behavior of the query-creator using the following query param structure_id STRING ) ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' - LOCATION "s3://concordqa-report-data/activity-structure/" + LOCATION "s3://concord-staging-report-data/activity-structure/" TBLPROPERTIES ( "projection.enabled" = "true", "projection.structure_id.type" = "injected", - "storage.location.template" = "s3://concordqa-report-data/activity-structure/${structure_id}" + "storage.location.template" = "s3://concord-staging-report-data/activity-structure/${structure_id}" ) CREATE EXTERNAL TABLE IF NOT EXISTS learners ( @@ -262,12 +262,12 @@ You can change the behavior of the query-creator using the following query param query_id STRING ) ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' - LOCATION "s3://concordqa-report-data/learners/" + LOCATION "s3://concord-staging-report-data/learners/" TBLPROPERTIES ( "projection.enabled" = "true", "projection.query_id.type" = "injected", - "storage.location.template" = "s3://concordqa-report-data/learners/${query_id}" + "storage.location.template" = "s3://concord-staging-report-data/learners/${query_id}" ) CREATE EXTERNAL TABLE IF NOT EXISTS partitioned_answers ( @@ -296,11 +296,11 @@ You can change the behavior of the query-creator using the following query param escaped_url STRING ) ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' - LOCATION "s3://concordqa-report-data/partitioned-answers/" + LOCATION "s3://concord-staging-report-data/partitioned-answers/" TBLPROPERTIES ( "projection.enabled" = "true", "projection.escaped_url.type" = "injected", - "storage.location.template" = "s3://concordqa-report-data/partitioned-answers/${escaped_url}" + "storage.location.template" = "s3://concord-staging-report-data/partitioned-answers/${escaped_url}" ) ``` diff --git a/query-creator/env.sample.json b/query-creator/env.sample.json index 33372627..57cb6e83 100644 --- a/query-creator/env.sample.json +++ b/query-creator/env.sample.json @@ -1,6 +1,6 @@ { "Parameters": { - "OUTPUT_BUCKET": "concordqa-report-data", + "OUTPUT_BUCKET": "concord-staging-report-data", "REPORT_SERVICE_TOKEN": "replace_me_with_the_real_report_service_dev_token", "REPORT_SERVICE_URL": "https://us-central1-report-service-dev.cloudfunctions.net/api", "PORTAL_REPORT_URL": "https://portal-report.concord.org/branch/master/index.html", diff --git a/query-creator/samconfig.toml b/query-creator/samconfig.toml index d101145f..9595f18f 100644 --- a/query-creator/samconfig.toml +++ b/query-creator/samconfig.toml @@ -9,7 +9,7 @@ s3_prefix = "report-service-query-creator" region = "us-east-1" confirm_changeset = true capabilities = "CAPABILITY_IAM" -parameter_overrides = "OutputBucket=\"concordqa-report-data\" LogInputBucket=\"log-ingester-qa\" ReportServiceUrl=\"https://us-central1-report-service-dev.cloudfunctions.net/api\" ResearcherReportsUrl=\"https://researcher-reports.concord.org/branch/master/\" FirebaseApp=\"report-service-dev\" PortalReportUrl=\"https://portal-report.concord.org/branch/master/index.html\" LogAthenaDBName=\"log_ingester_qa\"" +parameter_overrides = "OutputBucket=\"concord-staging-report-data\" LogInputBucket=\"log-ingester-qa\" ReportServiceUrl=\"https://us-central1-report-service-dev.cloudfunctions.net/api\" ResearcherReportsUrl=\"https://researcher-reports.concord.org/branch/master/\" FirebaseApp=\"report-service-dev\" PortalReportUrl=\"https://portal-report.concord.org/branch/master/index.html\" LogAthenaDBName=\"log_ingester_qa\"" image_repositories = [] [production] diff --git a/query-creator/template.yaml b/query-creator/template.yaml index 26557409..9ae336d4 100644 --- a/query-creator/template.yaml +++ b/query-creator/template.yaml @@ -50,7 +50,7 @@ Resources: Properties: CodeUri: create-query/ Handler: app.lambdaHandler - Runtime: nodejs12.x + Runtime: nodejs14.x Events: CreateQuery: Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api diff --git a/scripts/README.md b/scripts/README.md index b9c60db2..76931ba4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -23,7 +23,7 @@ https://console.cloud.google.com/apis/credentials?project=report-service-pro&org Run the script by passing in the destination S3 bucket and the source key for the Firestore database. e.g. ``` -node export-answers.js concordqa-report-data authoring.staging.concord.org +node export-answers.js concord-staging-report-data authoring.staging.concord.org ``` the script accepts up to three arguments: diff --git a/scripts/export-answers.js b/scripts/export-answers.js index 717e6882..588eb4d1 100755 --- a/scripts/export-answers.js +++ b/scripts/export-answers.js @@ -7,7 +7,7 @@ const util = require('util'); const parquet = require('parquetjs'); const { parquetInfo, getSyncDocId, hasLTIMetadata, getHash, schema } = require('../functions/src/shared/s3-answers') -const BUCKET = process.argv[2]; // e.g. concordqa-report-data +const BUCKET = process.argv[2]; // e.g. concord-staging-report-data const SOURCE = process.argv[3]; // e.g. authoring.staging.concord.org const DATE_REGEX = process.argv[4] // optional, e.g. `2021-(04|05)` // if a regex is included, it will *only* be used if there is a `created` field in the answer.