Add documentation on AWS report delivery, auth

README.md

@@ -10,21 +10,39 @@ renamed to ucsc-cgp/cloud-billing-report.
 
 ### Cloud setup
 
-Billing data is presented using
+Billing data is provided by AWS and GCP via:
 
-* the [S3 Cost and Usage Report][s3] feature for AWS, and
+* the [AWS Cost and Usage Report][s3] feature, and
 
-* the [GCS Cloud Billing][gcs] feature for GCP.
+* the [Cloud Billing][gcs] feature for GCP.
+
+AWS Cost and Usage Reports are delivered by Amazon to a specified S3 bucket in
+Parquet format. A BigQuery Data Transfer job runs daily, automatically importing
+the AWS billing data to a BigQuery table where it is queried for report
+generation. As such, report generation requires GCP resources even if GCP
+reports are not being generated.
+
+Google automatically loads billing data into a specified BigQuery dataset.
+(This must be set up manually.)
+
+At time of writing, many of these resources have been deployed manually by
+Erich. Some resources are managed with Terraform. In the former case, "example"
+Terraform configuration is included. All such configuration lives in
+[terraform/][terraform/].
 
 Credentials configured in `config.json` must be authorized for access to
-billing data generated these features.
+billing data generated by these features.
 
   [s3]: https://docs.aws.amazon.com/cur/latest/userguide/cur-s3.html
-  [gcs]: https://cloud.google.com/billing/docs/how-to/export-data-file
+  [gcs]: https://cloud.google.com/billing/docs/how-to/export-data-bigquery
 
-### Generating reports
 
-You'll need Python 3.x. I've only tested this using Python 3.8.6.
+### Local requirements
+
+* Python 3.8.6
+* Terraform 0.12
+
+### Generating reports
 
 First, populate `config.json` and install requirements:
 

terraform/aws.tf

@@ -0,0 +1,72 @@
+# These resources are not deployed; they are only provided for convenience
+# to document the resources that Erich has provisioned manually.
+# The actual state of these resources may differ.
+
+resource "aws_cur_report_definition" "report" {
+  report_name          = "ucsc_billing_report"
+  time_unit            = "DAILY"
+  format               = "Parquet"
+  compression          = "Parquet"
+  s3_bucket            = aws_s3_bucket.report
+  s3_prefix            = "ucsc_billing_report"
+  s3_region            = aws_s3_bucket.report.region
+  additional_artifacts = ["ATHENA"]
+  report_versioning    = "OVERWRITE_REPORT"
+}
+
+resource "aws_s3_bucket" "report" {
+  policy = <<POLICY
+{
+ "Version": "2008-10-17",
+ "Statement": [
+   {
+     "Effect": "Allow",
+     "Principal": {
+       "Service": "billingreports.amazonaws.com"
+     },
+     "Action": [
+       "s3:GetBucketAcl",
+       "s3:GetBucketPolicy"
+     ],
+     "Resource": "${aws_s3_bucket.report.arn}/*"
+   },
+   {
+     "Effect": "Allow",
+     "Principal": {
+       "Service": "billingreports.amazonaws.com"
+     },
+     "Action": [
+       "s3:PutObject"
+     ],
+     "Resource": "${aws_s3_bucket.report.arn}"
+   }
+ ]
+}
+POLICY
+}
+
+resource "aws_iam_role" "report" {
+  name               = "report"
+  assume_role_policy = <<POLICY
+{
+   "Version": "2012-10-17",
+   "Statement": [
+       {
+           "Effect": "Allow",
+           "Action": [
+               "s3:ListBucket",
+               "s3:GetBucketLocation"
+           ],
+           "Resource": "${aws_s3_bucket.report.arn}"
+       },
+       {
+           "Effect": "Allow",
+           "Action": [
+               "s3:GetObject"
+           ],
+           "Resource": "${aws_s3_bucket.report.arn}/*"
+       }
+   ]
+}
+POLICY
+}