Feature/snowflake

packages/snowflake/README.md

@@ -0,0 +1,19 @@
+# Snowflake Utilities
+This repo assumes you have an instance of Snowflake available to you with ACCOUNTADMIN access. If you don't yet have an account, you can start with a free trial [here](https://signup.snowflake.com/). If you already have an account but not ACCOUNTADMIN access, you'll need to reach out to your Snowflake administrator. 
+
+This deployment is using an instance of Snowflake deployed on Google Cloud Platform. This process can be completed on other cloud providers' Snowflake accounts, but will require a substantial amount of adaptation for the use of Snowpipe from bucket storage, and minor tweaks elsewhere in the code. We'll do our best to call out when these changes may occur. 
+
+You may also want to have the SnowSQL Client installed if you plan to interact with your database via the command line. Instructions can be found [here](https://docs.snowflake.com/en/user-guide/getting-started-tutorial-prerequisites.html#snowsql-installation)
+
+## Intro
+
+Contains documentation and scripts used to create and administer Snowflake databases, warehouses, tables, users, roles, pipes, and other objects either via [SnowSQL (CLI Client)](https://docs.snowflake.com/en/user-guide/snowsql.html) or through the Snowflake Web UI
+
+## Scripts 
+Scripts to be used via SnowSQL to execute functions, manipulate data, etc. 
+
+## Queries
+Raw SQL Queries to be used either through the Web UI or using the CLI
+
+### DDL 
+Queries used to create database infrastructure, roles, a user, and grant appropriate permissions

packages/snowflake/queries/ddl/CREATE_DATABASE.SQL

@@ -0,0 +1,33 @@
+// Creating the initial Snowflake database to store ingested data from a Google Cloud Bucket
+
+USE ROLE SYSADMIN // or a role which is granted the appropriate permissions to create database, schema, and warehouse objects
+// TODO: create a role which is limited to only the exact permissions needed to execute this
+
+CREATE DATABASE NC_CAMPAIGN_FINANCE;
+CREATE SCHEMA NCCF;
+CREATE WAREHOUSE NCCF_DEV_WH
+    WITH WAREHOUSE_SIZE = 'XSMALL'
+    WAREHOUSE_TYPE = 'STANDARD'
+    AUTO_SUSPEND = 60
+    AUTO_RESUME = TRUE;
+
+
+USE ROLE SECURITYADMIN; // or a role which is granted the appropriate permissions to grant permissions
+// TODO: limit this to just permissions required for these exact functions
+
+CREATE ROLE NCCF_DEV;
+GRANT ALL PRIVILEGES ON DATABASE NC_CAMPAIGN_FINANCE TO NCCF_DEV;
+GRANT ALL PRIVILEGES ON SCHEMA NC_CAMPAIGN_FINANCE.NCCF TO NCCF_DEV;
+GRANT ALL PRIVILEGES ON WAREHOUSE NCCF_DEV_WH to NCCF_DEV;
+GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE NC_CAMPAIGN_FINANCE TO NCCF_DEV;
+GRANT ALL PRIVILEGES ON FUTURE TABLES IN SCHEMA NC_CAMPAIGN_FINANCE.NCCF to NCCF_DEV;
+GRANT ALL PRIVILEGES ON FUTURE VIEWS IN SCHEMA NC_CAMPAIGN_FINANCE.NCCF to NCCF_DEV;
+
+GRANT ROLE NCCF_DEV to USER JIMMY; # add your username
+
+USE ROLE USERADMIN;
+
+CREATE USER NCCF_SVC PASSWORD = 'insert-really-good-password-here' DEFAULT_ROLE = "NCCF_DEV" DEFAULT_WAREHOUSE = 'NCCF_DEV_WH' DEFAULT_NAMESPACE = 'NC_CAMPAIGN_FINANCE.NCCF' MUST_CHANGE_PASSWORD = TRUE;
+
+USE ROLE SECURITYADMIN;
+GRANT ROLE "NCCF_DEV" to USER NCCF_SVC;

packages/snowflake/queries/ddl/CREATE_INGESTED_TRANSACTIONS_TABLE.SQL

@@ -0,0 +1,31 @@
+USE DATABASE NC_CAMPAIGN_FINANCE;
+USE SCHEMA NCCF;
+USE ROLE NCCF_DEV;
+USE WAREHOUSE NCCF_DEV_WH;
+
+CREATE OR ALTER TABLE INGESTED_TRANSACTIONS (
+    NAME VARCHAR,
+    STREET_LINE_1 VARCHAR,
+    STREET_LINE_2 VARCHAR,
+    CITY VARCHAR,
+    STATE VARCHAR,
+    ZIP VARCHAR,
+    PROFESSION_JOB_TITLE VARCHAR,
+    EMPLOYERS_NAME_SPECIFIC_FIELD VARCHAR,
+    TRANSACTION_TYPE VARCHAR(5),
+    COMMITTEE_NAME VARCHAR,
+    COMMITTEE_SBOE_ID VARCHAR,
+    COMMITTEE_STREET_1 VARCHAR,
+    COMMITTEE_STREET_2 VARCHAR,
+    COMMITTEE_CITY VARCHAR,
+    COMMITTEE_STATE VARCHAR,
+    COMMITTEE_ZIP_CODE VARCHAR,
+    REPORT_NAME VARCHAR,
+    DATE_OCCURED VARCHAR,
+    ACCOUNT_CODE VARCHAR,
+    AMOUNT VARCHAR,
+    FORM_OF_PAYMENT VARCHAR,
+    PURPOSE VARCHAR,
+    CANDIDATE_REFERENDUM_NAME VARCHAR,
+    DECLARATION VARCHAR
+    );

packages/snowflake/scripts/login.bat

@@ -0,0 +1 @@
+ snowsql -a vx50048.us-central1.gcp -u nccf_svc -d nc_campaign_finance_dev -s nccf