Gian dw poc

.github/workflows/hrm-lambda-deploy.yml

@@ -34,6 +34,7 @@ on:
           - hrm-domain/lambdas/files-urls
           - hrm-domain/lambdas/contact-retrieve-transcript
           - hrm-domain/lambdas/search-index-consumer
+          - lambdas/etl-twilio
           - lambdas/job-complete
           - resources-domain/lambdas/import-consumer
           - resources-domain/lambdas/import-producer

lambdas/etl-twilio/index.ts

@@ -0,0 +1,57 @@
+import type { HrmAccountId } from '@tech-matters/types';
+import { loadSsmCache, ssmCache } from '@tech-matters/ssm-cache';
+import { exportTaskrouterEvents } from './taskrouter';
+import { exportStudioExecutions } from './studio';
+import { exportDailyUsage } from './usage';
+
+const dateLakeBucketName = process.env.DL_BUCKET || '';
+const parentPath = 'twilio';
+
+const ssmCacheConfigs = [
+  {
+    path: `/${process.env.NODE_ENV}/twilio`,
+    regex: /\/.*\/account_sid/,
+  },
+];
+
+const loadSsmCacheConfig = async () => {
+  await loadSsmCache({ configs: ssmCacheConfigs });
+};
+
+export const handler = async (event: any) => {
+  console.log(event);
+  const startDate = new Date(event.startDate);
+  const endDate = new Date(event.endDate);
+
+  await loadSsmCacheConfig();
+
+  const accountSids = Object.entries(ssmCache.values)
+    .filter(([key]) => key.includes('/account_sid'))
+    .map(([, v]) => v!.value);
+
+  console.log('accountSids: ', accountSids);
+
+  for (const accountSid of accountSids) {
+    console.log('accountSid: ', accountSid);
+
+    const exportParams = {
+      accountSid: accountSid as HrmAccountId,
+      endDate,
+      startDate,
+      dateLakeBucketName,
+      parentPath,
+    };
+
+    await Promise.all([
+      exportTaskrouterEvents(exportParams),
+      exportStudioExecutions(exportParams),
+      exportDailyUsage(exportParams),
+    ]);
+  }
+
+  return {
+    statusCode: 200,
+    headers: { 'Content-Type': 'text/plain' },
+    body: `testing...`,
+  };
+};

lambdas/etl-twilio/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@tech-matters/etl-twilio",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "author": "",
+  "license": "AGPL",
+  "scripts": {
+    "test": "echo 'passed'"
+  },
+  "devDependencies": {
+    "@tech-matters/types": "^1.0.0",
+    "@tsconfig/node16": "^1.0.4",
+    "@types/aws-lambda": "^8.10.144"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/techmatters/hrm.git"
+  },
+  "bugs": {
+    "url": "https://github.com/techmatters/hrm/issues"
+  },
+  "homepage": "https://github.com/techmatters/hrm#readme",
+  "description": "",
+  "dependencies": {
+    "@tech-matters/s3-client": "^1.0.0",
+    "@tech-matters/ssm-cache": "^1.0.0",
+    "@tech-matters/twilio-client": "^1.0.0"
+  }
+}

lambdas/etl-twilio/studio.ts

@@ -0,0 +1,40 @@
+import type { HrmAccountId } from '@tech-matters/types';
+import { getClient } from '@tech-matters/twilio-client';
+import { putS3Object } from '@tech-matters/s3-client';
+
+export const exportStudioExecutions = async ({
+  accountSid,
+  startDate,
+  endDate,
+  dateLakeBucketName,
+  parentPath,
+}: {
+  accountSid: HrmAccountId;
+  startDate: Date;
+  endDate: Date;
+  dateLakeBucketName: string;
+  parentPath: string;
+}) => {
+  const client = await getClient({ accountSid });
+
+  const flows = await client.studio.v2.flows.list();
+  const executions = (
+    await Promise.all(
+      flows.map(f =>
+        f.executions().list({ dateCreatedFrom: startDate, dateCreatedTo: endDate }),
+      ),
+    )
+  ).flat();
+
+  await Promise.all(
+    executions.map(e => {
+      const { dateCreated, sid } = e;
+      const key = `${parentPath}/studio/${accountSid}/${dateCreated.toISOString()}-${sid}`;
+      return putS3Object({
+        bucket: dateLakeBucketName,
+        key,
+        body: JSON.stringify(e),
+      });
+    }),
+  );
+};

lambdas/etl-twilio/taskrouter.ts

@@ -0,0 +1,40 @@
+import type { HrmAccountId } from '@tech-matters/types';
+import { getClient } from '@tech-matters/twilio-client';
+import { putS3Object } from '@tech-matters/s3-client';
+
+export const exportTaskrouterEvents = async ({
+  accountSid,
+  startDate,
+  endDate,
+  dateLakeBucketName,
+  parentPath,
+}: {
+  accountSid: HrmAccountId;
+  startDate: Date;
+  endDate: Date;
+  dateLakeBucketName: string;
+  parentPath: string;
+}) => {
+  const client = await getClient({ accountSid });
+
+  const workspaces = await client.taskrouter.v1.workspaces.list();
+  const events = (
+    await Promise.all(
+      workspaces.map(w =>
+        client.taskrouter.v1.workspaces(w.sid).events.list({ startDate, endDate }),
+      ),
+    )
+  ).flat();
+
+  await Promise.all(
+    events.map(e => {
+      const { eventDate, sid } = e;
+      const key = `${parentPath}/taskrouter/${accountSid}/${eventDate.toISOString()}-${sid}`;
+      return putS3Object({
+        bucket: dateLakeBucketName,
+        key,
+        body: JSON.stringify(e),
+      });
+    }),
+  );
+};

lambdas/etl-twilio/tsconfig.build.json

@@ -0,0 +1,11 @@
+// This file is copied to the root at build time, so all paths are relative to the root
+{
+  "extends": "./tsconfig.base.json",
+  "files": [],
+  "references": [
+    { "path": "packages/ssm-cache" },
+    { "path": "packages/s3-client" },
+    { "path": "packages/twilio-client" },
+    { "path": "lambdas/etl-twilio" }
+  ]
+}

lambdas/etl-twilio/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "include": ["*.ts"],
+  "compilerOptions": {
+    "outDir": "./dist",
+    "moduleResolution": "Node16"
+  }
+}

lambdas/etl-twilio/usage.ts

@@ -0,0 +1,35 @@
+import type { HrmAccountId } from '@tech-matters/types';
+import { getClient } from '@tech-matters/twilio-client';
+import { putS3Object } from '@tech-matters/s3-client';
+
+export const exportDailyUsage = async ({
+  accountSid,
+  startDate,
+  endDate,
+  dateLakeBucketName,
+  parentPath,
+}: {
+  accountSid: HrmAccountId;
+  startDate: Date;
+  endDate: Date;
+  dateLakeBucketName: string;
+  parentPath: string;
+}) => {
+  const client = await getClient({ accountSid });
+
+  const usage = (await client.usage.records?.daily?.list({ startDate, endDate }))?.filter(
+    u => parseInt(u.count, 10) > 0,
+  );
+
+  await Promise.all(
+    (usage || []).map(u => {
+      const { startDate: from, endDate: to, category } = u;
+      const key = `${parentPath}/usage/${accountSid}/${from.toISOString()}-${to.toISOString()}-${category}`;
+      return putS3Object({
+        bucket: dateLakeBucketName,
+        key,
+        body: JSON.stringify(u),
+      });
+    }),
+  );
+};

lambdas/tsconfig.json

@@ -2,7 +2,7 @@
   "extends": "@tsconfig/node16/tsconfig.json",
   "compilerOptions": {
     "esModuleInterop": true,
-    "lib": ["ES2020"],
+    "lib": ["ES2022"],
     "strict": true,
     "allowSyntheticDefaultImports": true,
     "allowJs": true,

tsconfig.json

@@ -17,6 +17,7 @@
     { "path": "packages/service-discovery" },
     { "path": "packages/process" },
     { "path": "lambdas/packages/hrm-authentication" },
+    { "path": "lambdas/etl-twilio" },
     { "path": "lambdas/job-complete" },
     { "path": "hrm-domain/packages/hrm-types" },
     { "path": "hrm-domain/packages/hrm-search-config" },