Merge pull request #248 from concord-consortium/187653588-remove-firs…

…t-username-in-expanded-log-report feat: Remove log data username in wide learner logs [PT-187653588]
concord-consortium · May 23, 2024 · 2bf6ff6 · 2bf6ff6
2 parents ffc3616 + 5f014ed
commit 2bf6ff6
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 3 deletions.
diff --git a/query-creator/create-query/steps/aws.js b/query-creator/create-query/steps/aws.js
@@ -552,9 +552,10 @@ exports.generateSQL = (runnableInfo, usageReport, authDomain, sourceKey, hideNam
     GROUP BY l.run_remote_endpoint )`
 */
 
-const getLogCols = (hideNames) => {
+const getLogCols = (hideNames, removeUsername) => {
   return ["id", "session", "username", "application", "activity", "event", "event_value", "time", "parameters", "extras", "run_remote_endpoint", "timestamp"]
     .map(col => `"log"."${col}"`)
+    .filter(col => col === `"log"."username"` && removeUsername ? false : true)
     .map(col => col === `"log"."username"` ? maybeHashUsername(hideNames, `"log"."username"`) : col)
 }
 
@@ -573,7 +574,7 @@ exports.generateLearnerLogSQL = (queryIdsPerRunnable, hideNames) => {
   const runnableUrls = Object.keys(queryIdsPerRunnable);
   const queryIds = Object.values(queryIdsPerRunnable);
 
-  const logCols = getLogCols(hideNames)
+  const logCols = getLogCols(hideNames, true) // remove duplicate username in log columns
   const learnerCols = getLearnerCols(hideNames)
 
   const cols = logCols.concat(learnerCols).join(", ")
@@ -621,6 +622,7 @@ exports.generateUserLogSQL = (usernames, activities, start_date, end_date) => {
   -- reportType user-event-log
   -- usernames: ${JSON.stringify(usernames)}
   -- activities: ${JSON.stringify(activities)}
+
   SELECT *
   FROM "${logDb}"."logs_by_time" log
   WHERE ${where.join(" AND ")}

diff --git a/query-creator/create-query/tests/unit/generated-sql.js b/query-creator/create-query/tests/unit/generated-sql.js
@@ -816,4 +816,82 @@ FROM
   FROM "report-service"."learners" l
   WHERE l.query_id IN ('123456789')
   GROUP BY l.run_remote_endpoint )
+`
+
+exports.expectedUserLogSQLresult = `
+-- name log.username IN ('[email protected]', '[email protected]') AND log.activity IN ('activity-1', 'activity-2') AND time >= 1704171600 AND time <= 1709528400
+-- type user event log
+-- reportType user-event-log
+-- usernames: ["[email protected]","[email protected]"]
+-- activities: ["activity-1","activity-2"]
+
+SELECT *
+FROM "undefined"."logs_by_time" log
+WHERE log.username IN ('[email protected]', '[email protected]') AND log.activity IN ('activity-1', 'activity-2') AND time >= 1704171600 AND time <= 1709528400
+`
+
+exports.expectedNarrowLearnerLogWithNamesSqlResult = `
+-- name 0, 1
+-- type learner event log ⎯ [qids: qid_1, qid_2]
+-- reportType narrow-learner-event-log
+
+SELECT "log"."id", "log"."session", "log"."username", "log"."application", "log"."activity", "log"."event", "log"."event_value", "log"."time", "log"."parameters", "log"."extras", "log"."run_remote_endpoint", "log"."timestamp"
+FROM "undefined"."logs_by_time" log
+INNER JOIN "report-service"."learners" learner
+ON
+  (
+    learner.query_id IN ('qid_1','qid_2')
+    AND
+    learner.run_remote_endpoint = log.run_remote_endpoint
+  )
+`
+
+exports.expectedNarrowLearnerLogWithoutNamesSqlResult = `
+-- name 0, 1
+-- type learner event log ⎯ [qids: qid_1, qid_2]
+-- reportType narrow-learner-event-log
+
+SELECT "log"."id", "log"."session", to_hex(sha1(cast(('no-username-salt-provided' || "log"."username") as varbinary))) as username, "log"."application", "log"."activity", "log"."event", "log"."event_value", "log"."time", "log"."parameters", "log"."extras", "log"."run_remote_endpoint", "log"."timestamp"
+FROM "undefined"."logs_by_time" log
+INNER JOIN "report-service"."learners" learner
+ON
+  (
+    learner.query_id IN ('qid_1','qid_2')
+    AND
+    learner.run_remote_endpoint = log.run_remote_endpoint
+  )
+`
+
+exports.expectedWideLearnerLogWithNameSqlResult = `
+-- name qid_1, qid_2
+-- type learner event log ⎯ [qids: http://example.com/runnable_1, http://example.com/runnable_2]
+-- reportType learner-event-log
+-- hideNames false
+
+SELECT "log"."id", "log"."session", "log"."application", "log"."activity", "log"."event", "log"."event_value", "log"."time", "log"."parameters", "log"."extras", "log"."run_remote_endpoint", "log"."timestamp", "learner"."learner_id", "learner"."run_remote_endpoint", "learner"."class_id", "learner"."runnable_url", "learner"."student_id", "learner"."class", "learner"."school", "learner"."user_id", "learner"."offering_id", "learner"."permission_forms", "learner"."username", "learner"."student_name", "learner"."teachers", "learner"."last_run", "learner"."query_id"
+FROM "undefined"."logs_by_time" log
+INNER JOIN "report-service"."learners" learner
+ON
+  (
+    learner.query_id IN ('http://example.com/runnable_1','http://example.com/runnable_2')
+    AND
+    learner.run_remote_endpoint = log.run_remote_endpoint
+  )
+`
+
+exports.expectedWideLearnerLogWithoutNamesSqlResult = `
+-- name qid_1, qid_2
+-- type learner event log ⎯ [qids: http://example.com/runnable_1, http://example.com/runnable_2]
+-- reportType learner-event-log
+-- hideNames true
+
+SELECT "log"."id", "log"."session", "log"."application", "log"."activity", "log"."event", "log"."event_value", "log"."time", "log"."parameters", "log"."extras", "log"."run_remote_endpoint", "log"."timestamp", "learner"."learner_id", "learner"."run_remote_endpoint", "learner"."class_id", "learner"."runnable_url", "learner"."student_id", "learner"."class", "learner"."school", "learner"."user_id", "learner"."offering_id", "learner"."permission_forms", to_hex(sha1(cast(('no-username-salt-provided' || "learner"."username") as varbinary))) as username, "learner"."student_id" as student_name, "learner"."teachers", "learner"."last_run", "learner"."query_id"
+FROM "undefined"."logs_by_time" log
+INNER JOIN "report-service"."learners" learner
+ON
+  (
+    learner.query_id IN ('http://example.com/runnable_1','http://example.com/runnable_2')
+    AND
+    learner.run_remote_endpoint = log.run_remote_endpoint
+  )
 `
diff --git a/query-creator/create-query/tests/unit/test-handler.js b/query-creator/create-query/tests/unit/test-handler.js
@@ -7,7 +7,11 @@ const chai = require('chai');
 const queryString = require('query-string');
 const nock = require('nock');
 
-const { expectedDetailsReportWithNamesSQLresult, expectedDetailsReportHideNamesSQLresult, expectedUsageReportWithNamesSQLresult, expectedUsageReportWithoutNamesSQLresult, expectedNoRunnableWithNamesSQLresult, expectedNoRunnableWithoutNamesSQLresult } = require('./generated-sql.js');
+const { expectedDetailsReportWithNamesSQLresult, expectedDetailsReportHideNamesSQLresult, expectedUsageReportWithNamesSQLresult,
+        expectedUsageReportWithoutNamesSQLresult, expectedNoRunnableWithNamesSQLresult, expectedNoRunnableWithoutNamesSQLresult, expectedUserLogSQLresult,
+        expectedNarrowLearnerLogWithNamesSqlResult, expectedNarrowLearnerLogWithoutNamesSqlResult, expectedWideLearnerLogWithoutNamesSqlResult,
+        expectedWideLearnerLogWithNameSqlResult
+      } = require('./generated-sql.js');
 
 const expect = chai.expect;
 const event = {
@@ -236,3 +240,41 @@ describe('Query creation unreportable runnable', function () {
     expect(normalizeSQL(expectedNoRunnableWithNamesSQLresult)).not.to.be.equal(normalizeSQL(expectedNoRunnableWithoutNamesSQLresult));
   })
 });
+
+describe('Query creation for log data', function () {
+
+  it('verifies successful query creation of user log data', async () => {
+    const generatedSQLresult = await aws.generateUserLogSQL(["[email protected]", "[email protected]"], ["activity-1", "activity-2"], "1/2/2024", "3/4/2024");
+    const normalizedGeneratedSQLresult = normalizeSQL(generatedSQLresult);
+    const normalizedExpectedSQLresult = normalizeSQL(expectedUserLogSQLresult);
+    expect(normalizedGeneratedSQLresult).to.be.equal(normalizedExpectedSQLresult);
+  });
+
+  it('verifies successful query creation of narrow learner log data with names', async () => {
+    const generatedSQLresult = await aws.generateNarrowLogSQL(["qid_1", "qid_2"], false);
+    const normalizedGeneratedSQLresult = normalizeSQL(generatedSQLresult);
+    const normalizedExpectedSQLresult = normalizeSQL(expectedNarrowLearnerLogWithNamesSqlResult);
+    expect(normalizedGeneratedSQLresult).to.be.equal(normalizedExpectedSQLresult);
+  });
+
+  it('verifies successful query creation of narrow learner log data without names', async () => {
+    const generatedSQLresult = await aws.generateNarrowLogSQL(["qid_1", "qid_2"], true);
+    const normalizedGeneratedSQLresult = normalizeSQL(generatedSQLresult);
+    const normalizedExpectedSQLresult = normalizeSQL(expectedNarrowLearnerLogWithoutNamesSqlResult);
+    expect(normalizedGeneratedSQLresult).to.be.equal(normalizedExpectedSQLresult);
+  });
+
+  it('verifies successful query creation of wide learner log data with names', async () => {
+    const generatedSQLresult = await aws.generateLearnerLogSQL({"qid_1": "http://example.com/runnable_1", "qid_2": "http://example.com/runnable_2"}, false);
+    const normalizedGeneratedSQLresult = normalizeSQL(generatedSQLresult);
+    const normalizedExpectedSQLresult = normalizeSQL(expectedWideLearnerLogWithNameSqlResult);
+    expect(normalizedGeneratedSQLresult).to.be.equal(normalizedExpectedSQLresult);
+  });
+
+  it('verifies successful query creation of wide learner log data without names', async () => {
+    const generatedSQLresult = await aws.generateLearnerLogSQL({"qid_1": "http://example.com/runnable_1", "qid_2": "http://example.com/runnable_2"}, true);
+    const normalizedGeneratedSQLresult = normalizeSQL(generatedSQLresult);
+    const normalizedExpectedSQLresult = normalizeSQL(expectedWideLearnerLogWithoutNamesSqlResult);
+    expect(normalizedGeneratedSQLresult).to.be.equal(normalizedExpectedSQLresult);
+  });
+});