From 85a09b144c26270dd85e3c4aec633ff72409d267 Mon Sep 17 00:00:00 2001 From: Brian Lovin Date: Tue, 4 Dec 2018 11:21:04 -0800 Subject: [PATCH 1/2] Remove getspectrum as a toxicity check provider --- api/mutations/thread/publishThread.js | 11 ++--- athena/queues/moderationEvents/message.js | 15 ++----- athena/queues/moderationEvents/perspective.js | 2 +- athena/queues/moderationEvents/spectrum.js | 42 ------------------- athena/queues/moderationEvents/thread.js | 12 +----- email-templates/adminToxicContent.html | 4 -- .../queues/send-admin-toxic-content-email.js | 11 +---- now.json | 2 +- 8 files changed, 12 insertions(+), 87 deletions(-) delete mode 100644 athena/queues/moderationEvents/spectrum.js diff --git a/api/mutations/thread/publishThread.js b/api/mutations/thread/publishThread.js index 42c5156a69..e505995edc 100644 --- a/api/mutations/thread/publishThread.js +++ b/api/mutations/thread/publishThread.js @@ -19,7 +19,6 @@ import { _adminProcessToxicThreadQueue, _adminProcessUserSpammingThreadsQueue, } from 'shared/bull/queues'; -import getSpectrumScore from 'athena/queues/moderationEvents/spectrum'; import getPerspectiveScore from 'athena/queues/moderationEvents/perspective'; import { events } from 'shared/analytics'; import { trackQueue } from 'shared/bull/queues'; @@ -275,27 +274,23 @@ export default requireAuth( const title = thread.content.title; const text = `${title} ${body}`; - const scores = await Promise.all([ - getSpectrumScore(text, dbThread.id, dbThread.creatorId).catch(err => 0), - getPerspectiveScore(text).catch(err => 0), - ]).catch(err => + const scores = await getPerspectiveScore(text).catch(err => console.error( 'Error getting thread moderation scores from providers', err.message ) ); - const spectrumScore = scores && scores[0]; const perspectiveScore = scores && scores[1]; // if neither models returned results - if (!spectrumScore && !perspectiveScore) { + if (!perspectiveScore) { debug('Toxicity checks from providers say not toxic'); return false; } // if both services agree that the thread is >= 98% toxic - if ((spectrumScore + perspectiveScore) / 2 >= 0.9) { + if (perspectiveScore >= 0.9) { debug('Thread is toxic according to both providers'); return true; } diff --git a/athena/queues/moderationEvents/message.js b/athena/queues/moderationEvents/message.js index a091c91e79..6713ad308e 100644 --- a/athena/queues/moderationEvents/message.js +++ b/athena/queues/moderationEvents/message.js @@ -5,7 +5,6 @@ import { getThreadById } from '../../models/thread'; import { getCommunityById } from '../../models/community'; import { getChannelById } from '../../models/channel'; import { toState, toPlainText } from 'shared/draft-utils'; -import getSpectrumScore from './spectrum'; import getPerspectiveScore from './perspective'; import { _adminSendToxicContentEmailQueue } from 'shared/bull/queues'; import type { Job, AdminToxicMessageJobData } from 'shared/bull/types'; @@ -21,11 +20,8 @@ export default async (job: Job) => { ? toPlainText(toState(JSON.parse(message.content.body))) : message.content.body; - const scores = await Promise.all([ - getSpectrumScore(text, message.id, message.senderId), - getPerspectiveScore(text), - ]).catch(err => - console.error('Error getting message moderation scores from providers', { + const perspectiveScore = await getPerspectiveScore(text).catch(err => + console.error('Error getting message moderation score from providers', { error: err.message, data: { text, @@ -34,11 +30,7 @@ export default async (job: Job) => { }) ); - const spectrumScore = scores && scores[0]; - const perspectiveScore = scores && scores[1]; - - // if neither models returned results - if (!spectrumScore && !perspectiveScore) return; + if (!perspectiveScore) return; const [user, thread] = await Promise.all([ getUserById(message.senderId), @@ -58,7 +50,6 @@ export default async (job: Job) => { community, channel, toxicityConfidence: { - spectrumScore, perspectiveScore, }, }); diff --git a/athena/queues/moderationEvents/perspective.js b/athena/queues/moderationEvents/perspective.js index a4a41a5b43..ed59299092 100644 --- a/athena/queues/moderationEvents/perspective.js +++ b/athena/queues/moderationEvents/perspective.js @@ -29,7 +29,7 @@ export default async (text: string) => { }); // if something failed? - if (!request || !request.data) return; + if (!request || !request.data) return null; // get the scores from the request const { attributeScores } = request.data; diff --git a/athena/queues/moderationEvents/spectrum.js b/athena/queues/moderationEvents/spectrum.js deleted file mode 100644 index 13613b4471..0000000000 --- a/athena/queues/moderationEvents/spectrum.js +++ /dev/null @@ -1,42 +0,0 @@ -// @flow -const debug = require('debug')('athena:queue:moderation-events:spectrum'); -require('now-env'); -import axios from 'axios'; -const SPECTRUM_MODERATION_API_KEY = process.env.SPECTRUM_MODERATION_API_KEY; - -if (!SPECTRUM_MODERATION_API_KEY) { - debug('No API key for Spectrum provided, not sending moderation events.'); -} - -export default async (text: string, contextId: string, userId: string) => { - if (!SPECTRUM_MODERATION_API_KEY) return; - const request = await axios({ - method: 'post', - url: 'https://api.prod.getspectrum.io/api/v1/classification', - headers: { - Authorization: `Apikey ${SPECTRUM_MODERATION_API_KEY}`, - 'Content-Type': 'application/json', - }, - data: { - jsonrpc: '2.0', - method: 'classifyText', - params: { - text: text, - meta: { - authorId: userId, - }, - }, - id: contextId, - }, - }); - - const { data } = request; - - if (!data || !data.result) return; - - const { toxicityConfidence } = data.result; - - if (toxicityConfidence > 0.9) return toxicityConfidence; - - return null; -}; diff --git a/athena/queues/moderationEvents/thread.js b/athena/queues/moderationEvents/thread.js index 938cce46d0..7815637fc2 100644 --- a/athena/queues/moderationEvents/thread.js +++ b/athena/queues/moderationEvents/thread.js @@ -4,7 +4,6 @@ import { getUserById } from 'shared/db/queries/user'; import { getCommunityById } from '../../models/community'; import { getChannelById } from '../../models/channel'; import { toState, toPlainText } from 'shared/draft-utils'; -import getSpectrumScore from './spectrum'; import getPerspectiveScore from './perspective'; import { _adminSendToxicContentEmailQueue } from 'shared/bull/queues'; import type { Job, AdminToxicThreadJobData } from 'shared/bull/types'; @@ -26,10 +25,7 @@ export default async (job: Job) => { const title = thread.content.title; const text = `${title} ${body}`; - const scores = await Promise.all([ - getSpectrumScore(text, thread.id, thread.creatorId), - getPerspectiveScore(text), - ]).catch(err => + const perspectiveScore = await getPerspectiveScore(text).catch(err => console.error('Error getting thread moderation scores from providers', { error: err.message, data: { @@ -39,11 +35,8 @@ export default async (job: Job) => { }) ); - const spectrumScore = scores && scores[0]; - const perspectiveScore = scores && scores[1]; - // if neither models returned results - if (!spectrumScore && !perspectiveScore) return; + if (!perspectiveScore) return; const [user, community, channel] = await Promise.all([ getUserById(thread.creatorId), @@ -59,7 +52,6 @@ export default async (job: Job) => { community, channel, toxicityConfidence: { - spectrumScore, perspectiveScore, }, }); diff --git a/email-templates/adminToxicContent.html b/email-templates/adminToxicContent.html index 76aa140d88..115323f1f4 100644 --- a/email-templates/adminToxicContent.html +++ b/email-templates/adminToxicContent.html @@ -31,10 +31,6 @@

Info:

    - {{#data.toxicityConfidence.spectrumPercent}} -
  • Spectrum: {{.}}% confident
    • - {{/data.toxicityConfidence.spectrumPercent}} - {{#data.toxicityConfidence.perspectivePercent}}
  • Perspective: {{.}}% confident
    • {{/data.toxicityConfidence.perspectivePercent}} diff --git a/hermes/queues/send-admin-toxic-content-email.js b/hermes/queues/send-admin-toxic-content-email.js index 3b088c7091..23cdd89572 100644 --- a/hermes/queues/send-admin-toxic-content-email.js +++ b/hermes/queues/send-admin-toxic-content-email.js @@ -17,20 +17,14 @@ export default job => { thread, community, channel, - toxicityConfidence: { spectrumScore, perspectiveScore }, + toxicityConfidence: { perspectiveScore }, } = job.data; const toPercent = (num: number) => Math.round(num * 100); - const spectrumPercent = spectrumScore ? toPercent(spectrumScore) : null; const perspectivePercent = perspectiveScore ? toPercent(perspectiveScore) : null; - let avgPercent; - if (spectrumPercent && perspectivePercent) { - avgPercent = (spectrumPercent + perspectivePercent) / 2; - } else { - avgPercent = spectrumPercent || perspectivePercent || 0; - } + let avgPercent = perspectivePercent; const subject = `Toxic alert (${avgPercent.toString()}%): ${text}`; @@ -50,7 +44,6 @@ export default job => { community, channel, toxicityConfidence: { - spectrumPercent, perspectivePercent, }, }, diff --git a/now.json b/now.json index a9da14b8d3..017bf7860f 100644 --- a/now.json +++ b/now.json @@ -46,7 +46,7 @@ "API_TOKEN_SECRET": "@api-token-secret", "SENTRY_DSN_CLIENT": "@sentry-dsn-client", "SENTRY_DSN_SERVER": "@sentry-dsn-server", - "SPECTRUM_MODERATION_API_KEY": "@spectrum-moderation-api-key", + "AMPLITUDE_API_KEY": "@amplitude-api-key", "AMPLITUDE_API_KEY_DEVELOPMENT": "@amplitude-api-key-development", "ENCRYPTION_KEY": "@encryption-key", From 22d32d58ebb9a8e494ea331370d1e5263db49226 Mon Sep 17 00:00:00 2001 From: Brian Lovin Date: Tue, 4 Dec 2018 12:30:28 -0800 Subject: [PATCH 2/2] Fix flow --- hermes/queues/send-admin-toxic-content-email.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hermes/queues/send-admin-toxic-content-email.js b/hermes/queues/send-admin-toxic-content-email.js index 23cdd89572..a0a3a2f66e 100644 --- a/hermes/queues/send-admin-toxic-content-email.js +++ b/hermes/queues/send-admin-toxic-content-email.js @@ -21,12 +21,10 @@ export default job => { } = job.data; const toPercent = (num: number) => Math.round(num * 100); - const perspectivePercent = perspectiveScore - ? toPercent(perspectiveScore) - : null; - let avgPercent = perspectivePercent; - const subject = `Toxic alert (${avgPercent.toString()}%): ${text}`; + const perspectivePercent = perspectiveScore.toPercent(perspectiveScore); + + const subject = `Toxic alert (${perspectivePercent.toString()}%): ${text}`; try { return sendEmail({