diff --git a/bin/migrate-down-all.sh b/bin/migrate-down-all.sh index c3e49ba00..e9903fc95 100644 --- a/bin/migrate-down-all.sh +++ b/bin/migrate-down-all.sh @@ -1,4 +1,5 @@ #!/bin/sh +psql $DATABASE_URL -U eyeshade --single-transaction -v ON_ERROR_STOP=1 -f ./migrations/0020_remove_potential_payments_ads/down.sql psql $DATABASE_URL -U eyeshade --single-transaction -v ON_ERROR_STOP=1 -f ./migrations/0019_add_inserted_at/down.sql psql $DATABASE_URL -U eyeshade --single-transaction -v ON_ERROR_STOP=1 -f ./migrations/0018_add_snapshot_index/down.sql psql $DATABASE_URL -U eyeshade --single-transaction -v ON_ERROR_STOP=1 -f ./migrations/0017_include_indexes/down.sql diff --git a/eyeshade/migrations/0008_on_delete_cascade/up.sql b/eyeshade/migrations/0008_on_delete_cascade/up.sql index e8e5c0bbd..49771cfe7 100644 --- a/eyeshade/migrations/0008_on_delete_cascade/up.sql +++ b/eyeshade/migrations/0008_on_delete_cascade/up.sql @@ -3,7 +3,7 @@ select execute($$ insert into migrations (id, description) values ('0008', 'on_delete_cascade'); ALTER TABLE potential_payments_ads - DROP CONSTRAINT potential_payments_ads_payout_report_id_fkey; + DROP CONSTRAINT if exists potential_payments_ads_payout_report_id_fkey; ALTER TABLE potential_payments_ads ADD CONSTRAINT cascade_payout_report_id diff --git a/eyeshade/migrations/0009_unique_payment_ids/down.sql b/eyeshade/migrations/0009_unique_payment_ids/down.sql index 8b39a29b3..4bca7d318 100644 --- a/eyeshade/migrations/0009_unique_payment_ids/down.sql +++ b/eyeshade/migrations/0009_unique_payment_ids/down.sql @@ -3,7 +3,7 @@ select execute($$ delete from migrations where id = '0009'; alter table potential_payments_ads - drop constraint unique_payment_id_payout_report_id; + drop constraint if exists unique_payment_id_payout_report_id; $$) where exists (select * from migrations where id = '0009'); diff --git a/eyeshade/migrations/0019_add_inserted_at/down.sql b/eyeshade/migrations/0019_add_inserted_at/down.sql index 5c136f231..a746cf5da 100644 --- a/eyeshade/migrations/0019_add_inserted_at/down.sql +++ b/eyeshade/migrations/0019_add_inserted_at/down.sql @@ -3,8 +3,6 @@ select execute($$ alter table if exists transactions drop column if exists inserted_at; -drop index if exists transactions_inserted_at; - delete from migrations where id = '0019'; $$) where exists (select * from migrations where id = '0019'); diff --git a/eyeshade/migrations/0020_remove_potential_payments_ads/down.sql b/eyeshade/migrations/0020_remove_potential_payments_ads/down.sql new file mode 100644 index 000000000..eb3e86f7f --- /dev/null +++ b/eyeshade/migrations/0020_remove_potential_payments_ads/down.sql @@ -0,0 +1,19 @@ +select execute($$ + +delete from migrations where id = '0020'; + +create table payout_reports_ads( + id uuid primary key default uuid_generate_v4(), + created_at timestamp with time zone not null default current_timestamp +); + +create table potential_payments_ads( + id uuid primary key default uuid_generate_v4(), + payout_report_id uuid not null references payout_reports_ads(id), + payment_id uuid not null, + provider_id uuid not null, + amount numeric(28, 18) not null check (amount > 0.0), + created_at timestamp with time zone not null default current_timestamp +); + +$$) where exists (select * from migrations where id = '0020'); diff --git a/eyeshade/migrations/0020_remove_potential_payments_ads/up.sql b/eyeshade/migrations/0020_remove_potential_payments_ads/up.sql new file mode 100644 index 000000000..eb22ce474 --- /dev/null +++ b/eyeshade/migrations/0020_remove_potential_payments_ads/up.sql @@ -0,0 +1,7 @@ +select execute($$ + +drop table potential_payments_ads; +drop table payout_reports_ads; +insert into migrations (id, description) values ('0020', 'remove_potential_payments_ads'); + +$$) where not exists (select * from migrations where id = '0020'); diff --git a/eyeshade/worker.js b/eyeshade/worker.js index 409c888c8..fa9c38ba2 100644 --- a/eyeshade/worker.js +++ b/eyeshade/worker.js @@ -8,7 +8,6 @@ const referralsWorker = require('./workers/referrals') const reportsWorker = require('./workers/reports') const surveyorsWorker = require('./workers/surveyors') const walletWorker = require('./workers/wallet') -const adsWorker = require('./workers/ads') const snapshotsWorker = require('./workers/snapshots') const { @@ -30,7 +29,6 @@ const parentModules = [ reportsWorker, surveyorsWorker, walletWorker, - adsWorker, snapshotsWorker ] diff --git a/eyeshade/workers/ads.js b/eyeshade/workers/ads.js deleted file mode 100644 index 3ccf79122..000000000 --- a/eyeshade/workers/ads.js +++ /dev/null @@ -1,65 +0,0 @@ -const cron = require('cron-parser') -const uuidV4 = require('uuid/v4') -const underscore = require('underscore') - -const createPayoutReportQuery = 'insert into payout_reports_ads (id) values ($1)' - -const selectWalletBalancesQuery = ` - with ads_balances as ( - select - account_id, - sum(amount) as balance - from account_transactions - where account_type = 'payment_id' - and created_at < date_trunc('month', current_date) - group by account_id - order by balance desc - ) - select - account_id, - balance - from ads_balances - where balance > 0 -` - -const createPotentialPaymentsQuery = 'insert into potential_payments_ads (payout_report_id, payment_id, provider_id, amount) values ($1, $2, $3, $4)' - -// Takes a snapshot of ad account balances -// and inserts them into potential_payments -const monthly = async (debug, runtime) => { - const client = await runtime.postgres.connect() - const walletsCollection = runtime.database.get('wallets', debug) - const payoutReportId = uuidV4() - - try { - await client.query('BEGIN') - // First create the payout report - await runtime.postgres.query(createPayoutReportQuery, [payoutReportId], client) - // Next get all the payment_id, balance pairs for all the wallets - const walletBalances = (await runtime.postgres.query(selectWalletBalancesQuery, [], client)).rows - // Now insert the balance snapshots as potential ads payments - for (let i = 0; i < walletBalances.length; i += 1) { - const walletBalance = walletBalances[i] - const wallet = await walletsCollection.findOne({ paymentId: walletBalance.account_id }) - const providerId = wallet.providerId - runtime.postgres.query(createPotentialPaymentsQuery, [payoutReportId, walletBalance.account_id, providerId, walletBalance.balance], client) - } - await client.query('COMMIT') - } catch (e) { - await client.query('ROLLBACK') - throw (e) - } finally { - client.release() - } -} - -exports.initialize = async (debug, runtime) => { - // Limit the dynos that can run this worker to 1 - if ((typeof process.env.DYNO !== 'undefined') && (process.env.DYNO !== 'worker.1')) return - - const interval = cron.parseExpression('0 0 1 * *', {}) - const next = interval.next().getTime() - setTimeout(() => { monthly(debug, runtime) }, next - underscore.now()) -} - -exports.monthly = monthly diff --git a/test/eyeshade/ads.integration.test.js b/test/eyeshade/ads.integration.test.js deleted file mode 100644 index 5f1301b09..000000000 --- a/test/eyeshade/ads.integration.test.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict' - -const { - serial: test -} = require('ava') -const uuidV4 = require('uuid/v4') -const { - cleanPgDb, - cleanDbs, - dbUri -} = require('../utils') -const Postgres = require('bat-utils/lib/runtime-postgres') -const { monthly } = require('../../eyeshade/workers/ads') -const { Runtime } = require('bat-utils') - -const postgres = new Postgres({ postgres: { url: process.env.BAT_POSTGRES_URL } }) -const runtime = new Runtime({ - postgres: { url: process.env.BAT_POSTGRES_URL }, - database: { - mongo: dbUri('ledger') - } -}) - -test.afterEach.always(async t => { - await cleanPgDb(postgres)() - await cleanDbs() -}) - -test('ads payout report cron job takes a snapshot of balances', async t => { - const ledgerMongo = runtime.database - const wallets = ledgerMongo.get('wallets', {}) - - // Create the wallet that will receive payment - const paymentId = uuidV4() - const providerId = uuidV4() - await wallets.insert({ paymentId, providerId }) - - // Create an ad transaction so there is a payment_id with a balance - const txId = uuidV4() - const createdAt = new Date() - createdAt.setMonth(createdAt.getMonth() - 2) - const description = 'funding tx for viewing ads' - const transactionType = 'ad' - const fromAccountType = 'uphold' - const fromAccount = uuidV4() - const toAccountType = 'payment_id' - const amount = 1000 - - const insertQuery = 'insert into transactions (id, created_at, description, transaction_type, from_account_type, from_account, to_account_type, to_account, amount) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)' - await postgres.query(insertQuery, [txId, createdAt, description, transactionType, fromAccountType, fromAccount, toAccountType, paymentId, amount]) - - await monthly({}, runtime) - - // Ensure the wallet balance made it in - const potentialPayments = (await postgres.query('select * from potential_payments_ads')).rows - t.is(potentialPayments.length, 1, 'the correct number of payments were inserted') - - const potentialPayment = potentialPayments[0] - t.is(potentialPayment.payment_id, paymentId, 'the inserted payment id matches') - t.is(potentialPayment.provider_id, providerId, 'the inserted provider id matches') - t.is(potentialPayment.amount, '1000.000000000000000000', 'the inserted amount matches') -}) diff --git a/test/eyeshade/migrations.test.js b/test/eyeshade/migrations.test.js deleted file mode 100644 index 2a8130dc3..000000000 --- a/test/eyeshade/migrations.test.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict' - -const { serial: test } = require('ava') -const Postgres = require('bat-utils/lib/runtime-postgres') -const uuidV4 = require('uuid/v4') - -const postgres = new Postgres({ postgres: { url: process.env.BAT_POSTGRES_URL } }) - -test('migrations table is up-to-date', async t => { - const latestInMigrationsTable = (await postgres.query('select id from migrations order by id desc limit 1;', [])).rows[0].id - const latestInMigrationsFolder = require('../../eyeshade/migrations/current') - - t.true(latestInMigrationsTable === latestInMigrationsFolder) -}) - -const createPayoutReportQuery = ` -insert into payout_reports_ads (id) -values ($1); -` -const createPotentialPaymentsQuery = ` -insert into potential_payments_ads (payout_report_id, payment_id, provider_id, amount) -values ($1, $2, $3, $4); -` -test('removal of payout_report_id removes payout_reports_ads as well', async (t) => { - let q - const id = uuidV4() - await postgres.query(createPayoutReportQuery, [id]) - await postgres.query(createPotentialPaymentsQuery, [id, uuidV4(), uuidV4(), '5']) - q = await postgres.query('SELECT * FROM potential_payments_ads WHERE payout_report_id = $1;', [id]) - t.is(q.rowCount, 1, 'found by payout_reports_ads id') - await postgres.query('DELETE FROM payout_reports_ads WHERE id = $1;', [id]) - q = await postgres.query('SELECT * FROM potential_payments_ads WHERE payout_report_id = $1;', [id]) - t.is(q.rowCount, 0, 'removal of potential_payments_ads when corresponding payout_reports_ads is deleted') -}) diff --git a/test/utils.js b/test/utils.js index fd078c510..9f38f5a91 100644 --- a/test/utils.js +++ b/test/utils.js @@ -323,8 +323,6 @@ function cleanPgDb (postgres) { const client = await postgres.connect() try { await Promise.all([ - client.query('DELETE from payout_reports_ads;'), - client.query('DELETE from potential_payments_ads;'), client.query('DELETE from transactions;'), client.query('DELETE from surveyor_groups;'), client.query('DELETE from geo_referral_countries;'),