Merge pull request #47 from ATOR-Development/46-publish-relay-uptime

WIP: Publish relay uptime

cli/cli.module.ts

@@ -2,7 +2,10 @@ import { Module } from '@nestjs/common'
 import { ConfigModule, ConfigService } from '@nestjs/config'
 import { MongooseModule } from '@nestjs/mongoose'
 
-import { SeedModule } from './commands/seed.module'
+import {
+  PopulateRelayUptimeModule
+} from './commands/populate-relay-uptime/populate-relay-uptime.module'
+import { SeedModule } from './commands/seed/seed.module'
 
 @Module({
   imports: [
@@ -13,6 +16,7 @@ import { SeedModule } from './commands/seed.module'
         uri: config.get<string>('MONGO_URI', { infer: true })
       })
     }),
+    PopulateRelayUptimeModule,
     SeedModule
   ]
 })

cli/commands/populate-relay-uptime/populate-relay-uptime-job.ts

@@ -0,0 +1,24 @@
+import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'
+import { HydratedDocument } from 'mongoose'
+
+@Schema()
+export class PopulateRelayUptimeJob {
+  @Prop({ type: String, required: true, index: -1 })
+  validation_date: string
+
+  @Prop({ type: Number, required: true, default: Date.now })
+  startedAt?: number
+
+  @Prop({ type: Number, required: false })
+  finishedAt?: number
+
+  @Prop({ type: Boolean, required: false })
+  success?: boolean
+
+  @Prop({ type: Number, required: true })
+  uptimeMinimumRunningCount: number
+}
+
+export type RelayUptimeJobDocument = HydratedDocument<PopulateRelayUptimeJob>
+export const RelayUptimeJobSchema =
+  SchemaFactory.createForClass(PopulateRelayUptimeJob)

cli/commands/populate-relay-uptime/populate-relay-uptime.command.ts

@@ -0,0 +1,120 @@
+import { Logger } from '@nestjs/common'
+import { InjectModel } from '@nestjs/mongoose'
+import _ from 'lodash'
+import { Model } from 'mongoose'
+import { Command, CommandRunner } from 'nest-commander'
+
+import { PopulateRelayUptimeJob } from './populate-relay-uptime-job'
+import extractIsodate from '../../../src/util/extract-isodate'
+import {
+  UptimeValidationService
+} from '../../../src/validation/uptime-validation.service'
+import { RelayData } from '../../../src/validation/schemas/relay-data'
+
+@Command({ name: 'populate-relay-uptime' })
+export class PopulateRelayUptimeCommand extends CommandRunner {
+  private readonly logger = new Logger(PopulateRelayUptimeCommand.name)
+
+  constructor(
+    private readonly uptimeValidationService: UptimeValidationService,
+    @InjectModel(PopulateRelayUptimeJob.name)
+    private readonly populateRelayUptimeJobModel: Model<PopulateRelayUptimeJob>,
+    @InjectModel(RelayData.name)
+    private readonly relayDataModel: Model<RelayData>
+  ) {
+    super()
+  }
+
+  private async determineStartDate() {
+    const lastJobRun = await this.populateRelayUptimeJobModel
+      .findOne({ success: true })
+      .sort({ validation_date: -1 })
+
+    if (!lastJobRun) {
+      this.logger.log(
+        'Did not find last job run, checking RelayData to determine start date'
+      )
+      const earliestRelayData = await this.relayDataModel
+        .findOne()
+        .sort({ validated_at: 1 })
+
+      if (!earliestRelayData) {
+        return null
+      }
+
+      const startDate = extractIsodate(earliestRelayData.validated_at)
+
+      this.logger.log(`Found earliest RelayData at ${startDate}`)
+
+      return startDate
+    }
+
+    this.logger.log(`Found previous job run ${lastJobRun.validation_date}`)
+    const startDate = new Date(lastJobRun.validation_date)
+
+    return extractIsodate(startDate.setDate(startDate.getDate() + 1))
+  }
+
+  async run() {
+    this.logger.log('Starting populate relay uptime job')
+    const startDate = await this.determineStartDate()
+
+    if (!startDate) {
+      this.logger.log(
+        'Could not determine start date, likely due to no RelayData'
+      )
+      return
+    }
+
+    const now = new Date()
+    const endDateTimestamp = now.setDate(now.getDate() - 1)
+    const endDate = extractIsodate(endDateTimestamp)
+
+    const validation_dates: string[] = []
+    let currentTimestamp = new Date(startDate).getTime()
+    while (currentTimestamp <= endDateTimestamp) {
+      validation_dates.push(extractIsodate(currentTimestamp))
+
+      const d = new Date(currentTimestamp)
+      currentTimestamp = d.setDate(d.getDate() + 1)
+    }
+
+    this.logger.log(
+      `Found dates needing relay uptime calcs: ${validation_dates.toString()}`
+    )
+
+    for (const validation_date of validation_dates) {
+      this.logger.log(`Populating relay uptime data for ${validation_date}`)
+      const existingJobRun = await this.populateRelayUptimeJobModel.findOne({
+        success: true,
+        validation_date
+      })
+
+      if (existingJobRun) {
+        this.logger.log(`Already ran job for ${validation_date}, skipping`)
+
+        continue
+      }
+
+      const jobRun = await this.populateRelayUptimeJobModel
+        .create<PopulateRelayUptimeJob>({
+          validation_date,
+          uptimeMinimumRunningCount:
+            this.uptimeValidationService.uptimeMinimumRunningCount
+        })
+
+      try {
+        await this.uptimeValidationService.populateRelayUptimesForDate(
+          validation_date
+        )
+        jobRun.success = true
+      } catch (error) {
+        jobRun.success = false
+        this.logger.log(`Job failed due to error`, error.stack)
+      }
+
+      jobRun.finishedAt = Date.now()
+      await jobRun.save()
+    }
+  }
+}

cli/commands/populate-relay-uptime/populate-relay-uptime.module.ts

@@ -0,0 +1,37 @@
+import { Module } from '@nestjs/common'
+import { ConfigModule } from '@nestjs/config'
+import { MongooseModule } from '@nestjs/mongoose'
+
+import {
+  PopulateRelayUptimeJob,
+  RelayUptimeJobSchema
+} from './populate-relay-uptime-job'
+import { PopulateRelayUptimeCommand } from './populate-relay-uptime.command'
+import {
+  RelayData,
+  RelayDataSchema
+} from '../../../src/validation/schemas/relay-data'
+import {
+  UptimeValidationService
+} from '../../../src/validation/uptime-validation.service'
+import {
+  RelayUptime,
+  RelayUptimeSchema
+} from '../../../src/validation/schemas/relay-uptime'
+
+@Module({
+  imports: [
+    ConfigModule,
+    MongooseModule.forFeature([
+      { name: PopulateRelayUptimeJob.name, schema: RelayUptimeJobSchema },
+      { name: RelayData.name, schema: RelayDataSchema },
+      { name: RelayUptime.name, schema: RelayUptimeSchema }
+    ])
+  ],
+  providers: [
+    UptimeValidationService,
+    ...PopulateRelayUptimeCommand.registerWithSubCommands()
+  ],
+  exports: [ PopulateRelayUptimeCommand ]
+})
+export class PopulateRelayUptimeModule {}

cli/commands/seed-relay-sale-data.subcommand.ts → cli/commands/seed/seed-relay-sale-data.subcommand.ts

@@ -5,7 +5,9 @@ import { Model } from 'mongoose'
 import { CommandRunner, Option, SubCommand } from 'nest-commander'
 
 import { SeedLock, SeedLockDocument } from './seed-lock'
-import { RelaySaleData } from '../../src/verification/schemas/relay-sale-data'
+import {
+  RelaySaleData
+} from '../../../src/verification/schemas/relay-sale-data'
 
 
 @SubCommand({ name: 'relay-sale-data' })

cli/commands/seed.command.ts → cli/commands/seed/seed.command.ts

@@ -1,5 +1,4 @@
-import { ConfigService } from '@nestjs/config'
-import { Command, CommandRunner, Option, SubCommand } from 'nest-commander'
+import { Command, CommandRunner } from 'nest-commander'
 
 import { RelaySaleDataSubCommand } from './seed-relay-sale-data.subcommand'
 
@@ -9,11 +8,7 @@ import { RelaySaleDataSubCommand } from './seed-relay-sale-data.subcommand'
   subCommands: [ RelaySaleDataSubCommand ]
 })
 export class SeedCommand extends CommandRunner {
-  constructor(
-    private readonly config: ConfigService
-  ) {
-    super()
-  }
+  constructor() { super() }
 
   async run(): Promise<void> {
     throw new Error('Unknown seed')

cli/commands/seed.module.ts → cli/commands/seed/seed.module.ts

@@ -7,7 +7,7 @@ import { SeedLock, SeedLockSchema } from './seed-lock'
 import {
   RelaySaleData,
   RelaySaleDataSchema
-} from '../../src/verification/schemas/relay-sale-data'
+} from '../../../src/verification/schemas/relay-sale-data'
 
 @Module({
   imports: [

operations/populate-relay-uptime-live.hcl

@@ -0,0 +1,37 @@
+job "populate-relay-uptime" {
+  datacenters = ["ator-fin"]
+  type = "batch"
+
+  periodic {
+    cron = "0 1 * * *" # every day at 1am
+    prohibit_overlap = true
+  }
+
+  group "populate-relay-uptime-group" {
+    count = 1
+
+    task "populate-relay-uptime-task" {
+      driver = "docker"
+
+      env {
+        UPTIME_MINIMUM_RUNNING_COUNT="16"
+      }
+
+      template {
+        data = <<EOH
+        {{- range service "validator-live-mongo" }}
+          MONGO_URI="mongodb://{{ .Address }}:{{ .Port }}/valid-ator-live-testnet"
+        {{- end }}
+        EOH
+        destination = "secrets/file.env"
+        env = true
+      }
+
+      config {
+        image = "ghcr.io/ator-development/valid-ator:[[.deploy]]"
+        entrypoint = [ "node" ]
+        args = [ "dist/cli/main.js", "populate-relay-uptime" ]
+      }
+    }
+  }
+}

operations/populate-relay-uptime-stage.hcl

@@ -0,0 +1,37 @@
+job "populate-relay-uptime" {
+  datacenters = ["ator-fin"]
+  type = "batch"
+
+  periodic {
+    cron = "0 1 * * *" # every day at 1am
+    prohibit_overlap = true
+  }
+
+  group "populate-relay-uptime-group" {
+    count = 1
+
+    task "populate-relay-uptime-task" {
+      driver = "docker"
+
+      env {
+        UPTIME_MINIMUM_RUNNING_COUNT="16"
+      }
+
+      template {
+        data = <<EOH
+        {{- range service "validator-stage-mongo" }}
+          MONGO_URI="mongodb://{{ .Address }}:{{ .Port }}/valid-ator-stage-testnet"
+        {{- end }}
+        EOH
+        destination = "secrets/file.env"
+        env = true
+      }
+
+      config {
+        image = "ghcr.io/ator-development/valid-ator:[[.deploy]]"
+        entrypoint = [ "node" ]
+        args = [ "dist/cli/main.js", "populate-relay-uptime" ]
+      }
+    }
+  }
+}

src/distribution/distribution.service.spec.ts

@@ -1,19 +1,25 @@
 import { Test, TestingModule } from '@nestjs/testing'
 import { DistributionService } from './distribution.service'
 import { ConfigModule } from '@nestjs/config'
-import { MongooseModule } from '@nestjs/mongoose'
+import { HttpModule } from '@nestjs/axios'
 
 describe('DistributionService', () => {
     let service: DistributionService
-    let testModule: TestingModule
+    let module: TestingModule
 
-    beforeAll(async () => {
-        testModule = await Test.createTestingModule({
-            imports: [ConfigModule.forRoot()],
-            providers: [DistributionService],
+    beforeEach(async () => {
+        module = await Test.createTestingModule({
+            imports: [ ConfigModule.forRoot(), HttpModule ],
+            providers: [ DistributionService ],
         }).compile()
 
-        service = testModule.get<DistributionService>(DistributionService)
+        service = module.get<DistributionService>(DistributionService)
+    })
+
+    afterEach(async () => {
+        if (module) {
+            await module.close()
+        }
     })
 
     it('should be defined', () => {

src/util/extract-isodate.ts

@@ -0,0 +1,7 @@
+export default function (timestamp: number) {
+  const [ date ] = new Date(timestamp)
+      .toISOString()
+      .split('T')
+
+  return date
+}