diff --git a/.gitignore b/.gitignore index f646938..89edf2f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ lerna-debug.log* # Warp cache/* -version.yml \ No newline at end of file +version.yml +data diff --git a/README.md b/README.md index 4bac83e..4d2c87f 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,19 @@ Entity-Type: } ``` +## CLI + +### Seeding Relay Sale Data + +- Seed Relay Sale Data +```bash +npm run cli -- seed relay-sale-data -d +``` + +- Remove Relay Sale Data +```bash +npm run cli -- seed relay-sale-data -d down +``` ## Development diff --git a/cli/cli.module.ts b/cli/cli.module.ts new file mode 100644 index 0000000..389b02c --- /dev/null +++ b/cli/cli.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common' +import { ConfigModule, ConfigService } from '@nestjs/config' +import { MongooseModule } from '@nestjs/mongoose' + +import { SeedModule } from './commands/seed.module' + +@Module({ + imports: [ + ConfigModule.forRoot({ isGlobal: true }), + MongooseModule.forRootAsync({ + inject: [ConfigService<{ MONGO_URI: string }>], + useFactory: (config: ConfigService) => ({ + uri: config.get('MONGO_URI', { infer: true }) + }) + }), + SeedModule + ] +}) +export class CliModule {} diff --git a/cli/commands/seed-lock.ts b/cli/commands/seed-lock.ts new file mode 100644 index 0000000..2d76e9c --- /dev/null +++ b/cli/commands/seed-lock.ts @@ -0,0 +1,17 @@ +import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' +import { HydratedDocument } from 'mongoose' + +@Schema() +export class SeedLock { + @Prop({ type: String, required: true }) + seedName: string + + @Prop({ type: Number, required: true, default: Date.now }) + startedAt?: number + + @Prop({ type: Number, required: false }) + finishedAt?: number +} + +export type SeedLockDocument = HydratedDocument +export const SeedLockSchema = SchemaFactory.createForClass(SeedLock) diff --git a/cli/commands/seed-relay-sale-data.subcommand.ts b/cli/commands/seed-relay-sale-data.subcommand.ts new file mode 100644 index 0000000..debd69f --- /dev/null +++ b/cli/commands/seed-relay-sale-data.subcommand.ts @@ -0,0 +1,121 @@ +import { Logger } from '@nestjs/common' +import { InjectModel } from '@nestjs/mongoose' +import fs from 'fs' +import { Model } from 'mongoose' +import { CommandRunner, Option, SubCommand } from 'nest-commander' + +import { SeedLock, SeedLockDocument } from './seed-lock' +import { RelaySaleData } from '../../src/verification/schemas/relay-sale-data' + + +@SubCommand({ name: 'relay-sale-data' }) +export class RelaySaleDataSubCommand extends CommandRunner { + private readonly seedName = 'relay-sale-data' + private readonly logger = new Logger(RelaySaleDataSubCommand.name) + + constructor( + @InjectModel(SeedLock.name) + private readonly seedLockModel: Model, + @InjectModel(RelaySaleData.name) + private readonly relaySaleDataModel: Model + ) { + super() + } + + @Option({ + flags: '-d, --data ', + description: 'Path to CSV seed data', + required: true + }) + parseSeedFilePath(path: string) { + const exists = fs.existsSync(path) + + if (!exists) { + this.logger.error(`File not found: ${path}`) + process.exit(1) + } + + return path + } + + async run( + params: string[], + options: { data: string } + ): Promise { + const existingLock = await this.seedLockModel.findOne({ + seedName: this.seedName + }) + + if (params.includes('down')) { + return this.down(existingLock) + } else { + return this.up(existingLock, options.data) + } + } + + private async up( + existingLock: SeedLockDocument | null, + dataFilePath: string + ) { + if (existingLock) { + this.logger.log( + `Found existing seed lock for ${this.seedName}`, + existingLock.toObject() + ) + + return + } + + const saleDataCsv = fs.readFileSync(dataFilePath).toString('utf-8') + const saleDataLines = saleDataCsv.split('\r\n') + + const seedLock = await this.seedLockModel.create({ + seedName: this.seedName + }) + + const session = await this.relaySaleDataModel.startSession() + session.startTransaction() + + this.logger.log(`Clearing existing ${this.seedName} data`) + await this.relaySaleDataModel.deleteMany() + + this.logger.log( + `Seeding ${saleDataLines.length - 1} ${this.seedName} documents.` + ) + for (let i = 1; i < saleDataLines.length; i++) { + const [ serial, unparsedNftId ] = saleDataLines[i].split(',') + const parsedNftId = Number.parseInt(unparsedNftId) + const nftId = Number.isNaN(parsedNftId) ? 0 : parsedNftId + + await this.relaySaleDataModel.create({ serial, nftId }) + } + + await session.commitTransaction() + await session.endSession() + + seedLock.finishedAt = Date.now() + await seedLock.save() + + this.logger.log( + `Done seeding ${saleDataLines.length - 1} ${this.seedName} documents.` + ) + } + + private async down(existingLock: SeedLockDocument | null) { + if (!existingLock) { + this.logger.log(`No seed lock found for ${this.seedName}, nothing to remove`) + + return + } + + this.logger.log( + `Removing existing seed lock for ${this.seedName}`, + existingLock + ) + await existingLock.deleteOne() + + this.logger.log(`Removing existing ${this.seedName} seed data`) + const result = await this.relaySaleDataModel.deleteMany() + this.logger.log(`Removed ${result.deletedCount}`) + } +} diff --git a/cli/commands/seed.command.ts b/cli/commands/seed.command.ts new file mode 100644 index 0000000..f1fe2c0 --- /dev/null +++ b/cli/commands/seed.command.ts @@ -0,0 +1,21 @@ +import { ConfigService } from '@nestjs/config' +import { Command, CommandRunner, Option, SubCommand } from 'nest-commander' + +import { RelaySaleDataSubCommand } from './seed-relay-sale-data.subcommand' + +@Command({ + name: 'seed', + arguments: '', + subCommands: [ RelaySaleDataSubCommand ] +}) +export class SeedCommand extends CommandRunner { + constructor( + private readonly config: ConfigService + ) { + super() + } + + async run(): Promise { + throw new Error('Unknown seed') + } +} diff --git a/cli/commands/seed.module.ts b/cli/commands/seed.module.ts new file mode 100644 index 0000000..769722c --- /dev/null +++ b/cli/commands/seed.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common' +import { ConfigModule } from '@nestjs/config' +import { MongooseModule } from '@nestjs/mongoose' + +import { SeedCommand } from './seed.command' +import { SeedLock, SeedLockSchema } from './seed-lock' +import { + RelaySaleData, + RelaySaleDataSchema +} from '../../src/verification/schemas/relay-sale-data' + +@Module({ + imports: [ + ConfigModule, + MongooseModule.forFeature([ + { name: RelaySaleData.name, schema: RelaySaleDataSchema }, + { name: SeedLock.name, schema: SeedLockSchema } + ]) + ], + providers: [ ...SeedCommand.registerWithSubCommands() ], + exports: [ SeedCommand ] +}) +export class SeedModule {} diff --git a/cli/main.ts b/cli/main.ts new file mode 100644 index 0000000..dcadc53 --- /dev/null +++ b/cli/main.ts @@ -0,0 +1,12 @@ +import { CommandFactory } from 'nest-commander' + +import { CliModule } from './cli.module' + +const bootstrap = async () => { + await CommandFactory.run(CliModule, { + logger: ['error', 'warn', 'log', 'debug'], + // errorHandler: (error) => { console.error('Validator CLI error', error) } + }) +} + +bootstrap() diff --git a/operations/seed-relay-sale-data.hcl b/operations/seed-relay-sale-data.hcl new file mode 100644 index 0000000..593bc54 --- /dev/null +++ b/operations/seed-relay-sale-data.hcl @@ -0,0 +1,54 @@ +job "seed-relay-sale-data" { + datacenters = ["ator-fin"] + type = "batch" + + group "seed-relay-sale-data-group" { + count = 1 + + volume "relay-sale-data-volume" { + type = "host" + read_only = true + source = "relay-sale-data-volume" + } + + network { + mode = "bridge" + port "validator" { + to = 3000 + host_network = "wireguard" + } + } + + vault { + // TODO + } + + task "seed-relay-sale-data-task" { + driver = "docker" + + volume_mount { + volume = "relay-sale-data-volume" + + read_only = true + } + + env { + // TODO + } + + template { + // TODO + } + + config { + image = "ghcr.io/ator-development/valid-ator:[[.deploy]]" + entrypoint = [ "npm" ] + args = [ + "run", "cli", "--", + "seed", "relay-sale-data", + "--data", "TODO -> pathtodata" // TODO + ] + } + } + } +} diff --git a/package-lock.json b/package-lock.json index 231a9d3..6c145ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@nestjs/core": "^9.4.0", "@nestjs/mongoose": "^9.2.2", "@nestjs/platform-express": "^9.4.0", + "@noble/curves": "^1.4.2", "@types/consul": "^0.40.3", "axios": "^1.3.6", "bignumber.js": "^9.1.1", @@ -27,6 +28,7 @@ "h3-js": "^4.1.0", "lodash": "^4.17.21", "mongoose": "^7.0.5", + "nest-commander": "^3.14.0", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.0", "warp-contracts": "^1.4.45", @@ -1434,9 +1436,9 @@ } }, "node_modules/@babel/generator": { - "version": "7.24.9", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.9.tgz", - "integrity": "sha512-G8v3jRg+z8IwY1jHFxvCNhOPYPterE4XljNgdGTYfSTtzzwjIswIzIaSPSLs3R7yFuqnqNeay5rjICfqVr+/6A==", + "version": "7.24.10", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.10.tgz", + "integrity": "sha512-o9HBZL1G2129luEUlG1hB4N/nlYNWHnpwlND9eOMclRqqu1YDy2sSYVCFUZwl8I1Gxh+QSRrP2vD7EpUmFVXxg==", "license": "MIT", "dependencies": { "@babel/types": "^7.24.9", @@ -5308,9 +5310,9 @@ } }, "node_modules/@mongodb-js/saslprep": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.7.tgz", - "integrity": "sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q==", + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.8.tgz", + "integrity": "sha512-qKwC/M/nNNaKUBMQ0nuzm47b7ZYWQHN3pcXq4IIcoSBc2hOIrflAxJduIvvqmhoz3gR2TacTAs8vlsCVPkiEdQ==", "license": "MIT", "optional": true, "dependencies": { @@ -7621,12 +7623,12 @@ } }, "node_modules/@solana/web3.js": { - "version": "1.95.0", - "resolved": "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.95.0.tgz", - "integrity": "sha512-iHwJ/HcWrF9qbnI1ctwI1UXHJ0vZXRpnt+lI5UcQIk8WvJNuQ5gV06icxzM6B7ojUES85Q1/FM4jZ49UQ8yZZQ==", + "version": "1.95.1", + "resolved": "https://registry.npmjs.org/@solana/web3.js/-/web3.js-1.95.1.tgz", + "integrity": "sha512-mRX/AjV6QbiOXpWcy5Rz1ZWEH2lVkwO7T0pcv9t97ACpv3/i3tPiqXwk0JIZgSR3wOSTiT26JfygnJH2ulS6dQ==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.24.7", + "@babel/runtime": "^7.24.8", "@noble/curves": "^1.4.2", "@noble/hashes": "^1.4.0", "@solana/buffer-layout": "^4.0.1", @@ -7637,7 +7639,7 @@ "bs58": "^4.0.1", "buffer": "6.0.3", "fast-stable-stringify": "^1.0.0", - "jayson": "^4.1.0", + "jayson": "^4.1.1", "node-fetch": "^2.7.0", "rpc-websockets": "^9.0.2", "superstruct": "^2.0.2" @@ -7912,6 +7914,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/inquirer": { + "version": "8.2.10", + "resolved": "https://registry.npmjs.org/@types/inquirer/-/inquirer-8.2.10.tgz", + "integrity": "sha512-IdD5NmHyVjWM8SHWo/kPBgtzXatwPkfwzyP3fN1jF2g9BWt5WO+8hL2F4o2GKIYsU40PpqeevuUWvkS/roXJkA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/through": "*", + "rxjs": "^7.2.0" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -7964,9 +7977,9 @@ } }, "node_modules/@types/lodash": { - "version": "4.17.6", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.6.tgz", - "integrity": "sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==", + "version": "4.17.7", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.7.tgz", + "integrity": "sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==", "dev": true, "license": "MIT" }, @@ -8112,6 +8125,16 @@ "@types/superagent": "*" } }, + "node_modules/@types/through": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/through/-/through-0.0.33.tgz", + "integrity": "sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/uuid": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.4.tgz", @@ -9110,7 +9133,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/array-flatten": { @@ -10022,7 +10044,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -11206,9 +11227,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.4.828", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.828.tgz", - "integrity": "sha512-QOIJiWpQJDHAVO4P58pwb133Cwee0nbvy/MV1CwzZVGpkH1RX33N3vsaWRCpR6bF63AAq366neZrRTu7Qlsbbw==", + "version": "1.4.829", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.829.tgz", + "integrity": "sha512-5qp1N2POAfW0u1qGAxXEtz6P7bO1m6gpZr5hdf5ve6lxpLM7MpiM4jIPz7xcrNlClQMafbyUDDWjlIQZ1Mw0Rw==", "license": "ISC" }, "node_modules/elliptic": { @@ -12459,9 +12480,9 @@ "peer": true }, "node_modules/flow-parser": { - "version": "0.239.1", - "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.239.1.tgz", - "integrity": "sha512-topOrETNxJ6T2gAnQiWqAlzGPj8uI2wtmNOlDIMNB+qyvGJZ6R++STbUOTAYmvPhOMz2gXnXPH0hOvURYmrBow==", + "version": "0.241.0", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.241.0.tgz", + "integrity": "sha512-82yKXpz7iWknWFsognZUf5a6mBQLnVrYoYSU9Nbu7FTOpKlu3v9ehpiI9mYXuaIO3J0ojX1b83M/InXvld9HUw==", "license": "MIT", "peer": true, "engines": { @@ -13253,7 +13274,6 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, "license": "MIT", "dependencies": { "parent-module": "^1.0.0", @@ -13476,9 +13496,9 @@ } }, "node_modules/is-core-module": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.14.0.tgz", - "integrity": "sha512-a5dFJih5ZLYlRtDc0dZWP7RiKr6xIKzmn/oAYCDvdLThadVgyJwlaoQPmRtMSpz+rk0OGAgIu+TcM9HUF0fk1A==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.0.tgz", + "integrity": "sha512-Dd+Lb2/zvk9SKy1TGCt1wFJFo/MWBPMX5x7KcvLajWTGuomczdQX61PvY5yK6SVACwpoexWo81IfFyoKY2QnTA==", "license": "MIT", "dependencies": { "hasown": "^2.0.2" @@ -14550,7 +14570,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -14663,7 +14682,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, "license": "MIT" }, "node_modules/json-schema-traverse": { @@ -14968,7 +14986,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, "license": "MIT" }, "node_modules/loader-runner": { @@ -16390,6 +16407,99 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/nest-commander": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/nest-commander/-/nest-commander-3.14.0.tgz", + "integrity": "sha512-3HEfsEzoKEZ/5/cptkXlL8/31qohPxtMevoFo4j9NMe3q5PgI/0TgTYN/6py9GnFD51jSasEfFGChs1BJ+Enag==", + "license": "MIT", + "dependencies": { + "@fig/complete-commander": "^3.0.0", + "@golevelup/nestjs-discovery": "4.0.1", + "commander": "11.1.0", + "cosmiconfig": "8.3.6", + "inquirer": "8.2.6" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@types/inquirer": "^8.1.3" + } + }, + "node_modules/nest-commander/node_modules/@fig/complete-commander": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@fig/complete-commander/-/complete-commander-3.2.0.tgz", + "integrity": "sha512-1Holl3XtRiANVKURZwgpjCnPuV4RsHp+XC0MhgvyAX/avQwj7F2HUItYOvGi/bXjJCkEzgBZmVfCr0HBA+q+Bw==", + "license": "MIT", + "dependencies": { + "prettier": "^3.2.5" + }, + "peerDependencies": { + "commander": "^11.1.0" + } + }, + "node_modules/nest-commander/node_modules/@golevelup/nestjs-discovery": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@golevelup/nestjs-discovery/-/nestjs-discovery-4.0.1.tgz", + "integrity": "sha512-HFXBJayEkYcU/bbxOztozONdWaZR34ZeJ2zRbZIWY8d5K26oPZQTvJ4L0STW3XVRGWtoE0WBpmx2YPNgYvcmJQ==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.21" + }, + "peerDependencies": { + "@nestjs/common": "^10.x", + "@nestjs/core": "^10.x" + } + }, + "node_modules/nest-commander/node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, + "node_modules/nest-commander/node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/nest-commander/node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/nocache": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/nocache/-/nocache-3.0.4.tgz", @@ -16543,9 +16653,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", - "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", + "version": "2.0.17", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.17.tgz", + "integrity": "sha512-Ww6ZlOiEQfPfXM45v17oabk77Z7mg5bOt7AjDyzy7RjK9OrLrLC8dyZQoAPEOtFX9SaNf1Tdvr5gRJWdTJj7GA==", "license": "MIT" }, "node_modules/node-stream-zip": { @@ -16830,7 +16940,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, "license": "MIT", "dependencies": { "callsites": "^3.0.0" @@ -16843,7 +16952,6 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.0.0", @@ -16944,7 +17052,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -17850,7 +17957,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -18250,9 +18356,9 @@ } }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -19049,9 +19155,9 @@ } }, "node_modules/terser": { - "version": "5.31.2", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.2.tgz", - "integrity": "sha512-LGyRZVFm/QElZHy/CPr/O4eNZOZIzsrQ92y4v9UJe/pFJjypje2yI3C2FmPtvUEnhadlSbmG2nXtdcjHOjCfxw==", + "version": "5.31.3", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.3.tgz", + "integrity": "sha512-pAfYn3NIZLyZpa83ZKigvj6Rn9c/vd5KfYGX7cN1mnzqgDcxWvrU5ZtAfIKhEXz9nRecw4z3LXkjaq96/qZqAA==", "license": "BSD-2-Clause", "dependencies": { "@jridgewell/source-map": "^0.3.3", @@ -19576,7 +19682,7 @@ "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/package.json b/package.json index bc8829e..3b5c3ab 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "license": "AGPL-3.0-only", "scripts": { "build": "nest build", + "cli": "ts-node cli/main.ts", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "start": "nest start", "start:dev": "nest start --watch", @@ -32,6 +33,7 @@ "@nestjs/core": "^9.4.0", "@nestjs/mongoose": "^9.2.2", "@nestjs/platform-express": "^9.4.0", + "@noble/curves": "^1.4.2", "@types/consul": "^0.40.3", "axios": "^1.3.6", "bignumber.js": "^9.1.1", @@ -42,6 +44,7 @@ "h3-js": "^4.1.0", "lodash": "^4.17.21", "mongoose": "^7.0.5", + "nest-commander": "^3.14.0", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.0", "warp-contracts": "^1.4.45", diff --git a/src/checks/checks.module.ts b/src/checks/checks.module.ts index ec73571..13ff387 100644 --- a/src/checks/checks.module.ts +++ b/src/checks/checks.module.ts @@ -3,7 +3,6 @@ import { MongooseModule } from '@nestjs/mongoose' import { BalancesService } from './balances.service' import { BalancesData, BalancesDataSchema } from './schemas/balances-data' -import { UserBalancesService } from './user-balances.service' @Module({ imports: [ @@ -15,12 +14,10 @@ import { UserBalancesService } from './user-balances.service' ]), ], providers: [ - BalancesService, - UserBalancesService + BalancesService ], exports: [ - BalancesService, - UserBalancesService + BalancesService ], }) export class ChecksModule {} diff --git a/src/checks/user-balances.service.spec.ts b/src/checks/user-balances.service.spec.ts deleted file mode 100644 index 7d8164c..0000000 --- a/src/checks/user-balances.service.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing' -import { ConfigModule } from '@nestjs/config' - -import { UserBalancesService } from './user-balances.service' - -describe('UserBalancesService', () => { - let service: UserBalancesService - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [ - ConfigModule.forRoot(), - ], - providers: [UserBalancesService], - }).compile() - - service = module.get(UserBalancesService) - }) - - it('should be defined', () => { - expect(service).toBeDefined() - }) -}) diff --git a/src/checks/user-balances.service.ts b/src/checks/user-balances.service.ts deleted file mode 100644 index b109d6c..0000000 --- a/src/checks/user-balances.service.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common' -import { ConfigService } from '@nestjs/config' -import { Contract, ethers } from 'ethers' - -const RELAYUP_ABI = [ - 'function totalSupply() public view override returns (uint256)', - 'function balanceOf(address owner) public view returns (uint256)', - 'function ownerOf(uint256 tokenId) public view returns (address)' -] - -@Injectable() -export class UserBalancesService implements OnApplicationBootstrap { - private readonly logger = new Logger(UserBalancesService.name) - - private mainnetJsonRpc?: string - private mainnetProvider: ethers.JsonRpcProvider - - private relayupNftContractAddress?: string - private relayupNftContract?: Contract - - constructor( - private readonly config: ConfigService<{ - MAINNET_JSON_RPC: string, - RELAY_UP_NFT_CONTRACT_ADDRESS: string - }> - ) { - this.relayupNftContractAddress = this.config.get( - 'RELAY_UP_NFT_CONTRACT_ADDRESS', { infer: true } - ) - - this.mainnetJsonRpc = this.config.get( - 'MAINNET_JSON_RPC', - { infer: true } - ) - - if (!this.mainnetJsonRpc) { - this.logger.error('Missing MAINNET_JSON_RPC!') - } else if (!this.relayupNftContractAddress) { - this.logger.error('Missing RELAYUP NFT Contract address!') - } else { - this.mainnetProvider = new ethers.JsonRpcProvider(this.mainnetJsonRpc) - this.relayupNftContract = new Contract( - this.relayupNftContractAddress, - RELAYUP_ABI, - this.mainnetProvider - ) - - this.logger.log( - `Initialized user balance service for RELAYUP NFT Contract: ${this.relayupNftContractAddress}` - ) - } - } - - async onApplicationBootstrap() { - this.logger.log(`Bootstrapped User Balances Service`) - } - - async isOwnerOfRelayupNft( - address: string, - nftId: bigint - ) { - if (!this.relayupNftContract) { - this.logger.error( - `Could not check owner of RELAYUP NFT #${nftId}: No Contract` - ) - - return false - } - - const owner = await this.relayupNftContract.ownerOf(nftId) - - return address === owner - } -} diff --git a/src/util/address-evm.ts b/src/util/address-evm.ts index 9a74839..7857d4a 100644 --- a/src/util/address-evm.ts +++ b/src/util/address-evm.ts @@ -1,4 +1,5 @@ -import { LengthOfString, HexString } from './hex-string' +import { isHexString } from 'ethers' +import { LengthOfString, HexString, UPPER_HEX_CHARS, isHexStringValid } from './hex-string' export type EvmAddress = S extends '' ? never @@ -12,3 +13,13 @@ declare function onlyEvmAddress( address: S & EvmAddress, ): any // onlyEvmAddress('0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA') + +export function isAddressValid(address?: string) { + if (!address) { return false } + if (address.length !== 40) { return false } + if (!isHexStringValid(address, true)) { + return false + } + + return true +} diff --git a/src/util/ec-point-compress.ts b/src/util/ec-point-compress.ts new file mode 100644 index 0000000..5b9db5f --- /dev/null +++ b/src/util/ec-point-compress.ts @@ -0,0 +1,13 @@ +/** + * Point compress elliptic curve key + * @param {Uint8Array} x component + * @param {Uint8Array} y component + * @return {Uint8Array} Compressed representation + */ +export function ECPointCompress(x: Uint8Array, y: Uint8Array) { + const out = new Uint8Array(x.length + 1) + out[0] = 2 + (y[ y.length-1 ] & 1) + out.set(x, 1) + + return out +} diff --git a/src/util/fingerprint-tor.ts b/src/util/fingerprint-tor.ts deleted file mode 100644 index e56da38..0000000 --- a/src/util/fingerprint-tor.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { LengthOfString, HexString } from './hex-string' - -export type TorFingerprint = Uppercase & S extends '' - ? never - : LengthOfString extends 40 - ? HexString - : never - -declare function onlyTorFingerprint( - fingerprint: S & TorFingerprint, -): any - -// onlyTorFingerprint('AAAAABBBBBCCCCCDDDDDEEEEEFFFFF0000011111') diff --git a/src/util/fingerprint.ts b/src/util/fingerprint.ts new file mode 100644 index 0000000..6345cf3 --- /dev/null +++ b/src/util/fingerprint.ts @@ -0,0 +1,33 @@ +import { LengthOfString, HexString, UPPER_HEX_CHARS } from './hex-string' + +export type Fingerprint = Uppercase & S extends '' + ? never + : LengthOfString extends 40 + ? HexString + : never + +// onlyFingerprint('AAAAABBBBBCCCCCDDDDDEEEEEFFFFF0000011111') +declare function onlyFingerprint( + fingerprint: S & Fingerprint, +): any + +export function isFingerprintValid(fingerprint?: string) { + // ContractAssert(!!fingerprint, FINGERPRINT_REQUIRED) + if (!fingerprint) { return false } + + // ContractAssert(typeof fingerprint === 'string', INVALID_FINGERPRINT) + if (typeof fingerprint !== 'string') { return false } + + // ContractAssert(fingerprint.length === 40, INVALID_FINGERPRINT) + if (fingerprint.length !== 40) { return false} + + // ContractAssert( + // fingerprint.split('').every(c => UPPER_HEX_CHARS.includes(c)), + // INVALID_FINGERPRINT + // ) + if (!fingerprint.split('').every(c => UPPER_HEX_CHARS.includes(c))) { + return false + } + + return true +} diff --git a/src/util/hex-string.ts b/src/util/hex-string.ts index 0ee182b..d99cee8 100644 --- a/src/util/hex-string.ts +++ b/src/util/hex-string.ts @@ -21,3 +21,20 @@ export type HexString = S extends '' declare function onlyHexString( hexString: S & HexString, ): any + +export const UPPER_HEX_CHARS = '0123456789ABCDEF' +export const HEX_CHARS = `0123456789ABCDEFabcdef` + +export function isHexStringValid(hex?: string, uppercase: boolean = false) { + if (!hex) { return false } + + if ( + !hex + .split('') + .every(c => (uppercase ? UPPER_HEX_CHARS : HEX_CHARS).includes(c)) + ) { + return false + } + + return true +} diff --git a/src/validation/dto/relay-data-dto.ts b/src/validation/dto/relay-data-dto.ts index 101077f..174485d 100644 --- a/src/validation/dto/relay-data-dto.ts +++ b/src/validation/dto/relay-data-dto.ts @@ -15,4 +15,27 @@ export class RelayDataDto { readonly observed_bandwidth: number readonly advertised_bandwidth: number readonly effective_family: string[] + + readonly hardware_info?: { + id?: string + company?: string + format?: string + wallet?: string + fingerprint?: string + nftid?: string + build?: string + flags?: string + serNums?: { + type?: string + number?: string + }[] + pubKeys?: { + type?: string + number?: string + }[] + certs?: { + type?: string + signature?: string + }[] + } } diff --git a/src/validation/interfaces/8_3/relay-info.ts b/src/validation/interfaces/8_3/relay-info.ts index eccfbf3..66066ad 100644 --- a/src/validation/interfaces/8_3/relay-info.ts +++ b/src/validation/interfaces/8_3/relay-info.ts @@ -44,4 +44,26 @@ export interface RelayInfo { exit_probability?: number // Probability of this relay to be selected for the exit position. This probability is calculated based on consensus weights, relay flags, and bandwidth weights in the consensus. Path selection depends on more factors, so that this probability can only be an approximation. Omitted if the relay is not running, or the consensus does not contain bandwidth weights. measured?: boolean // Boolean field saying whether the consensus weight of this relay is based on a threshold of 3 or more measurements by Tor bandwidth authorities. Omitted if the network status consensus containing this relay does not contain measurement information. unreachable_or_addresses?: string[] // Array of IPv4 or IPv6 addresses and TCP ports or port lists where the relay claims in its descriptor to accept onion-routing connections but that the directory authorities failed to confirm as reachable. Contains only additional addresses of a relay that are found unreachable and only as long as a minority of directory authorities performs reachability tests on these additional addresses. Relays with an unreachable primary address are not included in the network status consensus and excluded entirely. Likewise, relays with unreachable additional addresses tested by a majority of directory authorities are not included in the network status consensus and excluded here, too. If at any point network status votes will be added to the processing, relays with unreachable addresses will be included here. Addresses are in arbitrary order. IPv6 hex characters are all lower-case. Omitted if empty. + hardware_info?: { + id?: string + company?: string + format?: string + wallet?: string + fingerprint?: string + nftid?: string + build?: string + flags?: string + serNums?: { + type?: string + number?: string + }[] + pubKeys?: { + type?: string + number?: string + }[] + certs?: { + type?: string + signature?: string + }[] + } } diff --git a/src/validation/schemas/relay-data.ts b/src/validation/schemas/relay-data.ts index ba7a5f4..016d54b 100644 --- a/src/validation/schemas/relay-data.ts +++ b/src/validation/schemas/relay-data.ts @@ -52,6 +52,30 @@ export class RelayData { @Prop({ type: String, required: false }) nickname?: string + + @Prop({ type: Object, required: false }) + hardware_info?: { + id?: string + company?: string + format?: string + wallet?: string + fingerprint?: string + nftid?: string + build?: string + flags?: string + serNums?: { + type?: string + number?: string + }[] + pubKeys?: { + type?: string + number?: string + }[] + certs?: { + type?: string + signature?: string + }[] + } } export const RelayDataSchema = SchemaFactory.createForClass(RelayData) diff --git a/src/validation/schemas/validated-relay.ts b/src/validation/schemas/validated-relay.ts index 7925f7e..80395a6 100644 --- a/src/validation/schemas/validated-relay.ts +++ b/src/validation/schemas/validated-relay.ts @@ -3,6 +3,29 @@ import { HydratedDocument } from 'mongoose' export type ValidatedRelayDocument = HydratedDocument +export type RelayHardwareInfo = { + id?: string + company?: string + format?: string + wallet?: string + fingerprint?: string + nftid?: string + build?: string + flags?: string + serNums?: { + type?: string + number?: string + }[] + pubKeys?: { + type?: string + number?: string + }[] + certs?: { + type?: string + signature?: string + }[] +} + @Schema() export class ValidatedRelay { @Prop({ type: String, required: true }) @@ -34,6 +57,9 @@ export class ValidatedRelay { @Prop({ type: String, required: false }) nickname?: string + + @Prop({ type: Object, required: false }) + hardware_info?: RelayHardwareInfo } export const ValidatedRelaySchema = SchemaFactory.createForClass(ValidatedRelay) diff --git a/src/validation/validation.service.ts b/src/validation/validation.service.ts index f27a65c..1cf25be 100644 --- a/src/validation/validation.service.ts +++ b/src/validation/validation.service.ts @@ -175,7 +175,8 @@ export class ValidationService { bandwidth_burst: info.bandwidth_burst ?? 0, observed_bandwidth: info.observed_bandwidth ?? 0, advertised_bandwidth: info.advertised_bandwidth ?? 0, - effective_family: info.effective_family ?? [] + effective_family: info.effective_family ?? [], + hardware_info: info.hardware_info }), ) @@ -193,17 +194,17 @@ export class ValidationService { } public async validateRelays( - relays: RelayDataDto[], + relaysDto: RelayDataDto[], ): Promise { const validationStamp = Date.now() - if (relays.length === 0) { + if (relaysDto.length === 0) { this.logger.debug(`No relays to validate at ${validationStamp}`) return { validated_at: validationStamp, relays: [], } } else { - const validatedRelays = relays + const validatedRelays = relaysDto .map((relay, index, array) => ({ fingerprint: relay.fingerprint, ator_address: this.extractAtorKey(relay.contact), @@ -213,8 +214,7 @@ export class ValidationService { running: relay.running, family: relay.effective_family, consensus_measured: relay.consensus_measured, - primary_address_hex: relay.primary_address_hex, - nickname: relay.nickname + primary_address_hex: relay.primary_address_hex })) .filter((relay, index, array) => relay.ator_address.length > 0) @@ -234,11 +234,10 @@ export class ValidationService { `Storing validation ${validationStamp} of ${relay.fingerprint}`, ) - const relayData = relays.find( - (value, index, array) => - value.fingerprint == relay.fingerprint, + const relayDto = relaysDto.find( + ({ fingerprint }) => fingerprint == relay.fingerprint, ) - if (relayData == undefined) { + if (relayDto == undefined) { this.logger.error( `Failed to find relay data for validated relay [${relay.fingerprint}]`, ) @@ -249,23 +248,24 @@ export class ValidationService { fingerprint: relay.fingerprint, ator_address: relay.ator_address, primary_address_hex: relay.primary_address_hex, - consensus_weight: relayData.consensus_weight, + consensus_weight: relayDto.consensus_weight, - running: relayData.running, - consensus_measured: relayData.consensus_measured, + running: relayDto.running, + consensus_measured: relayDto.consensus_measured, consensus_weight_fraction: - relayData.consensus_weight_fraction, - version: relayData.version, - version_status: relayData.version_status, - bandwidth_rate: relayData.bandwidth_rate, - bandwidth_burst: relayData.bandwidth_burst, - observed_bandwidth: relayData.observed_bandwidth, + relayDto.consensus_weight_fraction, + version: relayDto.version, + version_status: relayDto.version_status, + bandwidth_rate: relayDto.bandwidth_rate, + bandwidth_burst: relayDto.bandwidth_burst, + observed_bandwidth: relayDto.observed_bandwidth, advertised_bandwidth: - relayData.advertised_bandwidth, - family: relayData.effective_family, - nickname: relayData.nickname + relayDto.advertised_bandwidth, + family: relayDto.effective_family }) - .catch((error) => this.logger.error('Failed creating relay data model', error.stack)) + .catch( + (error) => this.logger.error('Failed creating relay data model', error.stack) + ) } }) diff --git a/src/verification/dto/relay-verification-result.ts b/src/verification/dto/relay-verification-result.ts index 0ab2497..f4f939d 100644 --- a/src/verification/dto/relay-verification-result.ts +++ b/src/verification/dto/relay-verification-result.ts @@ -2,4 +2,5 @@ export type RelayVerificationResult = | 'OK' | 'AlreadyVerified' | 'AlreadyRegistered' + | 'HardwareProofFailed' | 'Failed' diff --git a/src/verification/hardware-verification.service.spec.ts b/src/verification/hardware-verification.service.spec.ts new file mode 100644 index 0000000..e099464 --- /dev/null +++ b/src/verification/hardware-verification.service.spec.ts @@ -0,0 +1,89 @@ +import { Test, TestingModule } from '@nestjs/testing' +import { ConfigModule } from '@nestjs/config' +import { MongooseModule } from '@nestjs/mongoose' + +import { HardwareVerificationService } from './hardware-verification.service' +import { + VerifiedHardware, + VerifiedHardwareSchema +} from './schemas/verified-hardware' +import { RelaySaleData, RelaySaleDataSchema } from './schemas/relay-sale-data' + +describe('HardwareVerificationService', () => { + let module: TestingModule + let service: HardwareVerificationService + + beforeEach(async () => { + const dbName = 'validator-hardware-verification-service-tests' + + module = await Test.createTestingModule({ + imports: [ + ConfigModule.forRoot(), + MongooseModule.forRoot(`mongodb://localhost/${dbName}`), + MongooseModule.forFeature([ + { name: VerifiedHardware.name, schema: VerifiedHardwareSchema }, + { name: RelaySaleData.name, schema: RelaySaleDataSchema }, + ]), + ], + providers: [HardwareVerificationService], + }).compile() + + service = module.get(HardwareVerificationService) + }) + + afterEach(async () => { + await module.close() + }) + + it('should be defined', () => { + expect(service).toBeDefined() + }) + + it('should check owner of valid nft id', async () => { + const address = '0xe96caef5e3b4d6b3F810679637FFe95D21dEFa5B' + const nftId = BigInt(621) + + const isOwnerOfRelayupNft = await service.isOwnerOfRelayupNft( + address, + nftId + ) + + expect(isOwnerOfRelayupNft).toBe(true) + }) + + it('should check owner of invalid nft id', async () => { + const address = '0xe96caef5e3b4d6b3F810679637FFe95D21dEFa5B' + const nftId = BigInt(999) + + const isOwnerOfRelayupNft = await service.isOwnerOfRelayupNft( + address, + nftId + ) + + expect(isOwnerOfRelayupNft).toBe(false) + }) + + it('should validate hardware serial proofs', async () => { + const nodeId = 'relay' + const nftId = 0 + const deviceSerial = 'c2eeefaa42a50073' + const atecSerial = '01237da6e721fcee01' + const fingerprint = '6CF7AA4F7C8DABCF523DC1484020906C0E0F7A9C' + const address = '01FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF02' + const publicKey = '8ac7f77ca08a2402424608694e76cf9a126351cf62b27204c96b0d5d71887634240bf6a034d08c54dd7ea66c46cec9b97bf9861931bd3e69c2eac899551a66cb' + const signature = 'e84dad1da3bbc25e60d3e54676ad1610172a2239bb571db9031dd8ca1973c4bab68b23f9a94ecab9396433499333963889f4ebcce79e3f219dab93956b4719ef' + + const result = await service.verifyRelaySerialProof( + nodeId, + nftId, + deviceSerial, + atecSerial, + fingerprint, + address, + publicKey, + signature + ) + + expect(result).toBe(true) + }) +}) diff --git a/src/verification/hardware-verification.service.ts b/src/verification/hardware-verification.service.ts new file mode 100644 index 0000000..2bf3c72 --- /dev/null +++ b/src/verification/hardware-verification.service.ts @@ -0,0 +1,400 @@ +import { Injectable, Logger } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import { InjectModel } from '@nestjs/mongoose' +import { bytesToHex } from '@noble/curves/abstract/utils' +import { p256 } from '@noble/curves/p256' +import { createHash } from 'crypto' +import { + Contract as EthersContract, + JsonRpcProvider, + toUtf8Bytes +} from 'ethers' +import { Model } from 'mongoose' + +import relayUpAbi from './interfaces/relay-up-abi' +import { ECPointCompress } from '../util/ec-point-compress' +import { isFingerprintValid } from '../util/fingerprint' +import { isAddressValid } from '../util/address-evm' +import { isHexStringValid } from '../util/hex-string' +import { ValidatedRelay } from '../validation/schemas/validated-relay' +import { VerifiedHardware } from './schemas/verified-hardware' +import { RelaySaleData } from './schemas/relay-sale-data' + +@Injectable() +export class HardwareVerificationService { + private readonly logger = new Logger(HardwareVerificationService.name) + + private mainnetJsonRpc?: string + private mainnetProvider: JsonRpcProvider + + private relayupNftContractAddress?: string + private relayupNftContract?: EthersContract + + constructor( + private readonly config: ConfigService<{ + MAINNET_JSON_RPC: string, + RELAY_UP_NFT_CONTRACT_ADDRESS: string + }>, + @InjectModel(VerifiedHardware.name) + private readonly verifiedHardwareModel: Model, + @InjectModel(RelaySaleData.name) + private readonly relaySaleDataModel: Model + ) { + this.logger.log('Initializing HardwareVerificationService') + + this.relayupNftContractAddress = this.config.get( + 'RELAY_UP_NFT_CONTRACT_ADDRESS', { infer: true } + ) + + this.mainnetJsonRpc = this.config.get( + 'MAINNET_JSON_RPC', + { infer: true } + ) + + if (!this.mainnetJsonRpc) { + this.logger.error('Missing MAINNET_JSON_RPC!') + } else if (!this.relayupNftContractAddress) { + this.logger.error('Missing RELAYUP NFT Contract address!') + } else { + this.mainnetProvider = new JsonRpcProvider(this.mainnetJsonRpc) + this.relayupNftContract = new EthersContract( + this.relayupNftContractAddress, + relayUpAbi, + this.mainnetProvider + ) + + this.logger.log( + `Using RELAYUP NFT Contract: ${this.relayupNftContractAddress}` + ) + } + } + + public async isOwnerOfRelayupNft(address: string, nftId: bigint) { + if (!this.relayupNftContract) { + this.logger.error( + `Could not check owner of RELAYUP NFT #${nftId}: No Contract` + ) + + return false + } + + try { + const owner = await this.relayupNftContract.ownerOf(nftId) + + return address === owner + } catch (error) { + if (error.reason !== 'ERC721: invalid token ID') { + this.logger.error( + `Error thrown checking owner of NFT ID #${nftId}`, + error + ) + } + + return false + } + } + + public async verifyRelaySerialProof( + nodeId: string, + nftId: number, + deviceSerial: string, + atecSerial: string, + fingerprint: string, + address: string, + publicKey: string, + signature: string + ) { + if (!isFingerprintValid(fingerprint)) { + this.logger.log('Invalid fingerprint', fingerprint) + + return false + } + + if (!isAddressValid(address)) { + this.logger.log('Invalid address', address) + + return false + } + + const nodeIdHex = bytesToHex(toUtf8Bytes(nodeId)) + + const isDeviceSerialValid = deviceSerial.length === 16 + && isHexStringValid(deviceSerial) + if (!isDeviceSerialValid) { + this.logger.log('Invalid device serial', deviceSerial) + + return false + } + + const isAtecSerialValid = atecSerial.length === 18 + && isHexStringValid(atecSerial) + if (!isAtecSerialValid) { + this.logger.log('Invalid atec serial', atecSerial) + + return false + } + + const isSignatureFormatValid = signature.length === 128 + && isHexStringValid(signature) + if (!isSignatureFormatValid) { + this.logger.log('Invalid signature', signature) + + return false + } + + const nftIdHex = nftId.toString(16).padStart(4, '0') + const nftIdHexLsb = [ + nftIdHex[2], + nftIdHex[3], + nftIdHex[0], + nftIdHex[1] + ].join('') + const messageHexString = ( + nodeIdHex + + nftIdHexLsb + + deviceSerial + + atecSerial + + fingerprint + + address + ).toLowerCase() + const message = Uint8Array.from( + (messageHexString.match(/.{1,2}/g) || []) + .map((byte) => parseInt(byte, 16)) + ) + const messageHash = createHash('sha256').update(message).digest('hex') + const publicKeyBytes = Uint8Array.from( + (publicKey.match(/.{1,2}/g) || []).map((byte) => parseInt(byte, 16)) + ) + const publicKeyCompressed = ECPointCompress( + publicKeyBytes.slice(0, publicKeyBytes.length / 2), + publicKeyBytes.slice(publicKeyBytes.length / 2) + ) + + return p256.verify(signature, messageHash, publicKeyCompressed) + } + + private async validateDeviceSerial( + fingerprint: string, + deviceSerial?: string + ): Promise { + if (!deviceSerial) { + this.logger.log( + `Missing Device Serial in hardware info for relay [${fingerprint}]` + ) + + return false + } + + const existingVerifiedHardwareByDeviceSerial = await this + .verifiedHardwareModel + .exists({ deviceSerial }) + .exec() + + if (existingVerifiedHardwareByDeviceSerial) { + this.logger.log( + `Relay [${fingerprint}] tried to verify with ` + + `Device Serial [${deviceSerial}], but it was already verified` + ) + + return false + } + + return true + } + + private async validateAtecSerial( + fingerprint: string, + atecSerial?: string + ): Promise { + if (!atecSerial) { + this.logger.log( + `Missing ATEC Serial in hardware info for relay [${fingerprint}]` + ) + + return false + } + + const existingVerifiedHardwareByAtecSerial = await this + .verifiedHardwareModel + .exists({ atecSerial }) + .exec() + if (existingVerifiedHardwareByAtecSerial) { + this.logger.log( + `Relay [${fingerprint}] tried to verify with ` + + `ATEC Serial [${atecSerial}], but it was already verified.` + ) + + return false + } + + return true + } + + /** + * Check deviceSerial against known RelaySaleData + * - if no known RelaySaleData, fail + * - if nft ids don't match, fail + * - if address does not currently own nft id, fail + * - else, pass and return parsed nft id + * + * @todo Handle nftId of 0 for future relay sales / known relays + * + * @param fingerprint + * @param address + * @param deviceSerial + * @param nftId + * @returns Promise<{ valid: false } | { valid: true, nftId: number }> + */ + private async validateNftIdForAddressAndDeviceSerial( + fingerprint: string, + address: string, + deviceSerial: string, + nftId?: string + ): Promise<{ valid: false } | { valid: true, nftId: number }> { + if (!nftId) { + this.logger.log( + `Missing NFT ID in hardware info for relay [${fingerprint}]` + ) + + return { valid: false } + } + const parsedNftId = Number.parseInt(nftId) + const isNftIdValid = Number.isInteger(parsedNftId) + if (!isNftIdValid) { + this.logger.log( + `Invalid NFT ID [${parsedNftId}] in hardware info for ` + + `relay [${fingerprint}]` + ) + } + const existingVerifiedHardwareByNftId = await this + .verifiedHardwareModel + .exists({ nftId: parsedNftId }) + .exec() + if (existingVerifiedHardwareByNftId) { + this.logger.log( + `Relay [${fingerprint}] tried to verify with ` + + `NFT ID [${parsedNftId}], but it was already verified` + ) + + return { valid: false } + } + + const relaySaleData = await this + .relaySaleDataModel + .findOne({ deviceSerial }) + .exec() + if (!relaySaleData) { + this.logger.log( + `Relay [${fingerprint}] tried to verify with ` + + `NFT ID [${parsedNftId}] and Device Serial [${deviceSerial}], ` + + `but no known RelaySaleData matches` + ) + + return { valid: false } + } + if (relaySaleData.nftId !== parsedNftId) { + this.logger.log( + `Relay [${fingerprint}] tried to verify with ` + + `NFT ID [${parsedNftId}] and Device Serial [${deviceSerial}], ` + + `but we expected NFT ID [${relaySaleData.nftId}]` + ) + + return { valid: false } + } + + const isAddressOwnerOfNftId = await this.isOwnerOfRelayupNft( + address, + BigInt(parsedNftId) + ) + if (!isAddressOwnerOfNftId) { + this.logger.debug(`NFT ID [${parsedNftId}] is not owned by ${address}`) + + return { valid: false } + } + + return { valid: true, nftId: parsedNftId } + } + + public async isHardwareProofValid(relay: ValidatedRelay): Promise { + if (!relay.hardware_info) { return false } + + const { nftid, serNums, pubKeys, certs } = relay.hardware_info + + const deviceSerial = serNums?.find(s => s.type === 'DEVICE')?.number + const isDeviceSerialValid = await this.validateDeviceSerial( + relay.fingerprint, + deviceSerial + ) + if (!isDeviceSerialValid) { return false } + + const atecSerial = serNums?.find(s => s.type === 'ATEC')?.number + const isAtecSerialValid = await this.validateAtecSerial( + relay.fingerprint, + atecSerial + ) + if (!isAtecSerialValid) { return false } + + const publicKey = pubKeys + ?.find(p => p.type === 'DEVICE') + ?.number + if (!publicKey) { + this.logger.debug( + `Missing Public Key in hardware info for relay [${relay.fingerprint}]` + ) + + return false + } + + const signature = certs + ?.find(c => c.type === 'DEVICE') + ?.signature + if (!signature) { + this.logger.debug( + `Missing Signature in hardware info for relay [${relay.fingerprint}]` + ) + + return false + } + + const validateNftResult = await this.validateNftIdForAddressAndDeviceSerial( + relay.fingerprint, + relay.ator_address, + deviceSerial!, + nftid + ) + if (!validateNftResult.valid) { return false } + + const isHardwareProofValid = await this.verifyRelaySerialProof( + 'relay', + validateNftResult.nftId, + deviceSerial!, + atecSerial!, + relay.fingerprint, + relay.ator_address, + publicKey, + signature + ) + + if (!isHardwareProofValid) { + this.logger.debug( + `Hardware info proof failed verification for ` + + `relay [${relay.fingerprint}]` + ) + + return false + } + + await this.verifiedHardwareModel.create({ + verified_at: Date.now(), + deviceSerial, + atecSerial, + fingerprint: relay.fingerprint, + address: relay.ator_address, + publicKey, + signature, + nftId: validateNftResult.nftId + }) + + return true + } +} diff --git a/src/verification/interfaces/relay-up-abi.ts b/src/verification/interfaces/relay-up-abi.ts new file mode 100644 index 0000000..d377345 --- /dev/null +++ b/src/verification/interfaces/relay-up-abi.ts @@ -0,0 +1,5 @@ +export default [ + 'function totalSupply() public view override returns (uint256)', + 'function balanceOf(address owner) public view returns (uint256)', + 'function ownerOf(uint256 tokenId) public view returns (address)' +] diff --git a/src/verification/schemas/relay-sale-data.ts b/src/verification/schemas/relay-sale-data.ts new file mode 100644 index 0000000..e212fd3 --- /dev/null +++ b/src/verification/schemas/relay-sale-data.ts @@ -0,0 +1,14 @@ +import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' +import { HydratedDocument } from 'mongoose' + +@Schema() +export class RelaySaleData { + @Prop({ type: Number, required: true }) + nftId: number + + @Prop({ type: String, required: true }) + serial: string +} + +export type RelaySaleDataDocument = HydratedDocument +export const RelaySaleDataSchema = SchemaFactory.createForClass(RelaySaleData) diff --git a/src/verification/schemas/verified-hardware.ts b/src/verification/schemas/verified-hardware.ts new file mode 100644 index 0000000..1a8ec3d --- /dev/null +++ b/src/verification/schemas/verified-hardware.ts @@ -0,0 +1,33 @@ +import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' +import { HydratedDocument } from 'mongoose' + +@Schema() +export class VerifiedHardware { + @Prop({ type: Number, required: true }) + verified_at: number + + @Prop({ type: String, required: true }) + deviceSerial: string + + @Prop({ type: String, required: true }) + atecSerial: string + + @Prop({ type: String, required: true }) + fingerprint: string + + @Prop({ type: String, required: true }) + address: string + + @Prop({ type: String, required: true }) + publicKey: string + + @Prop({ type: String, required: true }) + signature: string + + @Prop({ type: Number, required: false }) + nftId?: number +} + +export type VerifiedHardwareDocument = HydratedDocument +export const VerifiedHardwareSchema = + SchemaFactory.createForClass(VerifiedHardware) diff --git a/src/verification/verification.module.ts b/src/verification/verification.module.ts index be7bccb..237eb43 100644 --- a/src/verification/verification.module.ts +++ b/src/verification/verification.module.ts @@ -1,18 +1,28 @@ import { Module } from '@nestjs/common' -import { VerificationService } from './verification.service' import { ConfigModule, ConfigService } from '@nestjs/config' import { MongooseModule } from '@nestjs/mongoose' +import { HttpModule } from '@nestjs/axios' + +import { VerificationService } from './verification.service' import { VerificationData, VerificationDataSchema, } from './schemas/verification-data' -import { HttpModule } from '@nestjs/axios' +import { + VerifiedHardware, + VerifiedHardwareSchema +} from './schemas/verified-hardware' +import { HardwareVerificationService } from './hardware-verification.service' +import { RelaySaleData, RelaySaleDataSchema } from './schemas/relay-sale-data' + @Module({ imports: [ ConfigModule, MongooseModule.forFeature([ { name: VerificationData.name, schema: VerificationDataSchema }, + { name: VerifiedHardware.name, schema: VerifiedHardwareSchema }, + { name: RelaySaleData.name, schema: RelaySaleDataSchema }, ]), HttpModule.registerAsync({ inject: [ConfigService], @@ -30,9 +40,9 @@ import { HttpModule } from '@nestjs/axios' { infer: true }, ), }), - }), + }) ], - providers: [VerificationService], - exports: [VerificationService], + providers: [VerificationService, HardwareVerificationService], + exports: [VerificationService, HardwareVerificationService], }) export class VerificationModule {} diff --git a/src/verification/verification.service.spec.ts b/src/verification/verification.service.spec.ts index e57c0bb..630207a 100644 --- a/src/verification/verification.service.spec.ts +++ b/src/verification/verification.service.spec.ts @@ -1,19 +1,27 @@ import { Test, TestingModule } from '@nestjs/testing' -import { VerificationService } from './verification.service' import { ConfigModule } from '@nestjs/config' import { MongooseModule } from '@nestjs/mongoose' +import { HttpModule } from '@nestjs/axios' + +import { VerificationService } from './verification.service' import { VerificationData, VerificationDataSchema, } from './schemas/verification-data' +import { + VerifiedHardware, + VerifiedHardwareSchema +} from './schemas/verified-hardware' describe('VerificationService', () => { + let module: TestingModule let service: VerificationService - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ + beforeEach(async () => { + module = await Test.createTestingModule({ imports: [ ConfigModule.forRoot(), + HttpModule, MongooseModule.forRoot( 'mongodb://localhost/validator-validation-service-tests', ), @@ -22,6 +30,10 @@ describe('VerificationService', () => { name: VerificationData.name, schema: VerificationDataSchema, }, + { + name: VerifiedHardware.name, + schema: VerifiedHardwareSchema + } ]), ], providers: [VerificationService], @@ -30,6 +42,10 @@ describe('VerificationService', () => { service = module.get(VerificationService) }) + afterEach(async () => { + await module.close() + }) + it('should be defined', () => { expect(service).toBeDefined() }) diff --git a/src/verification/verification.service.ts b/src/verification/verification.service.ts index bb93f93..e21d035 100644 --- a/src/verification/verification.service.ts +++ b/src/verification/verification.service.ts @@ -2,11 +2,8 @@ import { Injectable, Logger } from '@nestjs/common' import { firstValueFrom, catchError } from 'rxjs' import { Contract, LoggerFactory, Tag, Warp, WarpFactory } from 'warp-contracts' import { - AddClaimable, AddClaimableBatched, AddRegistrationCredits, - IsClaimable, - IsVerified, RelayRegistryState, SetFamilies } from './interfaces/relay-registry' @@ -17,8 +14,8 @@ import { } from 'warp-contracts-plugin-signature/server' import { EthersExtension } from 'warp-contracts-plugin-ethers' import { StateUpdatePlugin } from 'warp-contracts-subscription-plugin' -import { RelayVerificationResult } from './dto/relay-verification-result' import { VerificationData } from './schemas/verification-data' +import { VerifiedHardware } from './schemas/verified-hardware' import { VerificationResults } from './dto/verification-result-dto' import { ValidatedRelay } from 'src/validation/schemas/validated-relay' import { Model } from 'mongoose' @@ -27,9 +24,12 @@ import Bundlr from '@bundlr-network/client' import { RelayValidationStatsDto } from './dto/relay-validation-stats' import { HttpService } from '@nestjs/axios' import { AxiosError } from 'axios' -import { DreRelayRegistryResponse } from './interfaces/dre-relay-registry-response' +import { + DreRelayRegistryResponse +} from './interfaces/dre-relay-registry-response' import { setTimeout } from 'node:timers/promises' import _ from 'lodash' +import { HardwareVerificationService } from './hardware-verification.service' @Injectable() export class VerificationService { @@ -63,6 +63,8 @@ export class VerificationService { @InjectModel(VerificationData.name) private readonly verificationDataModel: Model, private readonly httpService: HttpService, + private readonly hardwareVerificationService: + HardwareVerificationService ) { LoggerFactory.INST.logLevel('error') @@ -163,7 +165,7 @@ export class VerificationService { function: 'addRegistrationCredits', credits: [{ address, fingerprint }] }, { - tags: [new Tag('EVM-TX', tx)] + tags: [ new Tag('EVM-TX', tx) ] }) this.logger.log( @@ -711,7 +713,10 @@ export class VerificationService { } = await this.getRelayRegistryStatuses() const alreadyClaimableFingerprints = Object.keys(claimable) const alreadyVerifiedFingerprints = Object.keys(verified) - const relaysToAddAsClaimable = [] + const relaysToAddAsClaimable: { + relay: ValidatedRelay, + isHardwareProofValid?: boolean + }[] = [] for (const relay of relays) { const isAlreadyClaimable = alreadyClaimableFingerprints.includes( relay.fingerprint @@ -734,8 +739,17 @@ export class VerificationService { `Already verified relay [${relay.fingerprint}]`, ) results.push({ relay, result: 'AlreadyVerified' }) + } else if (!relay.hardware_info) { + relaysToAddAsClaimable.push({ relay }) } else { - relaysToAddAsClaimable.push(relay) + const isHardwareProofValid = await this + .hardwareVerificationService + .isHardwareProofValid(relay) + if (isHardwareProofValid) { + relaysToAddAsClaimable.push({relay, isHardwareProofValid }) + } else { + results.push({ relay, result: 'HardwareProofFailed' }) + } } } @@ -750,20 +764,25 @@ export class VerificationService { for (const relayBatch of relayBatches) { await setTimeout(5000) this.logger.debug( - `Starting to add a batch of claimable relays for ${relayBatch} relays [${relayBatch.map(r => r.fingerprint)}]` + `Starting to add a batch of claimable relays for ${relayBatch} relays [${relayBatch.map(r => r.relay.fingerprint)}]` ) const response = await this.relayRegistryContract .writeInteraction({ function: 'addClaimableBatched', relays: relayBatch.map( ({ - fingerprint, - ator_address, - nickname + relay: { + fingerprint, + ator_address, + nickname + }, + isHardwareProofValid }) => ({ fingerprint, address: ator_address, - nickname + nickname, + hardwareVerified: + isHardwareProofValid || undefined }) ) }) @@ -775,13 +794,13 @@ export class VerificationService { } } catch (error) { this.logger.error( - `Exception when verifying relays [${relaysToAddAsClaimable.map(r => r.fingerprint)}]`, + `Exception when verifying relays [${relaysToAddAsClaimable.map(({ relay }) => relay.fingerprint)}]`, error.stack, ) return results.concat( relaysToAddAsClaimable.map( - relay => ({ relay, result: 'Failed' }) + ({ relay }) => ({ relay, result: 'Failed' }) ) ) } @@ -792,7 +811,7 @@ export class VerificationService { } return results.concat( - relaysToAddAsClaimable.map(relay => ({ relay, result: 'OK' })) + relaysToAddAsClaimable.map(({ relay }) => ({ relay, result: 'OK' })) ) } }