diff --git a/.github/workflows/validate-PR.yml b/.github/workflows/validate-PR.yml index 388f492f5..15679ca84 100644 --- a/.github/workflows/validate-PR.yml +++ b/.github/workflows/validate-PR.yml @@ -20,10 +20,8 @@ jobs: ## for local act testing # - run: npm install -g yarn - + - run: yarn install - name: Validate PR - run: yarn validate-PR - - + run: yarn validate-PR >> $GITHUB_STEP_SUMMARY diff --git a/package.json b/package.json index 7ae8aa0c4..6df028aa6 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "@actions/exec": "^1.1.1", "@actions/github": "^5.1.1", "@solana/web3.js": "^1.73.2", + "@types/minimist": "^1.2.5", "csv-writer": "^1.6.0", + "minimist": "^1.2.8", "node-downloader-helper": "^2.1.6", "node-fetch": "^2.6.6" }, diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 000000000..2aaa7eb89 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,13 @@ +import { validateValidatedTokensCsv } from "./logic"; +import minimist from "minimist"; +// CLI entrypoint which accepts an argument +(async () => { + try { + const argv = minimist(process.argv.slice(2)); + const returnCode = await validateValidatedTokensCsv(argv._[0]); + process.exit(returnCode); + } + catch (error: any) { + console.log(error.message) + } +})(); \ No newline at end of file diff --git a/src/logic.ts b/src/logic.ts new file mode 100644 index 000000000..ec5ca4527 --- /dev/null +++ b/src/logic.ts @@ -0,0 +1,91 @@ +import * as core from "@actions/core"; +import { exec } from "@actions/exec"; +import { canOnlyAddOneToken, detectDuplicateSymbol, detectDuplicateMints as detectDuplicateMints, validMintAddress, noEditsToPreviousLinesAllowed } from "./utils/validate"; +import { ValidatedTokensData } from "./types/types"; +import { indexToLineNumber } from "./utils/validate"; +import { parse } from "csv-parse/sync"; +import fs from "fs"; + +export async function validateValidatedTokensCsv(filename: string): Promise { + const [records, recordsRaw] = parseCsv(filename); + + const recordsPreviousRaw = await gitPreviousVersion("validated-tokens.csv"); + fs.writeFileSync(".validated-tokens-0.csv", recordsPreviousRaw); + const [recordsPrevious, _] = parseCsv(".validated-tokens-0.csv") + + let duplicateSymbols; + let duplicateMints; + let attemptsToAddMultipleTokens; + let invalidMintAddresses; + let notCommunityValidated; + let noEditsAllowed; + + duplicateSymbols = detectDuplicateSymbol(recordsPrevious, records); + duplicateMints = detectDuplicateMints(records); + attemptsToAddMultipleTokens = canOnlyAddOneToken(recordsPrevious, records) + invalidMintAddresses = validMintAddress(records); + noEditsAllowed = noEditsToPreviousLinesAllowed(recordsPrevious, records); + // notCommunityValidated = validCommunityValidated(records); + + console.log("No More Duplicate Symbols:", duplicateSymbols); + console.log("Duplicate Mints:", duplicateMints); + console.log("Attempts to Add Multiple Tokens:", attemptsToAddMultipleTokens); + console.log("Invalid Mint Addresses:", invalidMintAddresses); + console.log("Not Community Validated:", notCommunityValidated); + console.log("Edits to Existing Tokens:", noEditsAllowed); + return (duplicateSymbols + duplicateMints + attemptsToAddMultipleTokens + invalidMintAddresses + noEditsAllowed) +} + +// Get previous version of validated-tokens.csv from last commit +async function gitPreviousVersion(path: string): Promise { + let prevVersion = ""; + let gitCmdError = ""; + + try { + await exec("git", ["show", `origin/main:${path}`], { + listeners: { + stdout: (data: Buffer) => { + prevVersion += data.toString(); + }, + stderr: (data: Buffer) => { + gitCmdError += data.toString(); + }, + }, + silent: true + }); + } catch (error: any) { + core.setFailed(error.message); + } + + if (gitCmdError) { + core.setFailed(gitCmdError); + } + return prevVersion; +} + +function parseCsv(filename: string): [ValidatedTokensData[], string] { + const recordsRaw = fs.readFileSync(filename, "utf8") + const r = parse(recordsRaw, { + columns: true, + skip_empty_lines: true, + }); + const records = csvToRecords(r); + return [records, recordsRaw]; +} + +function csvToRecords(r: any): ValidatedTokensData[] { + const records: ValidatedTokensData[] = []; + r.forEach((record: any, i: number) => { + const rec: ValidatedTokensData = { + Name: record.Name, + Symbol: record.Symbol, + Mint: record.Mint, + Decimals: record.Decimals, + LogoURI: record.LogoURI, + "Community Validated": JSON.parse(record["Community Validated"]), + Line: indexToLineNumber(i) + }; + records.push(rec); + }); + return records; +} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 13050c0b2..6acee0e1b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,75 +1,13 @@ import * as core from "@actions/core"; -import { exec } from "@actions/exec"; -import { parseGitPatch } from "./utils/parse"; -import { validateGitPatch } from "./utils/validate"; -import { getValidated } from "./utils/get-jup-strict"; -import { ValidatedSet, ValidationError } from "./types/types"; -import { parse } from "csv-parse/sync"; -import fs from "fs"; -import assert from "assert"; - -function validateValidatedTokensCsv() { - const records = parse(fs.readFileSync("validated-tokens.csv", "utf8"), { - columns: true, - skip_empty_lines: true, - }); - assert.deepStrictEqual(Object.keys(records[0]), [ - "Name", - "Symbol", - "Mint", - "Decimals", - "LogoURI", - "Community Validated", - ]); -} - -// Validates diff between validated-tokens.csv in the branch vs origin/main -async function getDiffAndValidate(): Promise { - let gitDiff = ""; - let gitDiffError = ""; - +import { validateValidatedTokensCsv } from "./logic"; +// Github Actions entrypoint +(async () => { try { - await exec("git", ["diff", "origin/main", "validated-tokens.csv"], { - listeners: { - stdout: (data: Buffer) => { - gitDiff += data.toString(); - }, - stderr: (data: Buffer) => { - gitDiffError += data.toString(); - }, - }, - }); - } catch (error: any) { - core.setFailed(error.message); - } - - if (gitDiffError) { - core.setFailed(gitDiffError); + const returnCode = await validateValidatedTokensCsv("validated-tokens.csv"); + process.exit(returnCode); } - - // core.debug(`Git diff: ${gitDiff}`) - - // Get Jup tokens that are in the strict list to check for duplicates. - let validatedSet: ValidatedSet; - try { - validatedSet = await getValidated(); - - const errors: ValidationError[][] = []; - - parseGitPatch(gitDiff).forEach((patch) => { - const patchErrors = validateGitPatch(patch, validatedSet); - if (patchErrors && patchErrors.length > 0) { - errors.push(patchErrors); - } - }); - - if (errors.length > 0) { - core.setFailed(errors.join(",")); - } - } catch (error: any) { + catch (error: any) { core.setFailed(error.message); + console.log(error.message) } -} - -validateValidatedTokensCsv(); -// getDiffAndValidate(); +})(); diff --git a/src/types/types.ts b/src/types/types.ts index 098c474c4..cb5b98d42 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -68,12 +68,13 @@ export enum ValidationError { UNRELATED_CODE = "Changes to unrelated code are not allowed", MULTIPLE_TOKENS = "Only one token can be added at a time", DUPLICATE_NAME = "Token name already exists", - DUPLICATE_SYMBOL = "Token symbol already exists", + DUPLICATE_SYMBOL = "Token symbol already exists, please forbid even more duplicates", DUPLICATE_MINT = "Mint already exists", - INVALID_MINT = "Invalid mint address, not on ed25519 curve", + INVALID_MINT = "Invalid mint address, not base58 decodable", INVALID_DECIMALS = "Invalid decimals", INVALID_IMAGE_URL = "Invalid image URL", INVALID_COMMUNITY_VALIDATED = "Invalid community validated", + CHANGES_DISCOURAGED = "Tokens already in the CSV should not be edited" } export interface WormholeData { @@ -97,5 +98,12 @@ export interface ValidatedTokensData { Mint: string; Decimals: string; LogoURI: string; - "Community Validated": "false" | "true"; + "Community Validated": boolean; + Line: number; } + +export interface DuplicateSymbol { + Name: string; + Symbol: string; + Mint: string; +} \ No newline at end of file diff --git a/src/utils/duplicate-symbols.ts b/src/utils/duplicate-symbols.ts new file mode 100644 index 000000000..cdec12e01 --- /dev/null +++ b/src/utils/duplicate-symbols.ts @@ -0,0 +1,94 @@ +import { DuplicateSymbol } from "../types/types"; + +export const allowedDuplicateSymbols: DuplicateSymbol[] = [ + { + Name: 'Sallar', + Symbol: 'ALL', + Mint: '5EjMX8pZkJtkbJwT5vzJhzTexBPhECFUrq5ndD3UkQD1', + }, + { + Name: 'Arbitrum (Portal from Arbitrum)', + Symbol: 'ARB', + Mint: '8LH3QMo7xkMJx85Kg4pfiQY1g1ZgiVEe1KktSpaT89mP', + }, + { + Name: 'AVAX (Portal)', + Symbol: 'AVAX', + Mint: 'KgV1GvrHQmRBY8sHQQeUKwTm2r2h8t4C8qt12Cw1HVE', + }, + { + Name: 'Boo', + Symbol: 'BOO', + Mint: 'FfpyoV365c7iR8QQg5NHGCXQfahbqzY67B3wpzXkiLXr', + }, + { + Name: 'Food', + Symbol: 'FOOD', + Mint: 'foodQJAztMzX1DKpLaiounNe2BDMds5RNuPC6jsNrDG', + }, + { + Name: 'Helios Rising: Fuel', + Symbol: 'FUEL', + Mint: 'ZViNy4z9dquon7AVgr6neK1RCohTRFH8WTUMUsjhWhe', + }, + { + Name: 'Starbots GEAR', + Symbol: 'GEAR', + Mint: '23WuycvPjEuzJTsBPBZqnbFZFcBtBKAMTowUDHwagkuD', + }, + { + Name: 'GM', + Symbol: 'GM', + Mint: '3acxNNmfdKKZj9i35P4VDBFm74Ufdt8ojKWceVGynwC5', + }, + { + Name: 'LILY', + Symbol: 'LILY', + Mint: '7FYvphuZtRxB7BZd8PZ65yZmEEuWYYdxiHubthyd38BE', + }, + { + Name: 'MILK', + Symbol: 'MILK', + Mint: 'MLKmUCaj1dpBY881aFsrBwR9RUMoKic8SWT3u1q5Nkj', + }, + { + Name: 'NANA Token', + Symbol: 'NANA', + Mint: 'HxRELUQfvvjToVbacjr9YECdfQMUqGgPYB68jVDYxkbr', + }, + { + Name: 'NINJA TURTLES', + Symbol: 'NINJA', + Mint: 'DFrJxDoLMYt6bNYeNe8Wrjzj2UPUSLZLEMMYBLuTKcTk', + }, + { + Name: 'Only Possible On Solana', + Symbol: 'OPOS', + Mint: 'BqVHWpwUDgMik5gbTciFfozadpE2oZth5bxCDrgbDt52', + }, + { + Name: 'PEPESOLANA', + Symbol: 'PEPE', + Mint: 'CYuXNHURE8cF3rQc1687DZnmsWx9ATNG3mZPFmryu22S', + }, + { + Name: 'Rocky', + Symbol: 'ROCKY', + Mint: '4icEZCrEYNop2ZaMMCkRHaNzkt6xG9BpijMCQV7mpw6Z', + }, + { + Name: 'Soul Scanner', + Symbol: 'SOUL', + Mint: 'J4ywFdm8H7hjwKzCaEQujhkDRfCnRviVnHMvFNDAoLNQ', + }, + { + Name: 'sRLY (Rally Solana)', + Symbol: 'sRLY', + Mint: 'sRLY3migNrkC1HLgqotpvi66qGkdNedqPZ9TJpAQhyh', + }, + { + Name: 'WHEY', + Symbol: 'WHEY', + Mint: 'Ue4yjkPjA4QGis37eWbBsnqfzyK83BtY4AioDETp3Ab', + } +] \ No newline at end of file diff --git a/src/utils/parse.ts b/src/utils/parse.ts deleted file mode 100755 index d4e34fefe..000000000 --- a/src/utils/parse.ts +++ /dev/null @@ -1,65 +0,0 @@ -import {Patch} from '../types/types'; - -// This is a util function to parse a git diff patch into a more usable format -export function parseGitPatch(patch: string): Patch[] { - const lines = patch.split('\n'); - - let currentFiles: [string, string]; - let currentPatch: Patch | undefined; - const patches: Patch[] = []; - - // Need to parse this line by line - lines.forEach(line => { - const matches = line.match(/^diff --git a\/(.*?) b\/(.*)$/m); - - if (matches) { - currentFiles = [matches[1], matches[2]]; - return; - } - - const patchMatches = line.match( - /^@@ -(\d+)(?:,|)(\d*) \+(\d+)(?:,|)(\d*) @@/ - ); - - if (patchMatches) { - // push old patch - if (currentPatch) { - patches.push(currentPatch); - } - - currentPatch = { - removed: { - file: currentFiles[0], - start: Number(patchMatches[1]), - end: Number(patchMatches[1]) + Number(patchMatches[2]), - lines: [], - }, - added: { - file: currentFiles[1], - start: Number(patchMatches[3]), - end: Number(patchMatches[3]) + Number(patchMatches[4]), - lines: [], - }, - }; - return; - } - - const contentMatches = line.match(/^(-|\+)(.*)$/); - - if (contentMatches) { - // This can match `--- a/` and `+++ b/`, so ignore if no `currentPatch` object - if (!currentPatch) { - return; - } - - const patchType = contentMatches[1] === '-' ? 'removed' : 'added'; - currentPatch[patchType].lines.push(contentMatches[2]); - } - }); - - if (currentPatch) { - patches.push(currentPatch); - } - - return patches; -} \ No newline at end of file diff --git a/src/utils/validate.ts b/src/utils/validate.ts index a26b7389c..50ca5f07e 100644 --- a/src/utils/validate.ts +++ b/src/utils/validate.ts @@ -1,58 +1,162 @@ -import { Patch, ValidatedSet, ValidationError } from "../types/types"; +import { DuplicateSymbol, ValidatedTokensData, ValidationError } from "../types/types"; +import { allowedDuplicateSymbols } from "./duplicate-symbols"; import { PublicKey } from "@solana/web3.js"; -// Validates PR changes to the validated tokens csv file. -// Checks duplicates to JUP strict list and invalid inputs. -export function validateGitPatch(patch: Patch, validatedSet: ValidatedSet): ValidationError[] { - // console.log("Processing patch", patch); - const errors: ValidationError[] = []; +export function indexToLineNumber(index: number): number { + return index + 2; +} - // TODO: Flag changes to unrelated files - // ... +export function detectDuplicateMints(tokens: ValidatedTokensData[]): number { + let errorCount = 0; + const map = new Map(); + tokens.forEach((token, i) => { + if (map.has(token.Mint)) { + console.log(ValidationError.DUPLICATE_MINT) + console.log("Existing", map.get(token.Mint), "Duplicate", `(line ${token.Line})`, token); + errorCount++; + } else { + map.set(token.Mint, token); + } + }); + return errorCount; +} - // Flag removals - if (patch.removed.lines.length > 0) { - errors.push(ValidationError.UNRELATED_CODE); - } +export function detectDuplicateSymbol(tokensPreviously: ValidatedTokensData[], tokens: ValidatedTokensData[]): number { + const tokensPrevBySymbol = new Map(); + const tokensPrevByMint = new Map(); + // If we put tokens into a map by symbol, only tokens with duplicate symbols will be leftover. + const duplicateSymbolsPrev: ValidatedTokensData[] = []; + tokensPreviously.forEach((token, i) => { + if (!tokensPrevBySymbol.has(token.Symbol)) { + tokensPrevBySymbol.set(token.Symbol, token); + tokensPrevByMint.set(token.Mint, token); + } else { + duplicateSymbolsPrev.push(token); + } + }); - // Flag multiple line additions - if (patch.added.lines.length > 1) { - errors.push(ValidationError.MULTIPLE_TOKENS); - } + const tokensBySymbol = new Map(); + const tokensByMint = new Map(); + const duplicateSymbols: ValidatedTokensData[] = []; + tokens.forEach((token, i) => { + if (!tokensBySymbol.has(token.Symbol)) { + tokensBySymbol.set(token.Symbol, token); + tokensByMint.set(token.Mint, token); + } else { + duplicateSymbols.push(token); + } + }); + duplicateSymbols.sort((a, b) => a.Symbol.localeCompare(b.Symbol)); - const [tokenName, symbol, mint, decimals, imageURL, isCommunity] = - patch.added.lines[0].split(","); + // as of writing this code, we already have 18 tokens with duplicate symbols. the point is to make sure this number doesn't grow. + if (duplicateSymbols.length > allowedDuplicateSymbols.length) { + // we have a problem. we have more duplicate symbols than we did before. + // but what exactly was duplicated? + const sortedDuplicateSymbols: string[] = duplicateSymbols + .map((token) => token.Symbol) + .sort() - // Flag duplicates - if (validatedSet.names.has(tokenName)) { - errors.push(ValidationError.DUPLICATE_NAME); - } - - if (validatedSet.symbols.has(symbol)) { - errors.push(ValidationError.DUPLICATE_SYMBOL); - } - - if (validatedSet.mints.has(mint)) { - errors.push(ValidationError.DUPLICATE_MINT); + const theNewDuplicateSymbol = xorTokens(duplicateSymbols, allowedDuplicateSymbols) + console.log(ValidationError.DUPLICATE_SYMBOL, theNewDuplicateSymbol); + console.log(`(the last version of the CSV file had ${duplicateSymbolsPrev.length} duplicates)`) } + return duplicateSymbols.length - allowedDuplicateSymbols.length; +} + +function xorTokens(tokens: ValidatedTokensData[], allowedDuplicates: DuplicateSymbol[]): ValidatedTokensData[] { + const tokensSymbolMint = tokens.map((token) => `${token.Symbol}-${token.Mint}`).sort(); + const allowedDuplicatesSymbolMint = allowedDuplicates.map((token) => `${token.Symbol}-${token.Mint}`).sort(); + + const set1 = new Set(tokensSymbolMint); + const set2 = new Set(allowedDuplicatesSymbolMint); - // Flag invalid mint address - if(!PublicKey.isOnCurve(new PublicKey(mint))) { - errors.push(ValidationError.INVALID_MINT); - } + const setDifference = new Set([...set1, ...set2].filter(value => !set1.has(value) || !set2.has(value))); + // [ 'ARB-9xzZzEHsKnwFL1A3DyFJwj36KnZj3gZ7g4srWp9YTEoh' ] - if (isNaN(Number(decimals)) || Number(decimals) < 0 || Number(decimals) > 9) { - errors.push(ValidationError.INVALID_DECIMALS); + const duplicateSymbolMints = Array.from(setDifference).map((x) => x.split("-")) + // [['ARB', '9xzZzEHsKnwFL1A3DyFJwj36KnZj3gZ7g4srWp9YTEoh']...] + + const answer : ValidatedTokensData[] = []; + for (const [symbol, mint] of duplicateSymbolMints) { + const matchingElement = tokens.find((token) => token.Symbol === symbol && token.Mint === mint); + if(matchingElement) { + answer.push(matchingElement) + } } + return answer +}; + +export function canOnlyAddOneToken(prevTokens: ValidatedTokensData[], tokens: ValidatedTokensData[]): number { + let errorCount = 0; + const diffLength = tokens.length - prevTokens.length; - if (isCommunity !== "true") { - errors.push(ValidationError.INVALID_COMMUNITY_VALIDATED); + if (diffLength > 1) { + const offendingTokens: ValidatedTokensData[] = []; + for (let i = prevTokens.length; i < tokens.length; i++) { + offendingTokens.push(tokens[i]); + } + console.log(ValidationError.MULTIPLE_TOKENS, offendingTokens); + errorCount++; } + return errorCount; +} - // TODO: match with onchain data - // .... - // ... +export function validMintAddress(tokens: ValidatedTokensData[]): number { + let errorCount = 0; + + tokens.forEach((token, i) => { + try { + // will fail if mint address is not valid base58 + // a mint doesn't have to be on the edd25519 curve though + const _ = new PublicKey(token.Mint) + } catch (error) { + console.log(ValidationError.INVALID_MINT, `(line ${token.Line})`, token, error); + errorCount++; + } + }); + return errorCount; +} - // console.log("Patch Errors", errors); - return errors; +export function validDecimals(tokens: ValidatedTokensData[]): number { + let errorCount = 0; + tokens.forEach((token) => { + if (isNaN(Number(token.Decimals)) || Number(token.Decimals) < 0 || Number(token.Decimals) > 9) { + console.log(ValidationError.INVALID_DECIMALS, token); + errorCount++; + } + }); + return errorCount; } + +export function areRecordsEqual(r1: ValidatedTokensData, r2: ValidatedTokensData): boolean { + return ( + r1.Name === r2.Name && + r1.Symbol === r2.Symbol && + r1.Mint === r2.Mint && + r1.Decimals === r2.Decimals && + r1.LogoURI === r2.LogoURI && + r1["Community Validated"] === r2["Community Validated"] + ); +} + +// this function only works properly if there are no duplicate mints +export function noEditsToPreviousLinesAllowed(prevTokens: ValidatedTokensData[], tokens: ValidatedTokensData[]): number { + let errorCount = 0; + const map = new Map(); + prevTokens.forEach((token) => { + map.set(token.Mint, token) + }) + + tokens.forEach((token) => { + const prevToken = map.get(token.Mint); + if (prevToken !== undefined) { + // if prevToken is undefined, this means that the new file has a token that + // the older one didn't. that's completely normal + if (!areRecordsEqual(prevToken, token)) { + console.log(ValidationError.CHANGES_DISCOURAGED, prevToken, token) + errorCount++; + } + } + }) + return errorCount; +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 0c8a130da..bc07fd9e2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,7 @@ "strict": true /* Enable all strict type-checking options. */, "skipLibCheck": true /* Skip type checkng all .d.ts files. */, "allowJs": true, + "sourceMap": true }, "include": ["src/**/*"], "exclude": ["node_modules"] diff --git a/yarn.lock b/yarn.lock index 3a5ffb582..9a6abb5ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -183,6 +183,11 @@ dependencies: "@types/node" "*" +"@types/minimist@^1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e" + integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag== + "@types/node-fetch@^2.6.2": version "2.6.2" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.2.tgz#d1a9c5fd049d9415dce61571557104dec3ec81da" @@ -463,6 +468,11 @@ mime-types@^2.1.12: dependencies: mime-db "1.52.0" +minimist@^1.2.8: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"