Merge pull request #52 from w3f/notifier

matrix notifier

.circleci/config.yml

@@ -25,7 +25,6 @@ jobs:
       - checkout
       - setup_remote_docker:
           docker_layer_caching: true
-          version: 20.10.7
       - run:
           command: |
             /scripts/build-image.sh web3f/polkadot-payouts .

charts/polkadot-k8s-payouts/Chart.yaml

@@ -1,5 +1,5 @@
 description: Polkadot K8s Payouts
 name: polkadot-k8s-payouts
-version: v1.2.14
-appVersion: v1.2.14
+version: v1.3.0
+appVersion: v1.3.0
 apiVersion: v2

charts/polkadot-k8s-payouts/templates/cronjob.yaml

@@ -39,6 +39,8 @@ spec:
               mountPath: {{ dir .Values.config.claim.claimerKeystore.passwordPath }} 
             - name: wallet
               mountPath: {{ dir .Values.config.claim.claimerKeystore.filePath }}   
+            - name: store
+              mountPath: /app/store   
           volumes:
           - name: config
             configMap:
@@ -54,4 +56,11 @@ spec:
               secretName: {{ .Release.Name }}-wallet
               items:
               - key: claimer_wallet_json
-                path: {{ base .Values.config.claim.claimerKeystore.filePath }}           
+                path: {{ base .Values.config.claim.claimerKeystore.filePath }}      
+          - name: store
+          {{ if and (ne .Values.config.environment "ci") (.Values.config.matrix.enabled) }} 
+            persistentVolumeClaim:
+              claimName: {{ .Release.Name }}-store
+          {{ else }}
+            emptyDir: {}     
+          {{ end }}             

charts/polkadot-k8s-payouts/templates/pvc.yaml

@@ -0,0 +1,12 @@
+{{ if eq .Values.config.matrix.enabled true }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Release.Name }}-store
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Mi 
+{{ end }}

charts/polkadot-k8s-payouts/values.yaml

@@ -16,6 +16,13 @@ config:
   #       apiToken: xxx
   #     network: kusama
   #     url: https://github.com/w3f/1k-validators-be/raw/master/helmfile.d/config/kusama/otv-backend-prod.yaml.gotmpl
+  matrix: #optional
+    enabled: false
+    baseUrl: "https://matrix.org"
+    password: "xxx"
+    userId: "@bot:matrix.org"
+    room: "!xxx:matrix.org"
+    notifyRestarts: false
   targets: [] #optional
   deepCheck: 
     enabled: false

config/main.sample.yaml

@@ -15,6 +15,13 @@ targetsFromGit: #optional
       apiToken: xxx
     network: kusama
     url: https://your.gitlab.domain/api/v4/projects/number/repository/files/accounts.yaml/raw?ref=main  
+matrix: #optional
+  enabled: false
+  baseUrl: "https://matrix.org"
+  password: "xxx"
+  userId: "@bot:matrix.org"
+  room: "!xxx:matrix.org"
+  notifyRestarts: false
 targets: #optional  
 - alias: validator-000
     validatorAddress: "<validator-000-stash-address>"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "polkadot-payouts",
-  "version": "1.2.14",
+  "version": "1.3.0",
   "description": "Automated transfers among accounts",
   "repository": "[email protected]:w3f/accountant.git",
   "author": "W3F Infrastructure Team <[email protected]>",
@@ -20,13 +20,16 @@
     "start": "NODE_OPTIONS='--max-old-space-size=4096' node ./dist/index.js start"
   },
   "dependencies": {
+    "@matrix-org/olm": "https://gitlab.matrix.org/matrix-org/olm/-/package_files/2572/download",
     "@polkadot/api": "^12.3.1",
     "@w3f/config": "^0.1.1",
     "@w3f/logger": "^0.4.3",
     "async-wait-until": "^1.2.6",
     "bn.js": "^5.1.3",
     "commander": "^4.1.1",
+    "matrix-js-sdk": "^25",
     "node-fetch": "^2.6.6",
+    "node-localstorage": "^2",
     "yaml": "^2.2.1"
   },
   "devDependencies": {

src/actions/start.ts

@@ -6,6 +6,7 @@ import { LoggerSingleton } from '../logger';
 import { Claimer } from '../claimer';
 import { GitConfigLoaderFactory } from '../gitConfigLoader/gitConfigLoaderFactory';
 import { runAttempts } from '../constants';
+import { NotifierFactory } from '../notifier/NotifierFactory';
 
 const _loadConfig = async (config: any): Promise<InputConfig> =>{
     const cfg = new Config<InputConfig>().parse(config);
@@ -36,7 +37,9 @@ export async function startAction(cmd): Promise<void> {
 
     const api = await new Client(cfg).connect()
 
-    const claimer = new Claimer(cfg, api);
+    const notifier = await new NotifierFactory(cfg).makeNotifier()
+
+    const claimer = new Claimer(cfg, api, notifier);
 
     try {
         let leftAttemts = runAttempts

src/claimer.ts

@@ -1,4 +1,4 @@
-import { ClaimerInputConfig, Target, GracePeriod, ValidatorInfo, ValidatorsMap, ClaimPool } from './types';
+import { ClaimerInputConfig, Target, GracePeriod, ValidatorInfo, ValidatorsMap, ClaimPool, NewPayoutData } from './types';
 import { getActiveEraIndex, initKey, setDifference } from './utils';
 import { Logger, LoggerSingleton } from './logger';
 import { ApiPromise, Keyring } from '@polkadot/api';
@@ -7,6 +7,7 @@ import waitUntil from 'async-wait-until';
 import { BN } from 'bn.js';
 import { batchSize, claimAttempts, gracePeriod, isDeepCheckEnabled } from './constants';
 import { DeriveOwnExposure } from '@polkadot/api-derive/types';
+import { Notifier } from './notifier/INotifier';
 
 export class Claimer {
     private isDeepCheckEnabled = isDeepCheckEnabled
@@ -16,12 +17,14 @@ export class Claimer {
     private readonly logger: Logger = LoggerSingleton.getInstance()
     private currentEraIndex: number;
     private lastRewardMax: number;
+    private claimerAddress: string;
 
     private isFullSuccess = true;
 
     constructor(
         private readonly cfg: ClaimerInputConfig,
-        private readonly api: ApiPromise) {
+        private readonly api: ApiPromise,
+        private readonly notifier: Notifier) {
         cfg.targets.forEach(target=>this.targets.add(target))
         this.isDeepCheckEnabled = cfg.deepCheck.enabled
         this.gracePeriod = cfg.claim.gracePeriod
@@ -46,7 +49,8 @@ export class Claimer {
         if(this.cfg.claim.enabled) {
           this.logger.info(`Processing claims...`)
           const keyPair: KeyringPair = initKey(this.cfg.claim.claimerKeystore.filePath,this.cfg.claim.claimerKeystore.passwordPath);
-
+          this.claimerAddress = keyPair.address
+
           const claimPool = await this.buildClaimPool(validatorsMap)
           await this.claim(keyPair,claimPool,validatorsMap)
         }
@@ -58,6 +62,7 @@ export class Claimer {
           this.logger.info(`${validatorInfo.alias}|${address}`)
           validatorInfo.unclaimedPayouts.length>0 ? this.logger.info(`To be claimed Payouts: ${validatorInfo.unclaimedPayouts.toString()}`) : {}
           validatorInfo.claimedPayouts.length>0 ? this.logger.info(`Claimed Payouts: ${validatorInfo.claimedPayouts.toString()}`) : {}
+          validatorInfo.claimedPayouts.length>0 ? await this._notifyNewPayout(address,validatorInfo) : {}
           this.logger.info(`**********`)
         }
 
@@ -270,4 +275,17 @@ export class Claimer {
       this.logger.info(`Claimed ${totClaimed} payouts`);
     }
 
+    private _notifyNewPayout = async (address: string, info: ValidatorInfo): Promise<boolean> => {
+      this.logger.debug(`Delegating to the Notifier the New Payout notification...`)
+      const data: NewPayoutData = {
+        alias: info.alias,
+        address: address,
+        claimer: this.claimerAddress,
+        eras: info.claimedPayouts.toString(),
+        networkId: address.startsWith('1') ? 'polkadot' : 'kusama'
+      }
+      this.logger.debug(JSON.stringify(data))
+      return await this.notifier.newPayout(data)
+    }
+
 }

src/constants.ts

@@ -1,5 +1,6 @@
-export const batchSize = 9
+export const batchSize = 2
 export const isDeepCheckEnabled = false
 export const gracePeriod = {enabled: false, eras: 0}
 export const runAttempts = 3
-export const claimAttempts = 3
+export const claimAttempts = 3
+export const storeDir = "./store"

src/notifier/INotifier.ts

@@ -0,0 +1,5 @@
+import { NewPayoutData } from "../types";
+
+export interface Notifier {
+  newPayout(data: NewPayoutData): Promise<boolean>;
+}

src/notifier/NotifierFactory.ts

@@ -0,0 +1,17 @@
+import { InputConfig } from "../types";
+import { Notifier } from "./INotifier";
+import { Disabled } from "./disabled";
+import { Matrix } from "./matrix";
+
+export class NotifierFactory {
+  constructor(private readonly cfg: InputConfig){}
+  makeNotifier = async (): Promise<Notifier> => {
+
+    if(!this.cfg.matrix?.enabled)
+      return new Disabled()
+
+    const matrix = new Matrix(this.cfg)
+    await matrix.start()
+    return matrix  
+  }
+}

src/notifier/disabled.ts

@@ -0,0 +1,15 @@
+import { Logger, LoggerSingleton } from '../logger';
+
+import {
+    NewPayoutData,
+} from '../types';
+import { Notifier } from './INotifier';
+
+
+export class Disabled implements Notifier {
+    private readonly logger: Logger = LoggerSingleton.getInstance()
+    newPayout = async (_data: NewPayoutData): Promise<boolean> =>{
+        this.logger.info("Notifier disabled...")
+        return true
+    }
+}

src/notifier/matrix-crypto.ts

@@ -0,0 +1,49 @@
+/**
+ * This file is responsible for setting up the globals correctly before
+ * importing matrix and then exporting it.
+ */
+
+/**
+ * We must import olm and assign it to the global before importing matrix.
+ */
+import olm from "@matrix-org/olm";
+
+global.Olm = olm;
+
+/**
+ * We must also override the default fetch global to use the FS module when
+ * attempting to fetch the wasm since the default fetch does not support local
+ * files.
+ */
+import fs from "fs/promises";
+
+const oldFetch = fetch;
+
+global.fetch = async (input: RequestInfo | URL | string, init?: RequestInit): Promise<Response> => {
+	// Here we need to check if it is attempting to fetch the wasm file.
+	if (typeof input == "string" && input.charAt(0) === "/") {
+		const data = await fs.readFile(input);
+
+		// Return the wasm data as a typical response.
+		return new Response(data, {
+			headers: { "content-type": "application/wasm" }
+		});
+	}
+
+	// Since this is not fetching the wasm we can just use the old implementation.
+	return await oldFetch(input, init);
+};
+
+/**
+ * We will increase the logger severity to reduce clutter.
+ */
+// import { logger } from "../../../lib/logger.js";
+
+// logger.setLevel(5);
+
+/**
+ * Now we can import and export the matrix sdk.
+ */
+import * as sdk from "matrix-js-sdk";
+
+export default sdk;