Merge pull request #60 from orbs-network/feature/env-var-config-params

Support passing node configuration via environment values or config file

src/cli-args.test.ts

@@ -4,33 +4,100 @@ import { parseArgs } from './cli-args';
 import _ from 'lodash';
 import { exampleConfig } from './config.example';
 
-test.serial.afterEach.always(() => {
+let env: NodeJS.ProcessEnv;
+
+test.before(() => {
+  env = process.env;
+});
+
+test.beforeEach(() => {
+  process.env = { ...env };
+});
+
+test.afterEach.always(() => {
+  process.env = env;
   mockFs.restore();
 });
 
-test.serial('parseArgs default config file does not exist', (t) => {
+test('parseArgs with no config and no environment variables', (t) => {
+  t.throws(() => parseArgs([]));
+});
+
+test('parseArgs with no file', (t) => {
+  t.throws(() => parseArgs(['--config']));
+});
+
+test('parseArgs with environment variables and no config', (t) => {
+  const mockMgmtSvcEndpoint = 'http://localhost:8080';
+  const mockEthereumEndpoint = 'https://mainnet.infura.io/v3/1234567890';
+  const mockSignerEndpoint = 'http://localhost:8081';
+  const mockEthElectionsContract = '0x1234567890';
+  const mockNodeAddress = '555550a3c12e86b4b5f39b213f7e19d048276dae';
+
+  process.env.MANAGEMENT_SERVICE_ENDPOINT = mockMgmtSvcEndpoint;
+  process.env.ETHEREUM_ENDPOINT = mockEthereumEndpoint;
+  process.env.SIGNER_ENDPOINT = mockSignerEndpoint;
+  process.env.ETHEREUM_ELECTIONS_CONTRACT = mockEthElectionsContract;
+  process.env.NODE_ADDRESS = mockNodeAddress;
+
+  const output = parseArgs([]);
+
+  t.notThrows(() => parseArgs([]));
+  t.assert((output.EthereumEndpoint = mockEthereumEndpoint));
+  t.assert((output.ManagementServiceEndpoint = mockMgmtSvcEndpoint));
+  t.assert((output.SignerEndpoint = mockSignerEndpoint));
+  t.assert((output.EthereumElectionsContract = mockEthElectionsContract));
+  t.assert((output.NodeOrbsAddress = mockNodeAddress));
+});
+
+test('parseArgs: errors when incomplete env vars set', (t) => {
+  const mockMgmtSvcEndpoint = 'http://localhost:8080';
+  const mockEthereumEndpoint = 'https://mainnet.infura.io/v3/1234567890';
+
+  process.env.MANAGEMENT_SERVICE_ENDPOINT = mockMgmtSvcEndpoint;
+  process.env.ETHEREUM_ENDPOINT = mockEthereumEndpoint;
+
   t.throws(() => parseArgs([]));
 });
 
-test.serial('parseArgs default config file valid', (t) => {
+test('parseArgs: environment variables override config file', (t) => {
+  const mockMgmtSvcEndpoint = 'http://localhost:8080';
+  const mockEthereumEndpoint = 'https://mainnet.infura.io/v3/1234567890';
+  const mockSignerEndpoint = 'http://localhost:8081';
+  const mockEthElectionsContract = '0x1234567890';
+  const mockNodeAddress = '555550a3c12e86b4b5f39b213f7e19d048276dae';
+
+  process.env.MANAGEMENT_SERVICE_ENDPOINT = mockMgmtSvcEndpoint;
+  process.env.ETHEREUM_ENDPOINT = mockEthereumEndpoint;
+  process.env.SIGNER_ENDPOINT = mockSignerEndpoint;
+  process.env.ETHEREUM_ELECTIONS_CONTRACT = mockEthElectionsContract;
+  process.env.NODE_ADDRESS = mockNodeAddress;
+
   mockFs({
-    ['./config.json']: JSON.stringify(exampleConfig),
+    ['./some/file.json']: JSON.stringify(exampleConfig),
   });
-  t.deepEqual(parseArgs([]), exampleConfig);
+
+  const output = parseArgs(['--config', './some/file.json']);
+
+  t.assert((output.EthereumEndpoint = mockEthereumEndpoint));
+  t.assert((output.ManagementServiceEndpoint = mockMgmtSvcEndpoint));
+  t.assert((output.SignerEndpoint = mockSignerEndpoint));
+  t.assert((output.EthereumElectionsContract = mockEthElectionsContract));
+  t.assert((output.NodeOrbsAddress = mockNodeAddress));
 });
 
-test.serial('parseArgs custom config file does not exist', (t) => {
+test('parseArgs custom config file does not exist', (t) => {
   t.throws(() => parseArgs(['--config', './some/file.json']));
 });
 
-test.serial('parseArgs custom config file valid', (t) => {
+test('parseArgs custom config file valid', (t) => {
   mockFs({
     ['./some/file.json']: JSON.stringify(exampleConfig),
   });
   t.deepEqual(parseArgs(['--config', './some/file.json']), exampleConfig);
 });
 
-test.serial('parseArgs two valid custom config files merged', (t) => {
+test('parseArgs two valid custom config files merged', (t) => {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const mergedConfig: any = _.cloneDeep(exampleConfig);
   mergedConfig.SomeField = 'some value';
@@ -41,14 +108,14 @@ test.serial('parseArgs two valid custom config files merged', (t) => {
   t.deepEqual(parseArgs(['--config', './first/file1.json', './second/file2.json']), mergedConfig);
 });
 
-test.serial('parseArgs custom config file invalid JSON format', (t) => {
+test('parseArgs custom config file invalid JSON format', (t) => {
   mockFs({
     ['./some/file.json']: JSON.stringify(exampleConfig) + '}}}',
   });
   t.throws(() => parseArgs(['--config', './some/file.json']));
 });
 
-test.serial('parseArgs custom config file missing ManagementServiceEndpoint', (t) => {
+test('parseArgs custom config file missing ManagementServiceEndpoint', (t) => {
   const partialConfig = _.cloneDeep(exampleConfig);
   delete partialConfig.ManagementServiceEndpoint;
   mockFs({

src/cli-args.ts

@@ -1,4 +1,5 @@
 import { Configuration, validateConfiguration, defaultConfiguration } from './config';
+import { setConfigEnvVars } from './env-var-args';
 import yargs from 'yargs';
 import { readFileSync } from 'fs';
 import * as Logger from './logger';
@@ -12,24 +13,27 @@ export function parseArgs(argv: string[]): Configuration {
       type: 'array',
       required: false,
       string: true,
-      default: ['./config.json'],
       description: 'list of config files',
     })
     .exitProcess(false)
     .parse();
 
   // read input config JSON files
+  // If config.json not provided, required config values must be passed via environment variables
   try {
     res = Object.assign(
       {},
       defaultConfiguration,
-      ...args.config.map((configPath) => JSON.parse(readFileSync(configPath).toString()))
+      ...(args.config ?? []).map((configPath) => JSON.parse(readFileSync(configPath).toString()))
     );
   } catch (err) {
     Logger.error(`Cannot parse input JSON config files: [${args.config}].`);
     throw err;
   }
 
+  // Support passing required config values via environment variables
+  setConfigEnvVars(res);
+
   // validate JSON config
   try {
     validateConfiguration(res);

src/env-var-args.test.ts

@@ -0,0 +1,85 @@
+import test from 'ava';
+import { exampleConfig } from './config.example';
+import { setConfigEnvVars } from './env-var-args';
+import { Configuration } from './config';
+
+test('setConfigEnvVars uses default values when no environment variables', (t) => {
+  const input = { ...exampleConfig };
+  setConfigEnvVars(input);
+  t.deepEqual(input, exampleConfig);
+});
+
+/**
+ * Converts mockEnv property names to Configuration names
+ * Eg. ETHEREUM_ENDPOINT -> EthereumEndpoint
+ * */
+const camelCaseToSnakeCase = (str: string): string => {
+  const result = str.replace(/([a-z])([A-Z])/g, '$1_$2');
+  return result.toUpperCase();
+};
+
+const mockEnv = {
+  MANAGEMENT_SERVICE_ENDPOINT: 'http://localhost:8080',
+  ETHEREUM_ENDPOINT: 'https://mainnet.infura.io/v3/1234567890',
+  SIGNER_ENDPOINT: 'http://localhost:8081',
+  ETHEREUM_ELECTIONS_CONTRACT: '0x1234567890',
+  NODE_ADDRESS: '555550a3c12e86b4b5f39b213f7e19d048276dae',
+  MANAGEMENT_SERVICE_ENDPOINT_SCHEMA: 'http',
+  STATUS_JSON_PATH: '/path/to/status.json',
+  RUN_LOOP_POLL_TIME_SECONDS: 60,
+  ETHEREUM_BALANCE_POLL_TIME_SECONDS: 120,
+  ETHEREUM_CAN_JOIN_COMMITTEE_POLL_TIME_SECONDS: 180,
+  INVALID_ETHEREUM_SYNC_SECONDS: 240,
+  REPUTATION_SAMPLE_SIZE: 20,
+  INVALID_REPUTATION_CHECK_THRESHOLD: 2,
+  INVALID_REPUTATION_THRESHOLD: 0.5,
+  ORBS_REPUTATIONS_CONTRACT: '0x987654321',
+  ETHEREUM_SYNC_REQUIREMENT_SECONDS: 300,
+  FAIL_TO_SYNC_VCS_TIMEOUT_SECONDS: 360,
+  ELECTIONS_REFRESH_WINDOW_SECONDS: 420,
+  INVALID_REPUTATION_GRACE_SECONDS: 480,
+  VOTE_UNREADY_VALIDITY_SECONDS: 540,
+  ELECTIONS_AUDIT_ONLY: false,
+  SUSPEND_VOTE_UNREADY: false,
+  ETHEREUM_DISCOUNT_GAS_PRICE_FACTOR: 0.8,
+  ETHEREUM_DISCOUNT_TX_TIMEOUT_SECONDS: 600,
+  ETHEREUM_NON_DISCOUNT_TX_TIMEOUT_SECONDS: 660,
+  ETHEREUM_MAX_GAS_PRICE: 1000000000,
+  ETHEREUM_MAX_COMMITTED_DAILY_TX: 10,
+};
+
+test('setConfigEnvVars uses environment variables when set', (t) => {
+  const input: Configuration = { ...exampleConfig };
+
+  // Need to cast to stop TS complaining about number/string mismatch
+  process.env = { ...process.env, ...mockEnv } as unknown as NodeJS.ProcessEnv;
+
+  setConfigEnvVars(input);
+
+  for (const key of Object.keys(exampleConfig)) {
+    // Env var is NODE_ADDRESS, but config is NodeOrbsAddress
+    if (key === 'NodeOrbsAddress') {
+      t.assert(
+        input[key as keyof Configuration] === mockEnv[camelCaseToSnakeCase('NodeAddress') as keyof typeof mockEnv]
+      );
+      continue;
+    }
+    t.assert(input[key as keyof Configuration] === mockEnv[camelCaseToSnakeCase(key) as keyof typeof mockEnv]);
+  }
+});
+
+test('boolean environment variables are parsed correctly', (t) => {
+  const input: Configuration = { ...exampleConfig };
+
+  const testCases = [
+    { description: 'No env var set, should use default', envVar: undefined, expected: input.SuspendVoteUnready },
+    { description: 'Env var set to `true`, should resolve to true', envVar: 'true', expected: true },
+    { description: 'Env var set to `false`, should resolve to false', envVar: 'false', expected: false },
+  ];
+
+  for (const testCase of testCases) {
+    process.env.SUSPEND_VOTE_UNREADY = testCase.envVar;
+    setConfigEnvVars(input);
+    t.assert(input.SuspendVoteUnready === testCase.expected, testCase.description);
+  }
+});

src/env-var-args.ts

@@ -0,0 +1,79 @@
+import { Configuration } from './config';
+
+/**
+ * Parse required and optional node configuration from environment variables
+ *
+ * Environment variables override default configuration values
+ *
+ * Validation handled later in `validateConfiguration`
+ * @param config - The node configuration to update
+ * */
+export function setConfigEnvVars(config: Configuration): void {
+  config.ManagementServiceEndpoint = process.env.MANAGEMENT_SERVICE_ENDPOINT ?? config.ManagementServiceEndpoint;
+  config.EthereumEndpoint = process.env.ETHEREUM_ENDPOINT ?? config.EthereumEndpoint;
+  config.SignerEndpoint = process.env.SIGNER_ENDPOINT ?? config.SignerEndpoint;
+  config.EthereumElectionsContract = process.env.ETHEREUM_ELECTIONS_CONTRACT ?? config.EthereumElectionsContract;
+  // TODO: Rename NodeOrbsAddress globally
+  config.NodeOrbsAddress = process.env.NODE_ADDRESS ?? config.NodeOrbsAddress;
+  config.ManagementServiceEndpointSchema =
+    process.env.MANAGEMENT_SERVICE_ENDPOINT_SCHEMA ?? config.ManagementServiceEndpointSchema;
+  config.StatusJsonPath = process.env.STATUS_JSON_PATH ?? config.StatusJsonPath;
+  config.RunLoopPollTimeSeconds = process.env.RUN_LOOP_POLL_TIME_SECONDS
+    ? Number(process.env.RUN_LOOP_POLL_TIME_SECONDS)
+    : config.RunLoopPollTimeSeconds;
+  config.EthereumBalancePollTimeSeconds = process.env.ETHEREUM_BALANCE_POLL_TIME_SECONDS
+    ? Number(process.env.ETHEREUM_BALANCE_POLL_TIME_SECONDS)
+    : config.EthereumBalancePollTimeSeconds;
+  config.EthereumCanJoinCommitteePollTimeSeconds = process.env.ETHEREUM_CAN_JOIN_COMMITTEE_POLL_TIME_SECONDS
+    ? Number(process.env.ETHEREUM_CAN_JOIN_COMMITTEE_POLL_TIME_SECONDS)
+    : config.EthereumCanJoinCommitteePollTimeSeconds;
+  config.InvalidEthereumSyncSeconds = process.env.INVALID_ETHEREUM_SYNC_SECONDS
+    ? Number(process.env.INVALID_ETHEREUM_SYNC_SECONDS)
+    : config.InvalidEthereumSyncSeconds;
+  config.ReputationSampleSize = process.env.REPUTATION_SAMPLE_SIZE
+    ? Number(process.env.REPUTATION_SAMPLE_SIZE)
+    : config.ReputationSampleSize;
+  config.InvalidReputationCheckThreshold = process.env.INVALID_REPUTATION_CHECK_THRESHOLD
+    ? Number(process.env.INVALID_REPUTATION_CHECK_THRESHOLD)
+    : config.InvalidReputationCheckThreshold;
+  config.InvalidReputationThreshold = process.env.INVALID_REPUTATION_THRESHOLD
+    ? Number(process.env.INVALID_REPUTATION_THRESHOLD)
+    : config.InvalidReputationThreshold;
+  config.OrbsReputationsContract = process.env.ORBS_REPUTATIONS_CONTRACT ?? config.OrbsReputationsContract;
+  config.EthereumSyncRequirementSeconds = process.env.ETHEREUM_SYNC_REQUIREMENT_SECONDS
+    ? Number(process.env.ETHEREUM_SYNC_REQUIREMENT_SECONDS)
+    : config.EthereumSyncRequirementSeconds;
+  config.FailToSyncVcsTimeoutSeconds = process.env.FAIL_TO_SYNC_VCS_TIMEOUT_SECONDS
+    ? Number(process.env.FAIL_TO_SYNC_VCS_TIMEOUT_SECONDS)
+    : config.FailToSyncVcsTimeoutSeconds;
+  config.ElectionsRefreshWindowSeconds = process.env.ELECTIONS_REFRESH_WINDOW_SECONDS
+    ? Number(process.env.ELECTIONS_REFRESH_WINDOW_SECONDS)
+    : config.ElectionsRefreshWindowSeconds;
+  config.InvalidReputationGraceSeconds = process.env.INVALID_REPUTATION_GRACE_SECONDS
+    ? Number(process.env.INVALID_REPUTATION_GRACE_SECONDS)
+    : config.InvalidReputationGraceSeconds;
+  config.VoteUnreadyValiditySeconds = process.env.VOTE_UNREADY_VALIDITY_SECONDS
+    ? Number(process.env.VOTE_UNREADY_VALIDITY_SECONDS)
+    : config.VoteUnreadyValiditySeconds;
+  config.ElectionsAuditOnly = process.env.ELECTIONS_AUDIT_ONLY
+    ? process.env.ELECTIONS_AUDIT_ONLY === 'true'
+    : config.ElectionsAuditOnly;
+  config.SuspendVoteUnready = process.env.SUSPEND_VOTE_UNREADY
+    ? process.env.SUSPEND_VOTE_UNREADY === 'true'
+    : config.SuspendVoteUnready;
+  config.EthereumDiscountGasPriceFactor = process.env.ETHEREUM_DISCOUNT_GAS_PRICE_FACTOR
+    ? Number(process.env.ETHEREUM_DISCOUNT_GAS_PRICE_FACTOR)
+    : config.EthereumDiscountGasPriceFactor;
+  config.EthereumDiscountTxTimeoutSeconds = process.env.ETHEREUM_DISCOUNT_TX_TIMEOUT_SECONDS
+    ? Number(process.env.ETHEREUM_DISCOUNT_TX_TIMEOUT_SECONDS)
+    : config.EthereumDiscountTxTimeoutSeconds;
+  config.EthereumNonDiscountTxTimeoutSeconds = process.env.ETHEREUM_NON_DISCOUNT_TX_TIMEOUT_SECONDS
+    ? Number(process.env.ETHEREUM_NON_DISCOUNT_TX_TIMEOUT_SECONDS)
+    : config.EthereumNonDiscountTxTimeoutSeconds;
+  config.EthereumMaxGasPrice = process.env.ETHEREUM_MAX_GAS_PRICE
+    ? Number(process.env.ETHEREUM_MAX_GAS_PRICE)
+    : config.EthereumMaxGasPrice;
+  config.EthereumMaxCommittedDailyTx = process.env.ETHEREUM_MAX_COMMITTED_DAILY_TX
+    ? Number(process.env.ETHEREUM_MAX_COMMITTED_DAILY_TX)
+    : config.EthereumMaxCommittedDailyTx;
+}

src/read/guardians-reputations.ts

@@ -118,7 +118,13 @@ async function fetchMnmgnmtSrvStatusForGuardian(ethAddress: string, config: Repu
 
     const guardian = state.ManagementCurrentTopology.find(guardian => guardian.EthAddress === ethAddress);
 
-    let managementServiceEndpoint = config.ManagementServiceEndpointSchema.replace(/{{GUARDIAN_IP}}/g, guardian?.Ip + "");
+    let managementServiceEndpoint: string;
+
+    if (process.env.MANAGEMENT_SERVICE_ENDPOINT_SCHEMA) {
+        managementServiceEndpoint = process.env.MANAGEMENT_SERVICE_ENDPOINT_SCHEMA;
+      } else {
+        managementServiceEndpoint = config.ManagementServiceEndpointSchema.replace(/{{GUARDIAN_IP}}/g, guardian?.Ip + '');
+      }
 
     try {