Merge branch 'master' into bigquery-sender

.github/workflows/ci.yml

@@ -7,11 +7,35 @@ on:
     branches: [ master ]
 
 jobs:
+  console:
+    name: ci-console
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./hermes-console
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Run linter
+        run: yarn && yarn lint
+      - name: Run frontend tests
+        run: yarn test:unit
+      - name: Upload artifact
+        if: always() && github.ref == 'refs/heads/master'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-console
+          path: ./hermes-console/allure-results
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         tasks: [
+          # Add/remove task in Allure Report job also
           {alias: "unitTests", name: "check"},
           {alias: "integrationTests", name: "integrationTest"},
           {alias: "slowIntegrationTests", name: "slowIntegrationTest"},
@@ -47,3 +71,61 @@ jobs:
         with:
           paths: '**/build/test-results/**/TEST-*.xml'
           show: fail, skip
+      - name: Upload artifact
+        if: always() && github.ref == 'refs/heads/master' && matrix.tasks.alias != 'benchmark'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.tasks.alias }}
+          path: build/allure-results
+
+  allureReport:
+    if: always() && github.ref == 'refs/heads/master'
+    name: Generate Allure Report
+    needs: [ build, console ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download artifact unitTests
+        uses: actions/download-artifact@v4
+        if: always()
+        with:
+          name: unitTests
+          path: allure-results
+      - name: Download artifact integrationTests
+        uses: actions/download-artifact@v4
+        if: always()
+        with:
+          name: integrationTests
+          path: allure-results
+      - name: Download artifact slowIntegrationTests
+        uses: actions/download-artifact@v4
+        if: always()
+        with:
+          name: slowIntegrationTests
+          path: allure-results
+      - name: Download artifact ci-console
+        uses: actions/download-artifact@v4
+        if: always()
+        with:
+          name: ci-console
+          path: allure-results
+      - name: Load test report history
+        uses: actions/checkout@v4
+        if: always()
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - name: Build test report
+        uses: simple-elf/[email protected]
+        if: always()
+        with:
+          gh_pages: gh-pages
+          allure_history: allure-history
+          allure_results: allure-results
+      - name: Publish test report
+        uses: peaceiris/actions-gh-pages@v4
+        if: always()
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: allure-history

build.gradle

@@ -69,7 +69,8 @@ allprojects {
             junit_jupiter     : '5.10.2',
             testcontainers    : '1.19.8',
             spring            : '3.2.4',
-            assertj           : '3.25.3'
+            assertj           : '3.25.3',
+            allure            : '2.24.0'
     ]
 
     repositories {
@@ -80,6 +81,15 @@ allprojects {
         implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.4'
         implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
 
+        // Allure Spock adapter
+        testImplementation(platform("io.qameta.allure:allure-bom:${versions.allure}"))
+        testImplementation("io.qameta.allure:allure-spock2")
+        testImplementation("io.qameta.allure:allure-junit-platform")
+
+        // Spock framework
+        testImplementation(platform("org.spockframework:spock-bom:${versions.spock}"))
+        testImplementation("org.spockframework:spock-core")
+
         testImplementation group: 'junit', name: 'junit', version: '4.11'
         testImplementation group: 'com.tngtech.java', name: 'junit-dataprovider', version: '1.10.0'
         testImplementation group: 'pl.pragmatists', name: 'JUnitParams', version: '1.0.2'

docker/latest/management/management.yaml

@@ -54,6 +54,10 @@ schema.repository:
   serverUrl: http://schema-registry:8081
   validationEnabled: true
 
+prometheus:
+  client:
+    enabled: true
+
 console:
   configurationLocation: console/config-local.json
   configurationType: classpath_resource

hermes-api/build.gradle

@@ -18,7 +18,6 @@ dependencies {
     implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '4.0.5'
     implementation group: 'jakarta.annotation', name: 'jakarta.annotation-api', version: '3.0.0'
 
-
     testImplementation group: 'org.spockframework', name: 'spock-core', version: versions.spock
     testImplementation group: 'org.spockframework', name: 'spock-junit4', version: versions.spock
     testImplementation project(':hermes-test-helper')

hermes-api/src/test/resources/allure.properties

@@ -0,0 +1 @@
+allure.results.directory=../build/allure-results

hermes-client/src/test/resources/allure.properties

@@ -0,0 +1 @@
+allure.results.directory=../build/allure-results

hermes-common/src/test/resources/allure.properties

@@ -0,0 +1 @@
+allure.results.directory=../build/allure-results

hermes-console/.gitignore

@@ -8,6 +8,7 @@ pnpm-debug.log*
 lerna-debug.log*
 
 node_modules
+allure-results
 .DS_Store
 dist
 dist-ssr

hermes-console/json-server/db.json

@@ -91,6 +91,32 @@
       ]
     }
   ],
+  "inconsistentGroups3":[
+    {
+      "name": "pl.allegro.public.group",
+      "inconsistentMetadata": [],
+      "inconsistentTopics": [
+        {
+          "name": "pl.allegro.public.group.DummyEvent",
+          "inconsistentMetadata": [],
+          "inconsistentSubscriptions": [
+            {
+              "name": "pl.allegro.public.group.DummyEvent$foobar-service",
+              "inconsistentMetadata": [
+                {
+                  "datacenter": "DC1"
+                },
+                {
+                  "datacenter": "DC2",
+                  "content": "{\n  \"id\": \"foobar-service\",\n  \"topicName\": \"pl.allegro.public.group.DummyEvent\",\n  \"name\": \"foobar-service\",\n  \"endpoint\": \"service://foobar-service/events/dummy-event\",\n  \"state\": \"ACTIVE\",\n  \"description\": \"Test Hermes endpoint\",\n  \"subscriptionPolicy\": {\n    \"rate\": 10,\n    \"messageTtl\": 60,\n    \"messageBackoff\": 100,\n    \"requestTimeout\": 1000,\n    \"socketTimeout\": 0,\n    \"sendingDelay\": 0,\n    \"backoffMultiplier\": 1.0,\n    \"backoffMaxIntervalInSec\": 600,\n    \"retryClientErrors\": true,\n    \"backoffMaxIntervalMillis\": 600000\n  },\n  \"trackingEnabled\": false,\n  \"trackingMode\": \"trackingOff\",\n  \"owner\": {\n    \"source\": \"Service Catalog\",\n    \"id\": \"42\"\n  },\n  \"monitoringDetails\": {\n    \"severity\": \"NON_IMPORTANT\",\n    \"reaction\": \"\"\n  },\n  \"contentType\": \"JSON\",\n  \"deliveryType\": \"SERIAL\",\n  \"filters\": [\n    {\n      \"type\": \"avropath\",\n      \"path\": \"foobar\",\n      \"matcher\": \"^FOO_BAR$|^BAZ_BAR$\",\n      \"matchingStrategy\": \"any\"\n    },\n    {\n      \"type\": \"avropath\",\n      \"path\": \".foo.bar.baz\",\n      \"matcher\": \"true\",\n      \"matchingStrategy\": \"all\"\n    }\n  ],\n  \"mode\": \"ANYCAST\",\n  \"headers\": [\n    {\n      \"name\": \"X-My-Header\",\n      \"value\": \"boobar\"\n    },\n    {\n      \"name\": \"X-Another-Header\",\n      \"value\": \"foobar\"\n    }\n  ],\n  \"endpointAddressResolverMetadata\": {\n    \"additionalMetadata\": false,\n    \"nonSupportedProperty\": 2\n  },\n  \"http2Enabled\": false,\n  \"subscriptionIdentityHeadersEnabled\": false,\n  \"autoDeleteWithTopicEnabled\": false,\n  \"createdAt\": 1579507131.238,\n  \"modifiedAt\": 1672140855.813\n}"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ],
   "topicNames": [
     "pl.allegro.public.offer.product.ProductEventV1",
     "pl.allegro.public.offer.product.ProductEventV2",

hermes-console/json-server/routes.json

@@ -3,6 +3,7 @@
   "/consistency/groups": "/consistencyGroups",
   "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.offer*": "/inconsistentGroups",
   "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.group2*": "/inconsistentGroups2",
+  "/consistency/inconsistencies/groups?groupNames=pl.allegro.public.group": "/inconsistentGroups3",
   "/groups": "/groups",
   "/owners/sources/Service%20Catalog/:id": "/topicsOwners/:id",
   "/readiness/datacenters": "/readinessDatacenters",

hermes-console/json-server/server.ts

@@ -87,6 +87,22 @@ server.put(
   },
 );
 
+server.post(
+  '/consistency/sync/topics/pl.allegro.public.group.DummyEvent/subscriptions/barbaz-service*',
+  (req, res) => {
+    res.sendStatus(200);
+  },
+);
+
+server.post(
+  '/consistency/sync/topics/pl.allegro.public.group.DummyEvent*',
+  (req, res) => {
+    res.status(404).jsonp({
+      message: 'Group pl.allegro.public.group not found',
+    });
+  },
+);
+
 server.post('/filters/:topic', (req, res) => {
   res.jsonp(filterDebug);
 });

hermes-console/package.json

@@ -46,6 +46,8 @@
     "@vue/eslint-config-typescript": "13.0.0",
     "@vue/test-utils": "2.4.5",
     "@vue/tsconfig": "0.1.3",
+    "allure-commandline": "2.30.0",
+    "allure-vitest": "2.15.1",
     "eslint": "8.57.0",
     "eslint-plugin-sort-imports-es6-autofix": "0.6.0",
     "eslint-plugin-vue": "9.25.0",
@@ -57,7 +59,6 @@
     "sass": "1.75.0",
     "typescript": "4.9.5",
     "vite": "5.2.9",
-    "vite-plugin-rewrite-all": "1.0.1",
     "vite-plugin-vuetify": "2.0.3",
     "vitest": "1.5.0",
     "vue-tsc": "2.0.13"

hermes-console/src/api/hermes-client/index.ts

@@ -464,3 +464,45 @@ export function verifyFilters(
     },
   );
 }
+
+export function syncGroup(
+  groupName: string,
+  primaryDatacenter: string,
+): ResponsePromise<void> {
+  return axios.post<void>(`/consistency/sync/groups/${groupName}`, null, {
+    params: {
+      primaryDatacenter: primaryDatacenter,
+    },
+  });
+}
+
+export function syncTopic(
+  topicQualifiedName: string,
+  primaryDatacenter: string,
+): ResponsePromise<void> {
+  return axios.post<void>(
+    `/consistency/sync/topics/${topicQualifiedName}`,
+    null,
+    {
+      params: {
+        primaryDatacenter: primaryDatacenter,
+      },
+    },
+  );
+}
+
+export function syncSubscription(
+  topicQualifiedName: string,
+  subscriptionName: string,
+  primaryDatacenter: string,
+): ResponsePromise<void> {
+  return axios.post<void>(
+    `/consistency/sync/topics/${topicQualifiedName}/subscriptions/${subscriptionName}`,
+    null,
+    {
+      params: {
+        primaryDatacenter: primaryDatacenter,
+      },
+    },
+  );
+}

hermes-console/src/api/subscription.ts

@@ -128,6 +128,7 @@ export interface SerialSubscriptionPolicyJson {
   backoffMultiplier: number;
   messageBackoff: number;
   messageTtl: number;
+  inflightSize: number;
   rate: number;
   requestTimeout: number;
   sendingDelay: number;

hermes-console/src/composables/subscription/use-create-subscription/useCreateSubscription.ts

@@ -126,6 +126,7 @@ function initializeForm(
       inflightMessageTTL:
         loadedConfig.value.subscription.defaults.subscriptionPolicy
           .messageTtl || 3600,
+      inflightMessagesCount: null,
       retryBackoff: 1000,
       retryBackoffMultiplier: 1.0,
       backoffMaxIntervalInSec: 600,

hermes-console/src/composables/subscription/use-form-subscription/form-mapper.ts

@@ -76,6 +76,9 @@ function mapSerialSubscriptionPolicy(
     ),
     messageBackoff: parseFloat(String(form.subscriptionPolicy.retryBackoff)),
     messageTtl: parseFloat(String(form.subscriptionPolicy.inflightMessageTTL)),
+    inflightSize: parseFloat(
+      String(form.subscriptionPolicy.inflightMessagesCount),
+    ),
     rate: parseFloat(String(form.subscriptionPolicy.rateLimit!!)),
     requestTimeout: parseFloat(String(form.subscriptionPolicy.requestTimeout)),
     sendingDelay: parseFloat(String(form.subscriptionPolicy.sendingDelay)),

hermes-console/src/composables/subscription/use-form-subscription/types.ts

@@ -38,6 +38,7 @@ export interface SubscriptionForm {
 export interface FormSubscriptionPolicy {
   rateLimit: number | null;
   inflightMessageTTL: number;
+  inflightMessagesCount: number | null;
   retryBackoff: number;
   sendingDelay: number;
   retryBackoffMultiplier: number;
@@ -69,6 +70,7 @@ export interface FormValidators {
   requestTimeout: FieldValidator<number>[];
   sendingDelay: FieldValidator<number>[];
   inflightMessageTTL: FieldValidator<number>[];
+  inflightMessagesCount: FieldValidator<number>[];
   retryBackoff: FieldValidator<number>[];
   retryBackoffMultiplier: FieldValidator<number>[];
   backoffMaxIntervalInSec: FieldValidator<number>[];

hermes-console/src/composables/subscription/use-form-subscription/useFormSubscription.ts

@@ -68,6 +68,7 @@ function formValidators(form: Ref<SubscriptionForm>): FormValidators {
     ],
     sendingDelay: [required(), min(0), max(5000)],
     inflightMessageTTL: [required(), min(0), max(7200)],
+    inflightMessagesCount: [min(1)],
     retryBackoff: [required(), min(0), max(1000000)],
     retryBackoffMultiplier: [required(), min(1), max(10)],
     backoffMaxIntervalInSec: [required(), min(1), max(600)],
@@ -178,6 +179,7 @@ function createEmptyForm(): Ref<SubscriptionForm> {
     subscriptionPolicy: {
       rateLimit: null,
       inflightMessageTTL: 3600,
+      inflightMessagesCount: null,
       retryBackoff: 1000,
       retryBackoffMultiplier: 1.0,
       backoffMaxIntervalInSec: 600,
@@ -222,6 +224,7 @@ export function initializeFullyFilledForm(
           ? subscription.subscriptionPolicy.rate
           : null,
       inflightMessageTTL: subscription.subscriptionPolicy.messageTtl,
+      inflightMessagesCount: subscription.subscriptionPolicy.inflightSize,
       retryBackoff: subscription.subscriptionPolicy.messageBackoff,
       retryBackoffMultiplier:
         subscription.deliveryType === DeliveryType.SERIAL