feat: implement master-images component handlers + event/command writer

packages/server-analysis-manager-kit/src/components/master-images/constants.ts

@@ -8,7 +8,7 @@
 import { QueueRouterRoutingType } from '@privateaim/server-kit';
 
 export enum MasterImagesEvent {
-    SYNCHRONISING = 'synchronizing',
+    SYNCHRONIZING = 'synchronizing',
     SYNCHRONIZED = 'synchronized',
     SYNCHRONIZATION_FAILED = 'synchronizationFailed',
 
@@ -23,6 +23,8 @@ export enum MasterImagesEvent {
 
 export enum MasterImagesCommand {
     SYNCHRONIZE = 'synchronize',
+    BUILD = 'build',
+    PUSH = 'push',
 }
 
 export const MasterImagesEventQueueRouterRouting = {

packages/server-analysis-manager-kit/src/components/master-images/types.ts

@@ -12,45 +12,86 @@ export type MasterImagesBasePayload = {
     error?: Error
 };
 
+export type MasterImagesSynchronizeCommandPayload = {
+    url: string,
+    branch: string
+};
+
 export type MasterImagesSynchronizeCommandContext = {
     command: `${MasterImagesCommand.SYNCHRONIZE}`,
-    data: MasterImagesBasePayload,
+    data: MasterImagesSynchronizeCommandPayload,
 };
 
-export type MaterImagesSynchronizedPayload = MasterImagesBasePayload & {
+export type MasterImagesBuildCommandPayload = {
+    images: Image[],
+    directory: string
+};
+export type MasterImagesBuildCommandContext = {
+    command: `${MasterImagesCommand.BUILD}`,
+    data: MasterImagesBuildCommandPayload,
+};
+
+//-----------------------------------------------------------------------
+
+export type MasterImagesPushCommandPayloadTag = { name: string, registryId: string };
+export type MasterImagesPushCommandPayload = {
+    tags: MasterImagesPushCommandPayloadTag[]
+};
+
+export type MasterImagesPushCommandContext = {
+    command: `${MasterImagesCommand.PUSH}`,
+    data: MasterImagesPushCommandPayload,
+};
+
+//-----------------------------------------------------------------------
+
+export type MaterImagesSynchronizedEventPayload = {
     images: Image[],
     groups: Group[]
 };
 export type MasterImagesSynchronizedEventContext = {
-    data: MaterImagesSynchronizedPayload,
+    data: MaterImagesSynchronizedEventPayload,
     event: `${MasterImagesEvent.SYNCHRONIZED}`;
 };
 export type MasterImagesSynchronizingEventContext = {
     data: MasterImagesBasePayload,
-    event: `${MasterImagesEvent.SYNCHRONISING}`;
+    event: `${MasterImagesEvent.SYNCHRONIZING}`;
 };
-export type MasterImagesSynchronizationFailedEventContext = {
+
+//-----------------------------------------------------------------------
+
+export type MasterImagesFailedEventContext = {
     data: MasterImagesBasePayload,
-    event: `${MasterImagesEvent.SYNCHRONIZATION_FAILED}`;
+    event: `${MasterImagesEvent.BUILD_FAILED}` |
+        `${MasterImagesEvent.SYNCHRONIZATION_FAILED}` |
+        `${MasterImagesEvent.PUSH_FAILED}`;
 };
 
+//-----------------------------------------------------------------------
+
+export type MasterImagesBuildEventPayload = MasterImagesBuildCommandPayload;
 export type MasterImagesBuildEventContext = {
-    data: MasterImagesBasePayload,
-    event: `${MasterImagesEvent.BUILD_FAILED}` |
-        `${MasterImagesEvent.BUILDING}` |
+    data: MasterImagesBuildEventPayload,
+    event: `${MasterImagesEvent.BUILDING}` |
         `${MasterImagesEvent.BUILT}`;
 };
 
+//-----------------------------------------------------------------------
+
 export type MasterImagesPushEventContext = {
-    data: MasterImagesBasePayload,
-    event: `${MasterImagesEvent.PUSH_FAILED}` |
-        `${MasterImagesEvent.PUSHING}` |
+    data: MasterImagesPushCommandPayload,
+    event: `${MasterImagesEvent.PUSHING}` |
         `${MasterImagesEvent.PUSHED}`;
 };
 
-export type MasterImagesCommandContext = MasterImagesSynchronizeCommandContext;
+//-----------------------------------------------------------------------
+
+export type MasterImagesCommandContext = MasterImagesSynchronizeCommandContext |
+MasterImagesBuildCommandContext |
+MasterImagesPushCommandContext;
+
 export type MasterImagesEventContext = MasterImagesSynchronizedEventContext |
 MasterImagesSynchronizingEventContext |
-MasterImagesSynchronizationFailedEventContext |
 MasterImagesBuildEventContext |
-MasterImagesPushEventContext;
+MasterImagesPushEventContext |
+MasterImagesFailedEventContext;

packages/server-analysis-manager/package.json

@@ -32,6 +32,7 @@
         "@privateaim/storage-kit": "^0.8.0",
         "amqp-extension": "^4.0.0-beta.3",
         "dockerode": "^4.0.2",
+        "docker-scan": "^1.1.0",
         "dotenv": "^16.4.5",
         "envix": "^1.3.0",
         "gunzip-maybe": "^1.4.2",
@@ -40,11 +41,13 @@
         "rapiq": "^0.9.0",
         "redis-extension": "^1.5.0",
         "singa": "^1.0.0",
+        "tar-fs": "^3.0.6",
         "tar-stream": "^3.1.6",
         "uuid": "^10.0.0"
     },
     "devDependencies": {
         "@types/dockerode": "^3.3.29",
+        "@types/tar-fs": "^2.0.4",
         "@types/tar-stream": "^2.2.2",
         "@types/uuid": "^10.0.0",
         "ts-node": "^10.9.2"

packages/server-analysis-manager/src/components/builder/commands/push/module.ts

@@ -11,7 +11,12 @@ import {
 import { BuilderCommand } from '@privateaim/server-analysis-manager-kit';
 import type { BuilderBuildPayload } from '@privateaim/server-analysis-manager-kit';
 import {
-    buildDockerAuthConfig, buildRemoteDockerImageURL, cleanupDockerImages, pushDockerImage, useCoreClient, useDocker,
+    buildDockerAuthConfigFromRegistry,
+    buildRemoteDockerImageURL,
+    cleanupDockerImages,
+    pushDockerImage,
+    useCoreClient,
+    useDocker,
 } from '../../../../core';
 import type { ComponentPayloadExtended } from '../../../type';
 import { extendPayload } from '../../../utils';
@@ -33,11 +38,7 @@ export async function executePushCommand(
 
     // -----------------------------------------------------------------------------------
 
-    const authConfig = buildDockerAuthConfig({
-        host: data.registry.host,
-        user: data.registry.account_name,
-        password: data.registry.account_secret,
-    });
+    const authConfig = buildDockerAuthConfigFromRegistry(data.registry);
 
     const client = useCoreClient();
     const { data: analysisNodes } = await client.analysisNode.getMany({

packages/server-analysis-manager/src/components/index.ts

@@ -7,5 +7,6 @@
 
 export * from './builder';
 export * from './core';
+export * from './master-images';
 export * from './type';
 export * from './utils';

packages/server-analysis-manager/src/components/master-images/commands/build/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './module';

packages/server-analysis-manager/src/components/master-images/commands/build/module.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import { REGISTRY_MASTER_IMAGE_PROJECT_NAME } from '@privateaim/core-kit';
+import path from 'node:path';
+import tar from 'tar-fs';
+import type {
+    MasterImagesBuildCommandPayload, MasterImagesPushCommandPayloadTag,
+} from '@privateaim/server-analysis-manager-kit';
+import {
+    buildRemoteDockerImageURL, useCoreClient, useDocker, waitForDockerActionStream,
+} from '../../../../core';
+import { writePushCommand } from '../../queue';
+
+export async function executeMasterImagesBuildCommand(
+    payload: MasterImagesBuildCommandPayload,
+) {
+    const coreClient = useCoreClient();
+    const docker = useDocker();
+    const promises : Promise<unknown>[] = [];
+
+    const { data: registries } = await coreClient.registry.getMany();
+
+    const tags : MasterImagesPushCommandPayloadTag[] = [];
+
+    for (let i = 0; i < payload.images.length; i++) {
+        const imagePath = path.join(payload.directory, payload.images[i].path);
+
+        for (let j = 0; j < registries.length; j++) {
+            const imageTag = buildRemoteDockerImageURL({
+                hostname: registries[j].host,
+                projectName: REGISTRY_MASTER_IMAGE_PROJECT_NAME,
+                repositoryName: payload.images[i].virtualPath,
+                tagOrDigest: 'latest',
+            });
+
+            tags.push({
+                name: imageTag,
+                registryId: registries[j].id,
+            });
+
+            const pack = tar.pack(imagePath);
+            const promise = docker
+                .buildImage(pack, {
+                    t: imageTag,
+                })
+                .then((stream) => waitForDockerActionStream(stream));
+
+            promises.push(promise);
+        }
+    }
+
+    await Promise.all(promises);
+
+    await writePushCommand({
+        tags,
+    });
+
+    return payload;
+}

packages/server-analysis-manager/src/components/master-images/commands/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './build';
+export * from './push';
+export * from './synchronize';

packages/server-analysis-manager/src/components/master-images/commands/push/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './module';