diff --git a/package-lock.json b/package-lock.json index d68d75e34..93f0d9c93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7134,6 +7134,16 @@ "resolved": "https://registry.npmjs.org/@types/swagger-ui-dist/-/swagger-ui-dist-3.30.5.tgz", "integrity": "sha512-SrXhD9L8qeIxJzN+o1kmf3wXeVf/+Km3jIdRM1+Yq3I5b/dlF5TcGr5WCVM7I/cBYpgf43/gCPIucQ13AhICiw==" }, + "node_modules/@types/tar-fs": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/tar-fs/-/tar-fs-2.0.4.tgz", + "integrity": "sha512-ipPec0CjTmVDWE+QKr9cTmIIoTl7dFG/yARCM5MqK8i6CNLIG1P8x4kwDsOQY1ChZOZjH0wO9nvfgBvWl4R3kA==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/tar-stream": "*" + } + }, "node_modules/@types/tar-stream": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@types/tar-stream/-/tar-stream-2.2.3.tgz", @@ -9163,7 +9173,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", - "dev": true, "optional": true, "dependencies": { "bare-events": "^2.0.0", @@ -9175,14 +9184,12 @@ "version": "2.4.0", "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.0.tgz", "integrity": "sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg==", - "dev": true, "optional": true }, "node_modules/bare-path": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", - "dev": true, "optional": true, "dependencies": { "bare-os": "^2.1.0" @@ -9192,7 +9199,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.3.tgz", "integrity": "sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ==", - "dev": true, "optional": true, "dependencies": { "streamx": "^2.18.0" @@ -26324,6 +26330,7 @@ "@privateaim/server-kit": "^0.8.0", "@privateaim/storage-kit": "^0.8.0", "amqp-extension": "^4.0.0-beta.3", + "docker-scan": "^1.1.0", "dockerode": "^4.0.2", "dotenv": "^16.4.5", "envix": "^1.3.0", @@ -26333,11 +26340,13 @@ "redis-extension": "^1.5.0", "routup": "^4.0.0", "singa": "^1.0.0", + "tar-fs": "^3.0.6", "tar-stream": "^3.1.6", "uuid": "^10.0.0" }, "devDependencies": { "@types/dockerode": "^3.3.29", + "@types/tar-fs": "^2.0.4", "@types/tar-stream": "^2.2.2", "@types/uuid": "^10.0.0", "ts-node": "^10.9.2" @@ -26356,6 +26365,28 @@ "docker-scan": "^1.1.0" } }, + "packages/server-analysis-manager/node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "packages/server-analysis-manager/node_modules/tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, "packages/server-analysis-manager/node_modules/tar-stream": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", diff --git a/packages/server-analysis-manager-kit/src/components/master-images/constants.ts b/packages/server-analysis-manager-kit/src/components/master-images/constants.ts index 5583f1f7d..f1ea0efa4 100644 --- a/packages/server-analysis-manager-kit/src/components/master-images/constants.ts +++ b/packages/server-analysis-manager-kit/src/components/master-images/constants.ts @@ -8,7 +8,7 @@ import { QueueRouterRoutingType } from '@privateaim/server-kit'; export enum MasterImagesEvent { - SYNCHRONISING = 'synchronizing', + SYNCHRONIZING = 'synchronizing', SYNCHRONIZED = 'synchronized', SYNCHRONIZATION_FAILED = 'synchronizationFailed', @@ -23,6 +23,8 @@ export enum MasterImagesEvent { export enum MasterImagesCommand { SYNCHRONIZE = 'synchronize', + BUILD = 'build', + PUSH = 'push', } export const MasterImagesEventQueueRouterRouting = { diff --git a/packages/server-analysis-manager-kit/src/components/master-images/types.ts b/packages/server-analysis-manager-kit/src/components/master-images/types.ts index 5e08f36d7..d96de0451 100644 --- a/packages/server-analysis-manager-kit/src/components/master-images/types.ts +++ b/packages/server-analysis-manager-kit/src/components/master-images/types.ts @@ -12,45 +12,86 @@ export type MasterImagesBasePayload = { error?: Error }; +export type MasterImagesSynchronizeCommandPayload = { + url: string, + branch: string +}; + export type MasterImagesSynchronizeCommandContext = { command: `${MasterImagesCommand.SYNCHRONIZE}`, - data: MasterImagesBasePayload, + data: MasterImagesSynchronizeCommandPayload, }; -export type MaterImagesSynchronizedPayload = MasterImagesBasePayload & { +export type MasterImagesBuildCommandPayload = { + images: Image[], + directory: string +}; +export type MasterImagesBuildCommandContext = { + command: `${MasterImagesCommand.BUILD}`, + data: MasterImagesBuildCommandPayload, +}; + +//----------------------------------------------------------------------- + +export type MasterImagesPushCommandPayloadTag = { name: string, registryId: string }; +export type MasterImagesPushCommandPayload = { + tags: MasterImagesPushCommandPayloadTag[] +}; + +export type MasterImagesPushCommandContext = { + command: `${MasterImagesCommand.PUSH}`, + data: MasterImagesPushCommandPayload, +}; + +//----------------------------------------------------------------------- + +export type MaterImagesSynchronizedEventPayload = { images: Image[], groups: Group[] }; export type MasterImagesSynchronizedEventContext = { - data: MaterImagesSynchronizedPayload, + data: MaterImagesSynchronizedEventPayload, event: `${MasterImagesEvent.SYNCHRONIZED}`; }; export type MasterImagesSynchronizingEventContext = { data: MasterImagesBasePayload, - event: `${MasterImagesEvent.SYNCHRONISING}`; + event: `${MasterImagesEvent.SYNCHRONIZING}`; }; -export type MasterImagesSynchronizationFailedEventContext = { + +//----------------------------------------------------------------------- + +export type MasterImagesFailedEventContext = { data: MasterImagesBasePayload, - event: `${MasterImagesEvent.SYNCHRONIZATION_FAILED}`; + event: `${MasterImagesEvent.BUILD_FAILED}` | + `${MasterImagesEvent.SYNCHRONIZATION_FAILED}` | + `${MasterImagesEvent.PUSH_FAILED}`; }; +//----------------------------------------------------------------------- + +export type MasterImagesBuildEventPayload = MasterImagesBuildCommandPayload; export type MasterImagesBuildEventContext = { - data: MasterImagesBasePayload, - event: `${MasterImagesEvent.BUILD_FAILED}` | - `${MasterImagesEvent.BUILDING}` | + data: MasterImagesBuildEventPayload, + event: `${MasterImagesEvent.BUILDING}` | `${MasterImagesEvent.BUILT}`; }; +//----------------------------------------------------------------------- + export type MasterImagesPushEventContext = { - data: MasterImagesBasePayload, - event: `${MasterImagesEvent.PUSH_FAILED}` | - `${MasterImagesEvent.PUSHING}` | + data: MasterImagesPushCommandPayload, + event: `${MasterImagesEvent.PUSHING}` | `${MasterImagesEvent.PUSHED}`; }; -export type MasterImagesCommandContext = MasterImagesSynchronizeCommandContext; +//----------------------------------------------------------------------- + +export type MasterImagesCommandContext = MasterImagesSynchronizeCommandContext | +MasterImagesBuildCommandContext | +MasterImagesPushCommandContext; + export type MasterImagesEventContext = MasterImagesSynchronizedEventContext | MasterImagesSynchronizingEventContext | -MasterImagesSynchronizationFailedEventContext | MasterImagesBuildEventContext | -MasterImagesPushEventContext; +MasterImagesPushEventContext | +MasterImagesFailedEventContext; diff --git a/packages/server-analysis-manager/package.json b/packages/server-analysis-manager/package.json index 686aead17..ac498bf94 100644 --- a/packages/server-analysis-manager/package.json +++ b/packages/server-analysis-manager/package.json @@ -32,6 +32,7 @@ "@privateaim/storage-kit": "^0.8.0", "amqp-extension": "^4.0.0-beta.3", "dockerode": "^4.0.2", + "docker-scan": "^1.1.0", "dotenv": "^16.4.5", "envix": "^1.3.0", "gunzip-maybe": "^1.4.2", @@ -40,11 +41,13 @@ "rapiq": "^0.9.0", "redis-extension": "^1.5.0", "singa": "^1.0.0", + "tar-fs": "^3.0.6", "tar-stream": "^3.1.6", "uuid": "^10.0.0" }, "devDependencies": { "@types/dockerode": "^3.3.29", + "@types/tar-fs": "^2.0.4", "@types/tar-stream": "^2.2.2", "@types/uuid": "^10.0.0", "ts-node": "^10.9.2" diff --git a/packages/server-analysis-manager/src/components/builder/commands/push/module.ts b/packages/server-analysis-manager/src/components/builder/commands/push/module.ts index f9a0aa6a8..e303c76d5 100644 --- a/packages/server-analysis-manager/src/components/builder/commands/push/module.ts +++ b/packages/server-analysis-manager/src/components/builder/commands/push/module.ts @@ -11,7 +11,12 @@ import { import { BuilderCommand } from '@privateaim/server-analysis-manager-kit'; import type { BuilderBuildPayload } from '@privateaim/server-analysis-manager-kit'; import { - buildDockerAuthConfig, buildRemoteDockerImageURL, cleanupDockerImages, pushDockerImage, useCoreClient, useDocker, + buildDockerAuthConfigFromRegistry, + buildRemoteDockerImageURL, + cleanupDockerImages, + pushDockerImage, + useCoreClient, + useDocker, } from '../../../../core'; import type { ComponentPayloadExtended } from '../../../type'; import { extendPayload } from '../../../utils'; @@ -33,11 +38,7 @@ export async function executePushCommand( // ----------------------------------------------------------------------------------- - const authConfig = buildDockerAuthConfig({ - host: data.registry.host, - user: data.registry.account_name, - password: data.registry.account_secret, - }); + const authConfig = buildDockerAuthConfigFromRegistry(data.registry); const client = useCoreClient(); const { data: analysisNodes } = await client.analysisNode.getMany({ diff --git a/packages/server-analysis-manager/src/components/index.ts b/packages/server-analysis-manager/src/components/index.ts index 9c9a60917..bd66edef6 100644 --- a/packages/server-analysis-manager/src/components/index.ts +++ b/packages/server-analysis-manager/src/components/index.ts @@ -7,5 +7,6 @@ export * from './builder'; export * from './core'; +export * from './master-images'; export * from './type'; export * from './utils'; diff --git a/packages/server-analysis-manager/src/components/master-images/commands/build/index.ts b/packages/server-analysis-manager/src/components/master-images/commands/build/index.ts new file mode 100644 index 000000000..7424f1649 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/build/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './module'; diff --git a/packages/server-analysis-manager/src/components/master-images/commands/build/module.ts b/packages/server-analysis-manager/src/components/master-images/commands/build/module.ts new file mode 100644 index 000000000..752b53f2e --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/build/module.ts @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { REGISTRY_MASTER_IMAGE_PROJECT_NAME } from '@privateaim/core-kit'; +import path from 'node:path'; +import tar from 'tar-fs'; +import type { + MasterImagesBuildCommandPayload, MasterImagesPushCommandPayloadTag, +} from '@privateaim/server-analysis-manager-kit'; +import { + buildRemoteDockerImageURL, useCoreClient, useDocker, waitForDockerActionStream, +} from '../../../../core'; +import { writePushCommand } from '../../queue'; + +export async function executeMasterImagesBuildCommand( + payload: MasterImagesBuildCommandPayload, +) { + const coreClient = useCoreClient(); + const docker = useDocker(); + const promises : Promise[] = []; + + const { data: registries } = await coreClient.registry.getMany(); + + const tags : MasterImagesPushCommandPayloadTag[] = []; + + for (let i = 0; i < payload.images.length; i++) { + const imagePath = path.join(payload.directory, payload.images[i].path); + + for (let j = 0; j < registries.length; j++) { + const imageTag = buildRemoteDockerImageURL({ + hostname: registries[j].host, + projectName: REGISTRY_MASTER_IMAGE_PROJECT_NAME, + repositoryName: payload.images[i].virtualPath, + tagOrDigest: 'latest', + }); + + tags.push({ + name: imageTag, + registryId: registries[j].id, + }); + + const pack = tar.pack(imagePath); + const promise = docker + .buildImage(pack, { + t: imageTag, + }) + .then((stream) => waitForDockerActionStream(stream)); + + promises.push(promise); + } + } + + await Promise.all(promises); + + await writePushCommand({ + tags, + }); + + return payload; +} diff --git a/packages/server-analysis-manager/src/components/master-images/commands/index.ts b/packages/server-analysis-manager/src/components/master-images/commands/index.ts new file mode 100644 index 000000000..36eec2b9d --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './build'; +export * from './push'; +export * from './synchronize'; diff --git a/packages/server-analysis-manager/src/components/master-images/commands/push/index.ts b/packages/server-analysis-manager/src/components/master-images/commands/push/index.ts new file mode 100644 index 000000000..7424f1649 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/push/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './module'; diff --git a/packages/server-analysis-manager/src/components/master-images/commands/push/module.ts b/packages/server-analysis-manager/src/components/master-images/commands/push/module.ts new file mode 100644 index 000000000..6fbe0a2f6 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/push/module.ts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import type { + MasterImagesPushCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; +import { + buildDockerAuthConfigFromRegistry, + useCoreClient, + useDocker, + waitForDockerActionStream, +} from '../../../../core'; + +async function pushMasterImage(tag: string, registryId: string) { + const coreClient = useCoreClient(); + const docker = useDocker(); + + const registry = await coreClient.registry.getOne(registryId, { + fields: ['+account_secret'], + }); + + const image = docker.getImage(tag); + + const stream = await image.push({ + authconfig: buildDockerAuthConfigFromRegistry(registry), + }); + + await waitForDockerActionStream(stream); +} +export async function executeMasterImagesPushCommand( + payload: MasterImagesPushCommandPayload, +) { + const promises : Promise[] = []; + + for (let i = 0; i < payload.tags.length; i++) { + promises.push(pushMasterImage(payload.tags[i].name, payload.tags[i].registryId)); + } + + await Promise.all(promises); + + return payload; +} diff --git a/packages/server-analysis-manager/src/components/master-images/commands/synchronize/helpers.ts b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/helpers.ts new file mode 100644 index 000000000..32acec819 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/helpers.ts @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { createClient } from 'hapic'; +import fs from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { extract } from 'tar'; + +type GitRepositorySaveOptions = { + destination: string, + url: string, + branch: string +}; +export async function saveGitRepository(options: GitRepositorySaveOptions) { + const tmpFilePath = path.join(tmpdir(), 'master-images.tar.gz'); + + await fs.promises.rm(options.destination, { force: true, recursive: true }); + await fs.promises.mkdir(options.destination, { recursive: true }); + + const client = createClient(); + + // todo: add branch + repo option + const response = await client.get( + new URL(`archive/${options.branch}.tar.gz`, options.url).href, + { + responseType: 'stream', + }, + ); + + const writable = fs.createWriteStream(tmpFilePath); + + return new Promise((resolve, reject) => { + writable.on('error', (err) => { + reject(err); + }); + + writable.on('finish', async () => { + await extract({ + file: tmpFilePath, + cwd: options.destination, + onReadEntry(entry) { + entry.path = entry.path.split('/').splice(1).join('/'); + }, + }); + + resolve(); + }); + + const readStream = Readable.fromWeb(response.data as any); + readStream.pipe(writable); + }); +} diff --git a/packages/server-analysis-manager/src/components/master-images/commands/synchronize/index.ts b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/index.ts new file mode 100644 index 000000000..7424f1649 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './module'; diff --git a/packages/server-analysis-manager/src/components/master-images/commands/synchronize/module.ts b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/module.ts new file mode 100644 index 000000000..64528f81a --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/commands/synchronize/module.ts @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { scanDirectory } from 'docker-scan'; +import type { MasterImagesSynchronizeCommandPayload } from '@privateaim/server-analysis-manager-kit'; +import path from 'node:path'; +import { WRITABLE_DIRECTORY_PATH } from '../../../../config'; +import { writeBuildCommand } from '../../queue'; +import { saveGitRepository } from './helpers'; + +export async function executeMasterImagesSynchronizeCommand( + payload: MasterImagesSynchronizeCommandPayload, +) { + const outputDirectoryPath = path.join(WRITABLE_DIRECTORY_PATH, 'master-images'); + + await saveGitRepository({ + destination: outputDirectoryPath, + branch: payload.branch || 'master', + url: payload.url || 'https://github.com/PHT-Medic/master-images/', + }); + + const { images, groups } = await scanDirectory(path.join(outputDirectoryPath, 'data')); + + await writeBuildCommand({ + directory: outputDirectoryPath, + images, + }); + + return { + images, + groups, + }; +} diff --git a/packages/server-analysis-manager/src/components/master-images/index.ts b/packages/server-analysis-manager/src/components/master-images/index.ts new file mode 100644 index 000000000..e564c7a3d --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2023-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './module'; diff --git a/packages/server-analysis-manager/src/components/master-images/module.ts b/packages/server-analysis-manager/src/components/master-images/module.ts new file mode 100644 index 000000000..48b659bc3 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/module.ts @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import type { + MasterImagesBuildCommandPayload, + MasterImagesPushCommandPayload, + MasterImagesSynchronizeCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesCommand, + MasterImagesEvent, + MasterImagesTaskQueueRouterRouting, +} from '@privateaim/server-analysis-manager-kit'; +import type { Component, QueueRouterHandlers } from '@privateaim/server-kit'; +import { isQueueRouterUsable, useLogger, useQueueRouter } from '@privateaim/server-kit'; +import { EnvironmentName, useEnv } from '../../config'; +import { + executeMasterImagesBuildCommand, + executeMasterImagesPushCommand, + executeMasterImagesSynchronizeCommand, +} from './commands'; +import { + writeBuildingEvent, + writeBuiltEvent, + writeFailedEvent, + writePushedEvent, + writePushingEvent, + writeSynchronizedEvent, + writeSynchronizingEvent, +} from './queue'; + +function createHandlers() : QueueRouterHandlers<{ + [MasterImagesCommand.SYNCHRONIZE]: MasterImagesSynchronizeCommandPayload, + [MasterImagesCommand.BUILD]: MasterImagesBuildCommandPayload, + [MasterImagesCommand.PUSH]: MasterImagesPushCommandPayload +}> { + return { + [MasterImagesCommand.SYNCHRONIZE]: async (message) => { + await Promise.resolve(message.data) + .then((data) => writeSynchronizingEvent(data)) + .then((data) => executeMasterImagesSynchronizeCommand(data)) + .then((data) => writeSynchronizedEvent(data)) + .catch((err: Error) => { + // todo: use logger + console.error(err); + + return writeFailedEvent(MasterImagesEvent.SYNCHRONIZATION_FAILED, { + error: err, + }); + }); + }, + [MasterImagesCommand.BUILD]: async (message) => { + await Promise.resolve(message.data) + .then((data) => writeBuildingEvent(data)) + .then((data) => executeMasterImagesBuildCommand(data)) + .then((data) => writeBuiltEvent(data)) + .catch((err: Error) => { + // todo: use logger + console.error(err); + + return writeFailedEvent(MasterImagesEvent.SYNCHRONIZATION_FAILED, { + error: err, + }); + }); + }, + [MasterImagesCommand.PUSH]: async (message) => { + await Promise.resolve(message.data) + .then((data) => writePushingEvent((data))) + .then((data) => executeMasterImagesPushCommand(data)) + .then((data) => writePushedEvent(data)) + .catch((err: Error) => { + // todo: use logger + console.error(err); + + return writeFailedEvent(MasterImagesEvent.PUSH_FAILED, { + error: err, + }); + }); + }, + }; +} + +export function createMasterImagesComponent() : Component { + if (!isQueueRouterUsable() || useEnv('env') === EnvironmentName.TEST) { + // todo: maybe log + return { + start() { + useLogger().warn('MasterImages component could not be started'); + }, + }; + } + + const queueRouter = useQueueRouter(); + + return { + start() { + return queueRouter.consume(MasterImagesTaskQueueRouterRouting, createHandlers()); + }, + }; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/index.ts b/packages/server-analysis-manager/src/components/master-images/queue/index.ts new file mode 100644 index 000000000..13c5cbf10 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +export * from './write-build'; +export * from './write-building'; +export * from './write-built'; +export * from './write-snychronized'; +export * from './write-synchronizing'; +export * from './write-failed'; +export * from './write-push'; +export * from './write-pushed'; +export * from './write-pushing'; diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-build.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-build.ts new file mode 100644 index 000000000..eea502c19 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-build.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import { + MasterImagesCommand, + buildMasterImagesTaskQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; +import type { + MasterImagesBuildCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writeBuildCommand( + data: MasterImagesBuildCommandPayload, +) { + const client = useQueueRouter(); + await client.publish(buildMasterImagesTaskQueueRouterPayload({ + command: MasterImagesCommand.BUILD, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-building.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-building.ts new file mode 100644 index 000000000..0969fc5e7 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-building.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import type { MasterImagesBuildCommandPayload } from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesEvent, + buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writeBuildingEvent( + data: MasterImagesBuildCommandPayload, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.BUILDING, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-built.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-built.ts new file mode 100644 index 000000000..925d06a1d --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-built.ts @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import type { MasterImagesBuildCommandPayload } from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesEvent, + buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writeBuiltEvent( + data: MasterImagesBuildCommandPayload, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.BUILT, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-failed.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-failed.ts new file mode 100644 index 000000000..258889454 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-failed.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import type { + MasterImagesBasePayload, + + MasterImagesEvent, +} from '@privateaim/server-analysis-manager-kit'; +import { buildMasterImagesEventQueueRouterPayload } from '@privateaim/server-analysis-manager-kit'; +import { useQueueRouter } from '@privateaim/server-kit'; + +export async function writeFailedEvent( + event: `${MasterImagesEvent.BUILD_FAILED}` | + `${MasterImagesEvent.SYNCHRONIZATION_FAILED}` | + `${MasterImagesEvent.PUSH_FAILED}`, + data: T, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-push.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-push.ts new file mode 100644 index 000000000..d80af1547 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-push.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import type { + MasterImagesPushCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesCommand, + buildMasterImagesTaskQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writePushCommand( + data: MasterImagesPushCommandPayload, +) { + const client = useQueueRouter(); + await client.publish(buildMasterImagesTaskQueueRouterPayload({ + command: `${MasterImagesCommand.PUSH}`, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-pushed.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-pushed.ts new file mode 100644 index 000000000..8e32c3698 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-pushed.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import type { + MasterImagesPushCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesEvent, + buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writePushedEvent( + data: MasterImagesPushCommandPayload, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.PUSHED, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-pushing.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-pushing.ts new file mode 100644 index 000000000..d52393e3e --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-pushing.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import type { + MasterImagesPushCommandPayload, +} from '@privateaim/server-analysis-manager-kit'; +import { + MasterImagesEvent, + buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writePushingEvent( + data: MasterImagesPushCommandPayload, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.PUSHING, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-snychronized.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-snychronized.ts new file mode 100644 index 000000000..be1a940e9 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-snychronized.ts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import { + MasterImagesEvent, buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; +import type { + MaterImagesSynchronizedEventPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writeSynchronizedEvent( + data: MaterImagesSynchronizedEventPayload, +) { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.SYNCHRONIZED, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/components/master-images/queue/write-synchronizing.ts b/packages/server-analysis-manager/src/components/master-images/queue/write-synchronizing.ts new file mode 100644 index 000000000..c36bbdaa9 --- /dev/null +++ b/packages/server-analysis-manager/src/components/master-images/queue/write-synchronizing.ts @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022-2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useQueueRouter } from '@privateaim/server-kit'; +import { + MasterImagesEvent, + buildMasterImagesEventQueueRouterPayload, +} from '@privateaim/server-analysis-manager-kit'; + +export async function writeSynchronizingEvent( + data: T, +) : Promise { + const client = useQueueRouter(); + await client.publish(buildMasterImagesEventQueueRouterPayload({ + event: MasterImagesEvent.SYNCHRONIZING, + data, + })); + + return data; +} diff --git a/packages/server-analysis-manager/src/config/module.ts b/packages/server-analysis-manager/src/config/module.ts index e65e48cd2..a16a42295 100644 --- a/packages/server-analysis-manager/src/config/module.ts +++ b/packages/server-analysis-manager/src/config/module.ts @@ -8,7 +8,7 @@ import type { Aggregator, Component } from '@privateaim/server-kit'; import { guessAuthupTokenCreatorOptions } from '@privateaim/server-kit'; import { - createBuilderComponent, createCoreComponent, + createBuilderComponent, createCoreComponent, createMasterImagesComponent, } from '../components'; import { configureAMQP, configureCoreService, configureStorageService, setupLogger, setupVault, @@ -29,6 +29,7 @@ export function createConfig() : Config { const components : Component[] = [ createBuilderComponent(), createCoreComponent(), + createMasterImagesComponent(), ]; return { diff --git a/packages/server-analysis-manager/src/core/docker/auth-config.ts b/packages/server-analysis-manager/src/core/docker/auth-config.ts new file mode 100644 index 000000000..88fb97dc2 --- /dev/null +++ b/packages/server-analysis-manager/src/core/docker/auth-config.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import type { Registry } from '@privateaim/core-kit'; +import { getHostNameFromString } from '@privateaim/kit'; +import type { AuthConfig } from 'dockerode'; +import type { DockerAuthConfig, DockerConnectionOptions } from './type'; + +export function buildDockerAuthConfig(config: DockerConnectionOptions): DockerAuthConfig { + return { + username: config.user, + password: config.password, + serveraddress: getHostNameFromString(config.host), + }; +} + +export function buildDockerAuthConfigFromRegistry(registry: Registry) : AuthConfig { + return buildDockerAuthConfig({ + host: registry.host, + user: registry.account_name, + password: registry.account_secret, + }); +} diff --git a/packages/server-analysis-manager/src/core/docker/image-push.ts b/packages/server-analysis-manager/src/core/docker/image-push.ts index 2bd855081..6b07f4d55 100644 --- a/packages/server-analysis-manager/src/core/docker/image-push.ts +++ b/packages/server-analysis-manager/src/core/docker/image-push.ts @@ -5,33 +5,28 @@ * view the LICENSE file that was distributed with this source code. */ -import type { Image } from 'dockerode'; +import type { AuthConfig, Image } from 'dockerode'; import { useDocker } from './instance'; -import { findErrorInDockerModemResponse } from './modem-response'; -import type { DockerAuthConfig } from './type'; +import { waitForDockerActionStream } from './modem-wait'; -export async function pushDockerImage(image: Image | string, authConfig: DockerAuthConfig) { - const imageLatest = typeof image === 'string' ? - await useDocker().getImage(image) : - image; +export async function pushDockerImage( + input: Image | string, + authConfig: AuthConfig, +) { + const docker = useDocker(); + + let imageLatest : Image; + if (typeof input === 'string') { + imageLatest = docker.getImage(input); + } else { + imageLatest = input; + } const stream = await imageLatest.push({ authconfig: authConfig, }); - await new Promise((resolve, reject) => { - useDocker().modem.followProgress( - stream as any, - (error: Error, output: any[]) => { - error = error || findErrorInDockerModemResponse(output); - if (error) { - return reject(error); - } - - return resolve(output); - }, - ); - }); + await waitForDockerActionStream(stream); await imageLatest.remove({ force: true, diff --git a/packages/server-analysis-manager/src/core/docker/index.ts b/packages/server-analysis-manager/src/core/docker/index.ts index 0b8d6b35a..51cae5b5f 100644 --- a/packages/server-analysis-manager/src/core/docker/index.ts +++ b/packages/server-analysis-manager/src/core/docker/index.ts @@ -5,12 +5,14 @@ * view the LICENSE file that was distributed with this source code. */ +export * from './auth-config'; export * from './container-read'; export * from './container-save'; export * from './image-clenaup'; export * from './image-pull'; export * from './image-push'; export * from './image-remove'; +export * from './modem-wait'; export * from './modem-response'; export * from './instance'; export * from './registry'; diff --git a/packages/server-analysis-manager/src/core/docker/modem-wait.ts b/packages/server-analysis-manager/src/core/docker/modem-wait.ts new file mode 100644 index 000000000..7f602843b --- /dev/null +++ b/packages/server-analysis-manager/src/core/docker/modem-wait.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { useDocker } from './instance'; +import { findErrorInDockerModemResponse } from './modem-response'; + +export type DockerActionStreamWaitOptions = { + onError?: (error: Error) => any; + onCompleted?: () => any, + onProgress?: (res: any) => any +}; +export async function waitForDockerActionStream( + stream: NodeJS.ReadableStream, + options: DockerActionStreamWaitOptions = {}, +) { + return new Promise((resolve, reject) => { + useDocker().modem.followProgress(stream as any, (error: Error, output: any[]) => { + error = error || findErrorInDockerModemResponse(output); + if (error) { + if (options.onError) { + options.onError(error); + } + + reject(error); + return; + } + + if (options.onCompleted) { + options.onCompleted(); + } + + resolve(output); + }, (res) => { + if (options.onProgress) { + options.onProgress(res); + } + }); + }); +} diff --git a/packages/server-analysis-manager/src/core/docker/registry.ts b/packages/server-analysis-manager/src/core/docker/registry.ts index ad5c3db82..03535899f 100644 --- a/packages/server-analysis-manager/src/core/docker/registry.ts +++ b/packages/server-analysis-manager/src/core/docker/registry.ts @@ -6,7 +6,6 @@ */ import { getHostNameFromString } from '@privateaim/kit'; -import type { DockerAuthConfig, DockerConnectionOptions } from './type'; type RemoteDockerImageURLBuildContext = { projectName: string, @@ -31,11 +30,3 @@ export function buildRemoteDockerImageURL(context: RemoteDockerImageURLBuildCont return basePath; } - -export function buildDockerAuthConfig(config: DockerConnectionOptions) : DockerAuthConfig { - return { - username: config.user, - password: config.password, - serveraddress: getHostNameFromString(config.host), - }; -}