From a80c0bdce23ffdbe2afec2f62f394b23c82c5f7e Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Wed, 26 Jun 2024 19:32:37 +0100 Subject: [PATCH 1/6] Tusd fix --- caddy/Caddyfile | 3 --- caddy/Caddyfile.staging | 9 +++++++++ docker-compose.staging.yml | 2 +- src/services/api/api.contoller.test.ts | 16 ++++++++-------- src/services/api/api.controller.ts | 2 +- src/services/auth/auth.controller.test.ts | 12 ++++++------ src/services/auth/auth.controller.ts | 2 +- .../uploader/uploading.controller.test.ts | 2 +- src/services/uploader/uploading.controller.ts | 2 +- 9 files changed, 28 insertions(+), 22 deletions(-) delete mode 100644 caddy/Caddyfile create mode 100644 caddy/Caddyfile.staging diff --git a/caddy/Caddyfile b/caddy/Caddyfile deleted file mode 100644 index d142002..0000000 --- a/caddy/Caddyfile +++ /dev/null @@ -1,3 +0,0 @@ -staging.3speak.tv { - reverse_proxy acela-core:4569 -} \ No newline at end of file diff --git a/caddy/Caddyfile.staging b/caddy/Caddyfile.staging new file mode 100644 index 0000000..aad5673 --- /dev/null +++ b/caddy/Caddyfile.staging @@ -0,0 +1,9 @@ +staging.3speak.tv { + handle_path /api/* { + reverse_proxy acela-core:4569 + } + + handle_path /tusd/* { + reverse_proxy tusd:1080 + } +} diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 3378ce9..d5b3c65 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -6,7 +6,7 @@ services: - "80:80" - "443:443" volumes: - - ./caddy/Caddyfile:/etc/caddy/Caddyfile + - ./caddy/Caddyfile.staging:/etc/caddy/Caddyfile - /data/caddy/data:/data - /data/caddy/config:/config networks: diff --git a/src/services/api/api.contoller.test.ts b/src/services/api/api.contoller.test.ts index c77efea..7aff2c7 100644 --- a/src/services/api/api.contoller.test.ts +++ b/src/services/api/api.contoller.test.ts @@ -107,7 +107,7 @@ describe('ApiController', () => { await mongod.stop(); }); - describe('/POST /api/v1/hive/post_comment', () => { + describe('/POST /v1/hive/post_comment', () => { it('should post a comment to HIVE blockchain', async () => { const jwtToken = 'test_jwt_token'; const body = { @@ -118,7 +118,7 @@ describe('ApiController', () => { }; return request(app.getHttpServer()) - .post('/api/v1/hive/post_comment') + .post('/v1/hive/post_comment') .set('Authorization', `Bearer ${jwtToken}`) .send(body) .expect(201) @@ -133,13 +133,13 @@ describe('ApiController', () => { }); }); - describe('/POST /api/v1/hive/linkaccount', () => { + describe('/POST /v1/hive/linkaccount', () => { it('should link a Hive account', async () => { const jwtToken = 'test_jwt_token'; const body = { username: 'test-account' }; return request(app.getHttpServer()) - .post('/api/v1/hive/linkaccount') + .post('/v1/hive/linkaccount') .set('Authorization', `Bearer ${jwtToken}`) .send(body) .expect(201) @@ -151,12 +151,12 @@ describe('ApiController', () => { }); }); - describe('/GET /api/v1/profile', () => { + describe('/GET /v1/profile', () => { it('should get the user profile', async () => { const jwtToken = 'test_jwt_token'; return request(app.getHttpServer()) - .get('/api/v1/profile') + .get('/v1/profile') .set('Authorization', `Bearer ${jwtToken}`) .expect(200) .then(response => { @@ -168,7 +168,7 @@ describe('ApiController', () => { }); }); - describe('/GET /api/v1/hive/linked-account/list', () => { + describe('/GET /v1/hive/linked-account/list', () => { it('should list linked accounts', async () => { const jwtToken = 'test_jwt_token'; @@ -177,7 +177,7 @@ describe('ApiController', () => { await linkedAccountsRepository.verify(link._id); return request(app.getHttpServer()) - .get('/api/v1/hive/linked-account/list') + .get('/v1/hive/linked-account/list') .set('Authorization', `Bearer ${jwtToken}`) .expect(200) .then(response => { diff --git a/src/services/api/api.controller.ts b/src/services/api/api.controller.ts index 343ffee..98460d6 100644 --- a/src/services/api/api.controller.ts +++ b/src/services/api/api.controller.ts @@ -32,7 +32,7 @@ import { LinkedAccountRepository } from '../../repositories/linked-accounts/link import { EmailService } from '../email/email.service'; import { parseAndValidateRequest } from '../auth/auth.utils'; -@Controller('/api/v1') +@Controller('/v1') export class ApiController { readonly #logger = new Logger(); diff --git a/src/services/auth/auth.controller.test.ts b/src/services/auth/auth.controller.test.ts index 9461137..d573f35 100644 --- a/src/services/auth/auth.controller.test.ts +++ b/src/services/auth/auth.controller.test.ts @@ -115,7 +115,7 @@ describe('AuthController', () => { const jws = await did.createJWS(payload); return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/did') + .post('/v1/auth/login/singleton/did') .send(jws) .set('Content-Type', 'application/json') .set('Accept', 'application/json') @@ -144,7 +144,7 @@ describe('AuthController', () => { } return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/hive') + .post('/v1/auth/login/singleton/hive') .send(body) .expect(201) .then(response => { @@ -167,7 +167,7 @@ describe('AuthController', () => { } return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/hive') + .post('/v1/auth/login/singleton/hive') .send(body) .expect(401) .then(response => { @@ -192,7 +192,7 @@ describe('AuthController', () => { } return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/hive') + .post('/v1/auth/login/singleton/hive') .send(body) .expect(401) .then(response => { @@ -217,7 +217,7 @@ describe('AuthController', () => { } return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/hive') + .post('/v1/auth/login/singleton/hive') .send(body) .expect(401) .then(response => { @@ -240,7 +240,7 @@ describe('AuthController', () => { } return request(app.getHttpServer()) - .post('/api/v1/auth/login/singleton/hive') + .post('/v1/auth/login/singleton/hive') .send(body) .expect(401) .then(response => { diff --git a/src/services/auth/auth.controller.ts b/src/services/auth/auth.controller.ts index 7afe6f2..5a30ce4 100644 --- a/src/services/auth/auth.controller.ts +++ b/src/services/auth/auth.controller.ts @@ -42,7 +42,7 @@ import bcrypt from 'bcryptjs'; import { WithAuthData } from './auth.interface'; import { parseAndValidateRequest } from './auth.utils'; -@Controller('/api/v1/auth') +@Controller('/v1/auth') export class AuthController { readonly #logger = new Logger(AuthController.name); diff --git a/src/services/uploader/uploading.controller.test.ts b/src/services/uploader/uploading.controller.test.ts index 9b82452..f949e95 100644 --- a/src/services/uploader/uploading.controller.test.ts +++ b/src/services/uploader/uploading.controller.test.ts @@ -105,7 +105,7 @@ describe('UploadingController', () => { .toBuffer(); return request(app.getHttpServer()) - .post('/api/v1/upload/thumbnail') + .post('/v1/upload/thumbnail') .set('Authorization', `Bearer ${jwtToken}`) .attach('file', semiTransparentRedPng, 'test-image.png') .field('video_id', 'test_video_id') diff --git a/src/services/uploader/uploading.controller.ts b/src/services/uploader/uploading.controller.ts index 523ec6c..d0ff196 100644 --- a/src/services/uploader/uploading.controller.ts +++ b/src/services/uploader/uploading.controller.ts @@ -34,7 +34,7 @@ MulterModule.registerAsync({ }), }); -@Controller('/api/v1/upload') +@Controller('/v1/upload') export class UploadingController { readonly #logger = new Logger(UploadingController.name); From bd7bff27eed5db134ba1dc23e167eb4f3c755698 Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Wed, 26 Jun 2024 19:34:26 +0100 Subject: [PATCH 2/6] Improvements to docker compose file --- docker-compose.staging.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index d5b3c65..bff8d3f 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -12,12 +12,18 @@ services: networks: - web - acela-core - acela-core: # name of the service + environment: + - ACME_AGREE=true + depends_on: + - acela-core + - tusd + + acela-core: build: - context: . # the directory containing Docker context (".", meaning the current directory) - dockerfile: Dockerfile.staging # specify the local Dockerfile - container_name: acela-core # what to label the container for docker ps - restart: always # restart if failed, until we stop it ourselves + context: . + dockerfile: Dockerfile.staging + container_name: acela-core + restart: always env_file: - .env.staging #external_links: @@ -32,6 +38,7 @@ services: UPLOAD_PATH: /acela-uploads volumes: - ./data/acela-uploads:/acela-uploads + tusd: image: tusproject/tusd:v1.9 command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/api/v1/tus-callback @@ -45,6 +52,7 @@ services: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY AWS_REGION: eu-west-1 + minio: image: minio/minio:RELEASE.2019-08-14T20-37-41Z networks: From 3d8c2b493b332b7e8a44a6cb83f0853d447097b3 Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Wed, 26 Jun 2024 19:41:53 +0100 Subject: [PATCH 3/6] More changes relating to url pattern matching --- docker-compose.local.yml | 2 +- docker-compose.staging.yml | 4 ++-- runTusd.sh | 2 +- src/index.ts | 2 +- src/services/auth/auth.module.ts | 2 +- src/services/email/email.service.ts | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 547cef9..fe64d8f 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -23,7 +23,7 @@ services: - mongo tusd: image: tusproject/tusd:v1.9 - command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/api/v1/tus-callback + command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/tus-callback volumes: - ./data/tusd-uploads:/data networks: diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index bff8d3f..da78e01 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -41,7 +41,7 @@ services: tusd: image: tusproject/tusd:v1.9 - command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/api/v1/tus-callback + command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/tus-callback volumes: - ./data/tusd-uploads:/data networks: @@ -52,7 +52,7 @@ services: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY AWS_REGION: eu-west-1 - + minio: image: minio/minio:RELEASE.2019-08-14T20-37-41Z networks: diff --git a/runTusd.sh b/runTusd.sh index 18fd3d1..d5b46d4 100644 --- a/runTusd.sh +++ b/runTusd.sh @@ -1,2 +1,2 @@ -tusd -upload-dir=./data/ -base-path="/files" -host="127.0.0.1" -port="1080" -behind-proxy --hooks-enabled-events pre-create,post-create,post-finish,post-finish,post-terminate,post-receive -max-size 5000000000 -hooks-http http://localhost:4569/api/v1/upload/tus-callback \ No newline at end of file +tusd -upload-dir=./data/ -base-path="/files" -host="127.0.0.1" -port="1080" -behind-proxy --hooks-enabled-events pre-create,post-create,post-finish,post-finish,post-terminate,post-receive -max-size 5000000000 -hooks-http http://localhost:4569/v1/upload/tus-callback \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 32063cc..7457659 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,7 +15,7 @@ async function startup(): Promise { app.useGlobalPipes(new ValidationPipe()); const config = new DocumentBuilder().build(); const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('/api/v1/docs', app, document); + SwaggerModule.setup('/v1/docs', app, document); await app.listen(4569); } diff --git a/src/services/auth/auth.module.ts b/src/services/auth/auth.module.ts index 9ab5a58..e5ca90f 100644 --- a/src/services/auth/auth.module.ts +++ b/src/services/auth/auth.module.ts @@ -36,6 +36,6 @@ import { ConfigModule, ConfigService } from '@nestjs/config'; }) export class AuthModule implements NestModule { configure(consumer: MiddlewareConsumer) { - consumer.apply(AuthMiddleware).forRoutes('/api/v1/auth/login_singleton/did'); + consumer.apply(AuthMiddleware).forRoutes('/v1/auth/login_singleton/did'); } } diff --git a/src/services/email/email.service.ts b/src/services/email/email.service.ts index 11a594c..4290c2b 100644 --- a/src/services/email/email.service.ts +++ b/src/services/email/email.service.ts @@ -35,7 +35,7 @@ export class EmailService { await this.send( email, 'Complete 3Speak registration', - `test registration. Click here to verify email address.`, + `test registration. Click here to verify email address.`, ); } } From 6f4824a1bfd7b9920260eb0cd8ce9331bea012b9 Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Thu, 27 Jun 2024 20:29:05 +0100 Subject: [PATCH 4/6] Fix error in test --- src/services/auth/auth.controller.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/auth/auth.controller.test.ts b/src/services/auth/auth.controller.test.ts index 0e0c3ff..c5b5300 100644 --- a/src/services/auth/auth.controller.test.ts +++ b/src/services/auth/auth.controller.test.ts @@ -143,7 +143,7 @@ describe('AuthController', () => { // Make the request to the endpoint return request(app.getHttpServer()) - .post('/api/v1/auth/request_hive_account') + .post('/v1/auth/request_hive_account') .send({ username: 'test_user_id' }) .set('Authorization', 'Bearer ') .expect(201) @@ -163,7 +163,7 @@ describe('AuthController', () => { // Make the request to the endpoint return request(app.getHttpServer()) - .post('/api/v1/auth/request_hive_account') + .post('/v1/auth/request_hive_account') .send({ username: 'yeet' }) .set('Authorization', 'Bearer ') .expect(400) From bbce360d613151d6d52455e590b5ae1784759918 Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Fri, 28 Jun 2024 00:45:05 +0100 Subject: [PATCH 5/6] Tusd fix --- Dockerfile.staging | 2 +- caddy/Caddyfile.staging | 4 +--- docker-compose.local.yml | 2 +- docker-compose.staging.yml | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile.staging b/Dockerfile.staging index 2854754..9378669 100644 --- a/Dockerfile.staging +++ b/Dockerfile.staging @@ -1,7 +1,7 @@ FROM node:18.14.2-alpine WORKDIR /usr/app COPY package*.json ./ -RUN apk add --no-cache git python3 make g++ +RUN apk add --no-cache git python3 make g++ ffmpeg RUN npm install && npm install typescript -g COPY . . RUN tsc diff --git a/caddy/Caddyfile.staging b/caddy/Caddyfile.staging index aad5673..729540b 100644 --- a/caddy/Caddyfile.staging +++ b/caddy/Caddyfile.staging @@ -3,7 +3,5 @@ staging.3speak.tv { reverse_proxy acela-core:4569 } - handle_path /tusd/* { - reverse_proxy tusd:1080 - } + reverse_proxy /tusd/* tusd:1080 } diff --git a/docker-compose.local.yml b/docker-compose.local.yml index fe64d8f..c6d7386 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -23,7 +23,7 @@ services: - mongo tusd: image: tusproject/tusd:v1.9 - command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/tus-callback + command: -host=0.0.0.0 -port=1080 -base-path /tusd/files/ -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/upload/tus-callback volumes: - ./data/tusd-uploads:/data networks: diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index da78e01..696ac0e 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -41,7 +41,7 @@ services: tusd: image: tusproject/tusd:v1.9 - command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/tus-callback + command: -host=0.0.0.0 -port=1080 -base-path /tusd/files/ -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/upload/tus-callback volumes: - ./data/tusd-uploads:/data networks: From 072d888c3908d5175b86dd9f2afeb77d607599f3 Mon Sep 17 00:00:00 2001 From: Christopher Graney-Ward Date: Fri, 28 Jun 2024 02:08:38 +0100 Subject: [PATCH 6/6] Fix ts error --- src/services/storage-cluster/allocator.ts | 11 ++++++++--- src/services/storage-cluster/peer.ts | 2 ++ src/services/storage-cluster/types.ts | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/services/storage-cluster/allocator.ts b/src/services/storage-cluster/allocator.ts index b2e91dd..fe065c0 100644 --- a/src/services/storage-cluster/allocator.ts +++ b/src/services/storage-cluster/allocator.ts @@ -293,6 +293,7 @@ export class StorageClusterAllocator extends StorageCluster { allocations: { id: peerId, allocated_at: ts, + reported_size: 0, }, }, $inc: { @@ -410,6 +411,10 @@ export class StorageClusterAllocator extends StorageCluster { const reported_sizes = pinned.allocations .map((a) => a.reported_size) .filter((size) => size !== null && typeof size !== 'undefined'); + if (!completedPin.size) { + console.error('No size found for pin ' + completedPin.cid); + completedPin.size = 0; + } reported_sizes.push(completedPin.size); if (preAllocated === -1) { await this.pins.updateOne( @@ -420,7 +425,7 @@ export class StorageClusterAllocator extends StorageCluster { id: peerId, allocated_at: msgTs, pinned_at: msgTs, - reported_size: completedPin.size || 0, + reported_size: completedPin.size, }, }, $inc: { @@ -510,7 +515,7 @@ export class StorageClusterAllocator extends StorageCluster { id: peerId, allocated_at: msgTs, pinned_at: msgTs, - reported_size: newPin.size, + reported_size: newPin.size || 0, }; if (!alreadyExists || alreadyExists.status === 'deleted') await this.pins.updateOne( @@ -524,7 +529,7 @@ export class StorageClusterAllocator extends StorageCluster { last_updated: msgTs, allocations: [newAlloc], allocationCount: 1, - median_size: newPin.size || 0, // Ensure median_size is of type number + median_size: newPin.size, // Ensure median_size is of type number metadata: newPin.metadata || {}, }, }, diff --git a/src/services/storage-cluster/peer.ts b/src/services/storage-cluster/peer.ts index 1dac5bf..1f96821 100644 --- a/src/services/storage-cluster/peer.ts +++ b/src/services/storage-cluster/peer.ts @@ -232,6 +232,7 @@ export class StorageClusterPeer extends StorageCluster { allocations: { id: this.getPeerId(), allocated_at: msgTs, + reported_size: 0, }, }, $inc: { @@ -348,6 +349,7 @@ export class StorageClusterPeer extends StorageCluster { type: SocketMsgTypes.PIN_FAILED, data: { cid: cid, + size: 0, }, ts: failedTs, }); diff --git a/src/services/storage-cluster/types.ts b/src/services/storage-cluster/types.ts index 2577c1a..c062691 100644 --- a/src/services/storage-cluster/types.ts +++ b/src/services/storage-cluster/types.ts @@ -113,7 +113,7 @@ export interface PinAllocate { id: string; allocated_at: number; pinned_at?: number | undefined; - reported_size?: number | undefined; + reported_size: number; } export interface Pin {