Merge pull request #26 from spknetwork/tusd-fix

Tusd fix

Dockerfile.staging

@@ -1,7 +1,7 @@
 FROM node:18.14.2-alpine
 WORKDIR /usr/app
 COPY package*.json ./
-RUN apk add --no-cache git python3 make g++
+RUN apk add --no-cache git python3 make g++ ffmpeg
 RUN npm install && npm install typescript -g
 COPY . .
 RUN tsc

caddy/Caddyfile.staging

@@ -0,0 +1,7 @@
+staging.3speak.tv {
+  handle_path /api/* {
+    reverse_proxy acela-core:4569
+  }
+
+  reverse_proxy /tusd/* tusd:1080
+}

docker-compose.local.yml

@@ -23,7 +23,7 @@ services:
       - mongo
   tusd:
     image: tusproject/tusd:v1.9
-    command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/api/v1/tus-callback
+    command: -host=0.0.0.0 -port=1080 -base-path /tusd/files/ -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/upload/tus-callback
     volumes:
       - ./data/tusd-uploads:/data
     networks:

docker-compose.staging.yml

@@ -6,18 +6,24 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - ./caddy/Caddyfile:/etc/caddy/Caddyfile
+      - ./caddy/Caddyfile.staging:/etc/caddy/Caddyfile
       - /data/caddy/data:/data
       - /data/caddy/config:/config
     networks:
       - web
       - acela-core
-  acela-core: # name of the service
+    environment:
+      - ACME_AGREE=true
+    depends_on:
+      - acela-core
+      - tusd
+
+  acela-core:
     build:
-      context: . # the directory containing Docker context (".", meaning the current directory)
-      dockerfile: Dockerfile.staging # specify the local Dockerfile
-    container_name: acela-core # what to label the container for docker ps
-    restart: always # restart if failed, until we stop it ourselves
+      context: .
+      dockerfile: Dockerfile.staging
+    container_name: acela-core
+    restart: always
     env_file:
       - .env.staging
     #external_links:
@@ -32,9 +38,10 @@ services:
       UPLOAD_PATH: /acela-uploads
     volumes:
       - ./data/acela-uploads:/acela-uploads
+
   tusd:
     image: tusproject/tusd:v1.9
-    command: -s3-bucket mybucket -s3-endpoint http://minio:9000 -host=0.0.0.0 -port=1080 -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/api/v1/tus-callback
+    command: -host=0.0.0.0 -port=1080 -base-path /tusd/files/ -behind-proxy -max-size 5000000000 -hooks-http http://acela-core:4569/v1/upload/tus-callback
     volumes:
       - ./data/tusd-uploads:/data
     networks:
@@ -45,6 +52,7 @@ services:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
       AWS_REGION: eu-west-1
+
   minio:
     image: minio/minio:RELEASE.2019-08-14T20-37-41Z
     networks: 

runTusd.sh

@@ -1,2 +1,2 @@
 
-tusd -upload-dir=./data/ -base-path="/files" -host="127.0.0.1" -port="1080" -behind-proxy --hooks-enabled-events pre-create,post-create,post-finish,post-finish,post-terminate,post-receive -max-size 5000000000 -hooks-http http://localhost:4569/api/v1/upload/tus-callback
+tusd -upload-dir=./data/ -base-path="/files" -host="127.0.0.1" -port="1080" -behind-proxy --hooks-enabled-events pre-create,post-create,post-finish,post-finish,post-terminate,post-receive -max-size 5000000000 -hooks-http http://localhost:4569/v1/upload/tus-callback

src/index.ts

@@ -15,7 +15,7 @@ async function startup(): Promise<void> {
   app.useGlobalPipes(new ValidationPipe());
   const config = new DocumentBuilder().build();
   const document = SwaggerModule.createDocument(app, config);
-  SwaggerModule.setup('/api/v1/docs', app, document);
+  SwaggerModule.setup('/v1/docs', app, document);
   await app.listen(4569);
 }
 

src/services/api/api.contoller.test.ts

@@ -109,7 +109,7 @@ describe('ApiController', () => {
     await mongod.stop();
   });
 
-  describe('/POST /api/v1/hive/post_comment', () => {
+  describe('/POST /v1/hive/post_comment', () => {
     it('should post a comment to HIVE blockchain', async () => {
       const jwtToken = 'test_jwt_token';
       const body = {
@@ -120,7 +120,7 @@ describe('ApiController', () => {
       };
 
       return request(app.getHttpServer())
-        .post('/api/v1/hive/post_comment')
+        .post('/v1/hive/post_comment')
         .set('Authorization', `Bearer ${jwtToken}`)
         .send(body)
         .expect(201)
@@ -135,13 +135,13 @@ describe('ApiController', () => {
     });
   });
 
-  describe('/POST /api/v1/hive/linkaccount', () => {
+  describe('/POST /v1/hive/linkaccount', () => {
     it('should link a Hive account', async () => {
       const jwtToken = 'test_jwt_token';
       const body = { username: 'test-account' };
 
       return request(app.getHttpServer())
-        .post('/api/v1/hive/linkaccount')
+        .post('/v1/hive/linkaccount')
         .set('Authorization', `Bearer ${jwtToken}`)
         .send(body)
         .expect(201)
@@ -153,12 +153,12 @@ describe('ApiController', () => {
     });
   });
 
-  describe('/GET /api/v1/profile', () => {
+  describe('/GET /v1/profile', () => {
     it('should get the user profile', async () => {
       const jwtToken = 'test_jwt_token';
 
       return request(app.getHttpServer())
-        .get('/api/v1/profile')
+        .get('/v1/profile')
         .set('Authorization', `Bearer ${jwtToken}`)
         .expect(200)
         .then(response => {
@@ -172,7 +172,7 @@ describe('ApiController', () => {
     });
   });
 
-  describe('/GET /api/v1/hive/linked-account/list', () => {
+  describe('/GET /v1/hive/linked-account/list', () => {
     it('should list linked accounts', async () => {
       const jwtToken = 'test_jwt_token';
 
@@ -181,7 +181,7 @@ describe('ApiController', () => {
       await linkedAccountsRepository.verify(link._id);
 
       return request(app.getHttpServer())
-        .get('/api/v1/hive/linked-account/list')
+        .get('/v1/hive/linked-account/list')
         .set('Authorization', `Bearer ${jwtToken}`)
         .expect(200)
         .then(response => {

src/services/api/api.controller.ts

@@ -32,7 +32,7 @@ import { LinkedAccountRepository } from '../../repositories/linked-accounts/link
 import { EmailService } from '../email/email.service';
 import { parseAndValidateRequest } from '../auth/auth.utils';
 
-@Controller('/api/v1')
+@Controller('/v1')
 export class ApiController {
   readonly #logger = new Logger();
 

src/services/auth/auth.controller.test.ts

@@ -124,7 +124,7 @@ describe('AuthController', () => {
       const jws = await did.createJWS(payload);
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/did')
+        .post('/v1/auth/login/singleton/did')
         .send(jws)
         .set('Content-Type', 'application/json')
         .set('Accept', 'application/json')
@@ -143,7 +143,7 @@ describe('AuthController', () => {
 
       // Make the request to the endpoint
       return request(app.getHttpServer())
-        .post('/api/v1/auth/request_hive_account')
+        .post('/v1/auth/request_hive_account')
         .send({ username: 'test_user_id' })
         .set('Authorization', 'Bearer <your_mocked_jwt_token>')
         .expect(201)
@@ -163,7 +163,7 @@ describe('AuthController', () => {
 
       // Make the request to the endpoint
       return request(app.getHttpServer())
-        .post('/api/v1/auth/request_hive_account')
+        .post('/v1/auth/request_hive_account')
         .send({ username: 'yeet' })
         .set('Authorization', 'Bearer <your_mocked_jwt_token>')
         .expect(400)
@@ -191,7 +191,7 @@ describe('AuthController', () => {
       }
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/hive')
+        .post('/v1/auth/login/singleton/hive')
         .send(body)
         .expect(201)
         .then(response => {
@@ -214,7 +214,7 @@ describe('AuthController', () => {
       }
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/hive')
+        .post('/v1/auth/login/singleton/hive')
         .send(body)
         .expect(401)
         .then(response => {
@@ -239,7 +239,7 @@ describe('AuthController', () => {
       }
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/hive')
+        .post('/v1/auth/login/singleton/hive')
         .send(body)
         .expect(401)
         .then(response => {
@@ -264,7 +264,7 @@ describe('AuthController', () => {
       }
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/hive')
+        .post('/v1/auth/login/singleton/hive')
         .send(body)
         .expect(401)
         .then(response => {
@@ -287,7 +287,7 @@ describe('AuthController', () => {
       }
 
       return request(app.getHttpServer())
-        .post('/api/v1/auth/login/singleton/hive')
+        .post('/v1/auth/login/singleton/hive')
         .send(body)
         .expect(401)
         .then(response => {

src/services/auth/auth.controller.ts

@@ -44,7 +44,7 @@ import { parseAndValidateRequest } from './auth.utils';
 import { RequestHiveAccountDto } from '../api/dto/RequestHiveAccount.dto';
 import { HiveService } from '../hive/hive.service';
 
-@Controller('/api/v1/auth')
+@Controller('/v1/auth')
 export class AuthController {
   readonly #logger = new Logger(AuthController.name);
 

src/services/auth/auth.module.ts

@@ -46,6 +46,6 @@ import { HiveAccountModule } from '../../repositories/hive-account/hive-account.
 })
 export class AuthModule implements NestModule {
   configure(consumer: MiddlewareConsumer) {
-    consumer.apply(AuthMiddleware).forRoutes('/api/v1/auth/login_singleton/did');
+    consumer.apply(AuthMiddleware).forRoutes('/v1/auth/login_singleton/did');
   }
 }

src/services/email/email.service.ts

@@ -35,7 +35,7 @@ export class EmailService {
     await this.send(
       email,
       'Complete 3Speak registration',
-      `test registration. Click <a href="http://${this.configService.get('PUBLIC_CALLBACK_URL') || 'localhost:4569'}/api/v1/auth/verifyemail?code=${email_code}">here</a> to verify email address.`,
+      `test registration. Click <a href="http://${this.configService.get('PUBLIC_CALLBACK_URL') || 'localhost:4569'}/v1/auth/verifyemail?code=${email_code}">here</a> to verify email address.`,
     );
   }
 }

src/services/storage-cluster/allocator.ts

@@ -293,6 +293,7 @@ export class StorageClusterAllocator extends StorageCluster {
         allocations: {
           id: peerId,
           allocated_at: ts,
+          reported_size: 0,
         },
       },
       $inc: {
@@ -410,6 +411,10 @@ export class StorageClusterAllocator extends StorageCluster {
     const reported_sizes = pinned.allocations
       .map((a) => a.reported_size)
       .filter((size) => size !== null && typeof size !== 'undefined');
+    if (!completedPin.size) {
+      console.error('No size found for pin ' + completedPin.cid);
+      completedPin.size = 0;
+    }
     reported_sizes.push(completedPin.size);
     if (preAllocated === -1) {
       await this.pins.updateOne(
@@ -420,7 +425,7 @@ export class StorageClusterAllocator extends StorageCluster {
               id: peerId,
               allocated_at: msgTs,
               pinned_at: msgTs,
-              reported_size: completedPin.size || 0,
+              reported_size: completedPin.size,
             },
           },
           $inc: {
@@ -510,7 +515,7 @@ export class StorageClusterAllocator extends StorageCluster {
       id: peerId,
       allocated_at: msgTs,
       pinned_at: msgTs,
-      reported_size: newPin.size,
+      reported_size: newPin.size || 0,
     };
     if (!alreadyExists || alreadyExists.status === 'deleted')
       await this.pins.updateOne(
@@ -524,7 +529,7 @@ export class StorageClusterAllocator extends StorageCluster {
             last_updated: msgTs,
             allocations: [newAlloc],
             allocationCount: 1,
-            median_size: newPin.size || 0, // Ensure median_size is of type number
+            median_size: newPin.size, // Ensure median_size is of type number
             metadata: newPin.metadata || {},
           },
         },

src/services/storage-cluster/peer.ts

@@ -232,6 +232,7 @@ export class StorageClusterPeer extends StorageCluster {
               allocations: {
                 id: this.getPeerId(),
                 allocated_at: msgTs,
+                reported_size: 0,
               },
             },
             $inc: {
@@ -348,6 +349,7 @@ export class StorageClusterPeer extends StorageCluster {
       type: SocketMsgTypes.PIN_FAILED,
       data: {
         cid: cid,
+        size: 0,
       },
       ts: failedTs,
     });

src/services/storage-cluster/types.ts

@@ -113,7 +113,7 @@ export interface PinAllocate {
   id: string;
   allocated_at: number;
   pinned_at?: number | undefined;
-  reported_size?: number | undefined;
+  reported_size: number;
 }
 
 export interface Pin {

src/services/uploader/uploading.controller.test.ts

@@ -105,7 +105,7 @@ describe('UploadingController', () => {
         .toBuffer();
 
       return request(app.getHttpServer())
-        .post('/api/v1/upload/thumbnail')
+        .post('/v1/upload/thumbnail')
         .set('Authorization', `Bearer ${jwtToken}`)
         .attach('file', semiTransparentRedPng, 'test-image.png')
         .field('video_id', 'test_video_id')

src/services/uploader/uploading.controller.ts

@@ -34,7 +34,7 @@ MulterModule.registerAsync({
   }),
 });
 
-@Controller('/api/v1/upload')
+@Controller('/v1/upload')
 export class UploadingController {
   readonly #logger = new Logger(UploadingController.name);