api: Add export and transcode to Asset and Task (#893)

* added export * added generic handler to handle export * added scheduleTask - moved pub msgs to scheduleTask * changed ipfs - removed waiting from outside scheduler * parentAssetId to inputAssetId - outputAssetId * typo * fix playbackId length and prefix * removed task name - remove originTaskId - changed scheduler to support both output and input assets * schema: Update export task params+output schema * api: Fixes to the export asset API E2E tested, it works!! https://ipfs.io/ipfs/QmRpyvxmxWPUUNSDXodjqDVutsq1ubH9sc85B57Eo7tB75 * api: Add download URL to assets * api: Add dynamic IPFS URLs to export task outputs * added ingest to webhook * added live-to-vod-asset * same shardKey as stream - better types on shard schedule * same shardKey as stream - better types on shard schedule * api/task: Use our dedicated IPFS gateway * fixed signed url encoding * added origin to ingests * check if asset is ready before export * check if asset is ready before export * api/cannon: Remove redundant stream/session fetch * removed redundant mp4 record url from webhook cannon * removed baseIngest from cannon * baseIngest remove * api/store: Make cleanWriteOnlyResponse recursive * api: Avoid creating fields that dont exist * api: Remove transcode task type for now * api: Fix asset creation on request upload * api: Clean asset responses as well * api: Create type for new asset * api/scheduler: Fix task not getting updated * api: Do some oneOf magic on export-task-params * api: Fix asset subfield clean up * api: Improve task error handling - Set asset as failed in case upload fails - Do not allow upload if asset has already been uploaded - Handle corner cases on task event result procecssing * api: List assets and tasks by updatedAt desc * api: Some final nits * api/task: Set IPFS URLs on regular list as well * prettier Co-authored-by: Victor Elias <[email protected]> Co-authored-by: Victor Elias <[email protected]>
livepeer · Feb 10, 2022 · f72f964 · f72f964
1 parent 97a3496
commit f72f964
Show file tree

Hide file tree

Showing 14 changed files with 785 additions and 471 deletions.
diff --git a/packages/api/.env b/packages/api/.env
@@ -3,7 +3,7 @@ LP_FALLBACK_PROXY=http://localhost:3008
 LP_JWT_SECRET=secretsecret
 LP_JWT_AUDIENCE=livepeer
 LP_CLIENT_ID=211814034878-r7iugl9pa9fo2shd2pvmmadjatd5o4af.apps.googleusercontent.com
-LP_INGEST=[{"ingest":"rtmp://localhost/live","playback":"http://localhost/hls","base":"https://localhost"}]
+LP_INGEST=[{"ingest":"rtmp://localhost/live","playback":"http://localhost/hls","base":"http://localhost","origin":"http://localhost:3004"}]
 LP_POSTGRES_URL=postgresql://[email protected]:5432/livepeerapi
 LP_AMQP_URL=amqp://localhost:5672/livepeer
 LP_RECAPTCHA_SECRET_KEY=secretsecret
diff --git a/packages/api/src/app-router.ts b/packages/api/src/app-router.ts
@@ -16,7 +16,7 @@ import apiProxy from "./controllers/api-proxy";
 import proxy from "http-proxy-middleware";
 import { getBroadcasterHandler } from "./controllers/broadcaster";
 import WebhookCannon from "./webhooks/cannon";
-import TaskScheduler from "./task/task";
+import TaskScheduler from "./task/scheduler";
 import Queue, { NoopQueue, RabbitQueue } from "./store/queue";
 import Stripe from "stripe";
 import { CliArgs } from "./parse-cli";
@@ -84,13 +84,21 @@ export default async function makeApp(params: CliArgs) {
     ? await RabbitQueue.connect(amqpUrl)
     : new NoopQueue();
 
+  // Task Scheduler
+  const taskScheduler = new TaskScheduler({
+    queue,
+  });
+  await taskScheduler.start();
+
   // Webhooks Cannon
   const webhookCannon = new WebhookCannon({
     db,
     store,
     frontendDomain,
     sendgridTemplateId,
     sendgridApiKey,
+    taskScheduler,
+    vodObjectStoreId,
     supportAddr,
     verifyUrls: true,
     queue,
@@ -102,12 +110,6 @@ export default async function makeApp(params: CliArgs) {
     webhookCannon.stop();
   });
 
-  // Task Scheduler
-  const taskScheduler = new TaskScheduler({
-    queue,
-  });
-  await taskScheduler.start();
-
   process.on("beforeExit", (code) => {
     queue.close();
     taskScheduler.stop();
@@ -137,6 +139,7 @@ export default async function makeApp(params: CliArgs) {
     req.config = params;
     req.frontendDomain = frontendDomain; // defaults to livepeer.com
     req.queue = queue;
+    req.taskScheduler = taskScheduler;
     req.stripe = stripe;
     next();
   });
@@ -216,7 +219,6 @@ export default async function makeApp(params: CliArgs) {
   return {
     router: app,
     webhookCannon,
-    taskScheduler,
     store,
     db,
     queue,