WIP: Attempt to download all URLs found in messages using youtube-dl

.gitignore

@@ -60,7 +60,7 @@ typings/
 
 # dotenv environment variables file
 .env
-.env.test
+.env.*
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -88,4 +88,7 @@ dist
 out
 
 # Runtime artifacts
-*.mp4
+*.mp4
+
+# AWS build artifacts
+.aws-sam/

.vscode/extensions.json

@@ -1,3 +1,7 @@
 {
-  "recommendations": ["ms-azuretools.vscode-azurefunctions"]
+  "recommendations": [
+    "dbaeumer.vscode-eslint",
+    "github.vscode-pull-request-github",
+    "esbenp.prettier-vscode"
+  ]
 }

Dockerfile

@@ -0,0 +1,24 @@
+# Start with the most recent AWS Lambda Nodejs version
+FROM public.ecr.aws/lambda/nodejs:20
+
+# Add yarn
+RUN npm i -g yarn && npm cache clean --force
+
+# Add Python 3 & other dependencies
+RUN dnf install python3 openssl -y && dnf clean all && rm -rf /var/cache/yum
+
+# Add ffmpeg
+COPY ffmpeg/bin/ffmpeg ${LAMBDA_TASK_ROOT}/ffmpeg/bin/ffmpeg
+
+# Install dependencies with yarn
+COPY package.json yarn.lock ${LAMBDA_TASK_ROOT}/
+RUN yarn install --frozen-lockfile --production && yarn cache clean
+# For some reason the lambda function doesn't have access to the files otherwise:
+RUN chmod -R 755 ${LAMBDA_TASK_ROOT}/*
+
+# Copy function code
+COPY src/ ${LAMBDA_TASK_ROOT}/src
+RUN chmod -R 755 ${LAMBDA_TASK_ROOT}/src/*
+
+# Workaround for https://github.com/aws/aws-lambda-base-images/issues/137
+ENV LD_LIBRARY_PATH=""

README.md

@@ -1,10 +1,14 @@
 # VReddit Telegram Bot
 
-A telegram bot which listens for v.redd.it URLs and replies with the video file (including audio)
+A telegram bot which listens for video URLs and replies with the video file (including audio)
+
+Supports v.redd.it and some other video platforms.
 
 ## Usage
 
-Simply add [@vreddit_bot](https://t.me/vreddit_bot) to any group, or send it a private message containing a v.redd.it link.
+To use the live bot, add [@vreddit_bot](https://t.me/vreddit_bot) to any group, or send it a private message containing a video link.
+
+To run locally, see the **Development** section of this readme.
 
 ## TO DO
 
@@ -18,16 +22,16 @@ Simply add [@vreddit_bot](https://t.me/vreddit_bot) to any group, or send it a p
 - [x] Support reddit links
 - [x] Add reddit post title as video caption
 - [x] Include link to source as [inline keyboard](https://core.telegram.org/bots/2-0-intro#new-inline-keyboards)
+- [x] Use youtube-dl to download videos so that more sites are supported
+- [ ] Improve youtube-dl output message
+- [ ] If v.redd.it video is > 50 mb, try to use lower quality stream
 - [ ] If forwarding to the bot from a group chat, give a button to send the video back to that chat (e.g. via inline)
-- [ ] If video is > 50 mb, try to use lower quality stream
 - [ ] Reply to /help with a short text about what the bot can do
 - [ ] Stop the bot sometimes asking for location info when using inline mode
-- [ ] Use youtube-dl to add support for youtube & many other sites
-  - Check output for "ERROR: Unsupported URL: https://example.com"
-  - Probably need to add FFmpeg to PATH
-  - Format opts (in config file?): `-f 'bestvideo[ext=mp4][filesize<?45M]+bestaudio[ext=m4a][filesize<?5M]/best[ext=mp4][filesize<?50M]'`
-  - Note that for v.redd.it filesize is not known so we still need to check size of output
 - [ ] Use streamable.com for videos between 50-500MB? (max 720p & 10min)
+- [ ] /debug command which behaves as normal but enables verbose logging.
+- [ ] /video command for group chats which enables the log message for that link despite it being a group
+- [ ] /gif command which does video-only, and /audio which does audio-only
 
 ### Implementation details:
 
@@ -36,16 +40,13 @@ Simply add [@vreddit_bot](https://t.me/vreddit_bot) to any group, or send it a p
 - [x] Rename repo & npm package to vreddit-bot
 - [x] Increase code coverage / add badges
 - [x] Local dev server
-- [ ] Use async file operations & fix concurrency problems (globally unique file name?)
+- [x] Use async file operations & fix concurrency problems (globally unique file name?)
+- [x] Try other hosting options to see if it's faster and/or cheaper:
+  - [x] AWS Lambda --> was much faster & cheaper than Azure!
+- [ ] Swap CI from Azure to AWS
 - [ ] CI: Optimise so that we don't run checks twice on releases?
-- [ ] Set up [git-lfs](https://git-lfs.github.com/) to work with husky. See also: [1], [2], [3]
-- [ ] Tune Azure max workers param
-- [ ] Try other hosting options to see if it's faster and/or cheaper:
-  - [ ] Azure x64 Windows host
-  - [ ] Azure Linux host
-  - [ ] AWS Lambda
-  - [ ] Google Cloud Functions
 - [ ] Collect stats on inline option chosen?
+- [ ] Some kind of solution for when more than 20 seconds are needed to download a file?
 
 [1]: https://dev.to/mbelsky/pair-husky-with-git-lfs-in-your-javascript-project-2kh0
 [2]: https://github.com/typicode/husky/issues/108
@@ -60,42 +61,83 @@ Simply add [@vreddit_bot](https://t.me/vreddit_bot) to any group, or send it a p
    - Git
    - Node.js v14 (it must be 14 to match the version in Azure/AWS).  
       _Tip: You can install & manage multiple Node versions using tools like [nodist](https://github.com/nullivex/nodist) (Windows) or [n](https://github.com/tj/n) (Linux/MacOS/WSL)_
-   - [Yarn](https://yarnpkg.com/)
+   - [Yarn](https://yarnpkg.com/) (`npm install -g yarn`)
+   - Python (any version, but it must be on your `$PATH` as `python`. If it's called `python3` it won't be found by youtube-dl.)
 
 1. `git clone` the repo and `cd` into it
 
 1. Run `yarn install` to install all dependencies
 
 1. Create a `.env` file in the root of the project with the following params:
 
-   ```properties
-   BOT_ERROR_CHAT_ID=<your telegram chat ID>
+   ```bash
    BOT_API_TOKEN=<your personal dev bot API token>
+   BOT_ERROR_CHAT_ID=<your telegram chat ID>  # optional
+   DOWNLOAD_TIMEOUT=300  # optional (default = no timeout)
    ```
 
-   - If you don't konw your telegram chat ID, don't worry. Just set it to 0 and update it later once you've found out your ID from the bot logs.
-   - If you don't have a spare bot you can use for local development, make a new bot using the botfather. Don't try use a bot that you are already using for something else, it won't work.
-
 1. Run `yarn dev` to start local dev server. You can now message your dev bot in telegram to test your code.
 
-### Common Commands
+### Useful Commands
 
 ```sh
-# Run a local bot server connected to the bot configured in the `.env` file:
+# Run a local bot server (configured in .env), restarts any time you save a source code file:
 yarn dev
 
 # Run all tests in watch mode:
 yarn test:watch
 
 # Auto fix lint/formatting issues (where possible):
 yarn lint:fix
+```
+
+### Testing the DynamoDB video info caching
+
+For normal development purposes video caching is disabled, since that reduces the amount of setup needed, plus it's helpful for testing to _not_ have a cache.
+
+However, sometimes you might want to specifically test the caching logic locally.
+
+In that case you will need:
+
+- an AWS account
+- AWS credentials saved in your `~/.aws/credentials` file (see [Loading credentials in Node.js from the shared credentials file](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html))
+- a DynamoDB table with a primary key called `url` of type `String`.
+
+Then, add these properties to your `.env` file:
+
+```properties
+AWS_PROFILE=<(optional) credentials profile e.g. personal>
+AWS_REGION=<your DynamoDB table region e.g. eu-central-1>
+CACHE_TABLE_NAME=<your DynamoDB table name e.g. video-info-cache-dev>
+```
+
+Now run `yarn dev` as normal, and the cache table you specified will be used.
+
+### Manual deployment to staging
+
+1. Create a new file `.env.staging`, with the following content:
+
+```properties
+AWS_PROFILE=<(optional) credentials profile e.g. personal-account>
+AWS_REGION=eu-central-1
+CACHE_TABLE_NAME=video-info-cache-staging
+
+BOT_API_TOKEN=<staging bot API token>
+BOT_ERROR_CHAT_ID=<your telegram chat id>
 
-# Run the Azure function locally in watch mode:
-yarn start
+SAM_ENV=staging
 ```
 
+2. Install the AWS SAM CLI
+3. Run `yarn deploy`
+4. If needed, update the webook URL by running `npx env-cmd -f .env.staging set-webhook <URL>`
+
 ### CI/CD
 
 1. Open a **pull request** to run linting & tests
 1. Push to the **master branch** (e.g. merge a PR) to deploy to **staging** ([@staging_vreddit_bot](https://t.me/staging_vreddit_bot))
 1. Create a tag by running **`yarn release`** to deploy to **prod** ([@vreddit_bot](https://t.me/vreddit_bot))
+
+```
+
+```

events/message.json

@@ -0,0 +1,3 @@
+{
+  "body": "{\"update_id\":1,\"message\":{\"text\":\"https://v.redd.it/hf352syjjka61\",\"entities\":[{\"offset\":0,\"length\":31,\"type\":\"url\"}],\"chat\":{\"id\":60764253,\"first_name\":\"Dave\",\"last_name\":\"Rolle\",\"username\":\"DavidRolle\",\"is_bot\":false,\"language_code\":\"en\",\"type\":\"private\"},\"message_id\":42}}"
+}

package.json

@@ -6,37 +6,53 @@
     "name": "David Drem Rolle",
     "email": "[email protected]"
   },
+  "engines": {
+    "node": "^16 || ^18 || ^20"
+  },
   "license": "UNLICENSED",
   "private": "true",
+  "files": [
+    "src",
+    "ffmpeg"
+  ],
   "scripts": {
-    "dev": "nodemon -x env-cmd start-dev-server",
+    "dev": "nodemon -x env-cmd start-dev-server src/webhook.webhook",
     "start": "env-cmd func start",
     "lint": "eslint \"**/*.js\" && prettier -c \"**/*.{js,css,md,json}\" --ignore-path .gitignore",
     "lint:fix": "eslint --fix \"**/*.js\" && prettier -w \"**/*.{js,css,md,json}\" --ignore-path .gitignore",
     "test": "jest --ci --no-cache",
     "test:watch": "jest --watchAll --runInBand --no-cache",
     "test:update": "env-cmd jest -u",
     "update-deps": "ncu -u -x husky --doctor && yarn lint:fix && ncu -u -t minor --doctor && yarn lint",
-    "release": "ncu -e2 -t minor && npx np --no-publish"
+    "release": "ncu -e2 -t minor && npx np --no-publish",
+    "deploy": "sam build && env-cmd -f .env.staging npm run sam-deploy",
+    "deploy-prod": "sam build && env-cmd -f .env.prod npm run sam-deploy",
+    "sam-deploy": "sam deploy --config-env $SAM_ENV --parameter-overrides BotApiToken=$BOT_API_TOKEN BotErrorChatId=$BOT_ERROR_CHAT_ID CacheTableName=$CACHE_TABLE_NAME",
+    "logs": "env-cmd -f .env.staging npm run sam-logs",
+    "logs-prod": "env-cmd -f .env.prod npm run sam-logs",
+    "sam-logs": "sam logs --config-env $SAM_ENV -t"
   },
   "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.34.0",
+    "filenamify": "^4.3.0",
     "form-data": "^4.0.0",
-    "node-fetch": "^2.6.2",
-    "serverless-telegram": "^0.6.0"
+    "he": "^1.2.0",
+    "node-fetch": "^2.6.5",
+    "serverless-telegram": "^0.8.3",
+    "youtube-dl-exec": "^2.4.15"
   },
   "devDependencies": {
-    "@types/jest": "^27.0.1",
+    "@types/jest": "^27.0.2",
     "env-cmd": "^10.1.0",
     "eslint": "^7.32.0",
-    "eslint-plugin-jest": "^24.4.0",
-    "filenamify": "^4.3.0",
+    "eslint-plugin-jest": "^24.4.2",
     "husky": "^3.1.0",
-    "jest": "^27.1.1",
+    "jest": "^27.2.2",
     "lint-staged": "^11.1.2",
     "nock": "^13.1.3",
-    "nodemon": "^2.0.12",
+    "nodemon": "^2.0.13",
     "npm-check-updates": "^11.8.5",
-    "prettier": "^2.4.0"
+    "prettier": "^2.4.1"
   },
   "prettier": {
     "singleQuote": true,

samconfig.toml

@@ -0,0 +1,41 @@
+version = 0.1
+[default]
+[default.global.parameters]
+stack_name = "vreddit-bot-dev"
+[default.deploy]
+[default.deploy.parameters]
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-o69yjxke7qem"
+s3_prefix = "vreddit-bot-dev"
+region = "eu-central-1"
+capabilities = "CAPABILITY_IAM"
+image_repositories = ["WebhookFunction=192461062174.dkr.ecr.eu-central-1.amazonaws.com/vredditbotfe383b7b/webhookfunction2cdba7a2repo"]
+[default.logs.parameters]
+region = "eu-central-1"
+
+[staging]
+[staging.global.parameters]
+stack_name = "vreddit-bot-staging"
+[staging.deploy]
+[staging.deploy.parameters]
+stack_name = "vreddit-bot-staging"
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-o69yjxke7qem"
+s3_prefix = "vreddit-bot-staging"
+region = "eu-central-1"
+capabilities = "CAPABILITY_IAM"
+image_repositories = ["WebhookFunction=192461062174.dkr.ecr.eu-central-1.amazonaws.com/vredditbotfe383b7b/webhookfunction2cdba7a2repo"]
+[staging.logs.parameters]
+region = "eu-central-1"
+
+[prod]
+[prod.global.parameters]
+stack_name = "vreddit-bot"
+[prod.deploy]
+[prod.deploy.parameters]
+stack_name = "vreddit-bot"
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-o69yjxke7qem"
+s3_prefix = "vreddit-bot"
+region = "eu-central-1"
+capabilities = "CAPABILITY_IAM"
+image_repositories = ["WebhookFunction=192461062174.dkr.ecr.eu-central-1.amazonaws.com/vredditbotfe383b7b/webhookfunction2cdba7a2repo"]
+[prod.logs.parameters]
+region = "eu-central-1"