Persistent Storage (#13)

.dockerignore

@@ -0,0 +1,8 @@
+**/__pycache__/
+.dockerignore
+.git/
+.github/
+.gitignore
+data/
+docker/
+install.bash

.gitignore

@@ -104,5 +104,6 @@ venv.bak/
 
 # project specific
 client_token
+data/
 twitter_api_tokens.json
 .vscode/

README.md

@@ -25,5 +25,5 @@ Current commands that can be used in Discord:
     !role  - Self-contained role management
 
 ## Docker
-Memebot has a straightforward Docker image that can be build based on the [Dockerfile](./Dockerfile) in this 
+Memebot has a straightforward Docker image that can be build based on the [Dockerfile](./docker/Dockerfile) in this 
 repository. This image can be used for both deployment and testing purposes.

docker/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.8.1-slim
+
+ARG CLIENT_TOKEN_FILE
+ARG TWITTER_API_TOKENS_FILE
+
+ARG DATABASE_URI
+ENV DBURI $DATABASE_URI
+
+# pull in required files
+WORKDIR /home/memebot/
+COPY src src
+COPY requirements.txt requirements.txt
+COPY $CLIENT_TOKEN_FILE client_token
+COPY $TWITTER_API_TOKENS_FILE twitter_api_tokens.json
+
+# set up virtual environment
+ENV VIRTUAL_ENV "/venv"
+ENV PATH "$VIRTUAL_ENV/bin:$PATH"
+
+RUN \
+    apt update -y && \
+    # gcc is required to build package aiohttp (https://docs.aiohttp.org/en/stable/) required by discord.py
+    apt install -y gcc && \
+    python -m venv $VIRTUAL_ENV && \
+    # install dependencies
+    python -m pip install -r requirements.txt
+
+# run memebot
+CMD python3 src/main.py --database-uri ${DBURI}

docker/docker-compose.yaml

@@ -0,0 +1,35 @@
+version: '3.1'
+
+services:
+  bot:
+    container_name: memebot-bot
+    build:
+      dockerfile: docker/Dockerfile
+      context: ..
+      args:
+        CLIENT_TOKEN_FILE: client_token
+        TWITTER_API_TOKENS_FILE: twitter_api_tokens.json
+        DATABASE_URI: "mongodb://db:27017"
+    restart: always
+    depends_on:
+      - db
+    environment:
+      PYTHONUNBUFFERED: 1
+    networks:
+      default:
+  db:
+    container_name: memebot-db
+    image: mongo:4.4.4-bionic
+    restart: always
+    volumes:
+      - ../data/db:/data/db
+      - ../src/config/mongod.yaml:/etc/mongo/mongod.yaml:ro
+    networks:
+      default:
+
+networks:
+  default:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.19.0.0/24

requirements.txt

@@ -1,2 +1,3 @@
 discord.py
+pymongo
 python-twitter

src/commands/__init__.py

@@ -2,6 +2,7 @@
 import os
 from typing import List
 
+import config
 from . import registry, execution
 from .command import Command, CommandOutput
 
@@ -14,8 +15,9 @@ def dynamically_register_commands() -> None:
     def find_and_register_subcommands(top_level_command: Command, cmd_path: List[str]) -> None:
         # Do a depth-first search to register subcommands
         for subcommand in top_level_command.subcommands:
-            registry.register_subcommand(cmd_path, subcommand)
-            find_and_register_subcommands(subcommand, cmd_path + [subcommand.name])
+            if not (subcommand.requires_database and not config.database_enabled):
+                registry.register_subcommand(cmd_path, subcommand)
+                find_and_register_subcommands(subcommand, cmd_path + [subcommand.name])
 
     # Get all the packages located in the command package
     top_level_packages = [f.path for f in os.scandir(os.path.dirname(os.path.realpath(__file__))) if
@@ -31,8 +33,8 @@ def find_and_register_subcommands(top_level_command: Command, cmd_path: List[str
         if issubclass(cmd_class, Command):
             instance = cmd_class()
             # Registration machinery:
-            # If the command is a top-level command
-            if type(cmd_class.parent) is Command:
+            # If the command is a top-level command and meets configuration requirements
+            if type(cmd_class.parent) is Command and not (instance.requires_database and not config.database_enabled):
                 registry.register_top_level_command(instance)
                 find_and_register_subcommands(instance, [instance.name])
 

src/commands/command.py

@@ -48,6 +48,7 @@ def __init__(self, name: str = None, description: str = None, example_args: str
             raise ValueError(f"Every command needs to have a description! ({type(self).__name__})")
         self.description: str = description
         self.example_args: str = example_args
+        self.requires_database: bool = False
 
     def __repr__(self) -> str:
         return f"Command: {type(self).__name__}(name={self.name} description={self.description})"

src/config/__init__.py

@@ -1,9 +1,17 @@
 import argparse
 import pathlib
+import urllib.parse
 
+# Path to the file containing the Discord API token
 discord_api_token: pathlib.Path
+# Path to the JSON file containing the Twitter API tokens
 twitter_api_tokens: pathlib.Path
 
+# Flag which tells if a database connection is enabled
+database_enabled: bool
+# MongoDB URI
+database_uri: urllib.parse.ParseResult
+
 
 def populate_config_from_command_line():
     parser = argparse.ArgumentParser()
@@ -18,12 +26,33 @@ def populate_config_from_command_line():
                         default="twitter_api_tokens.json",
                         type=pathlib.Path)
 
+    # Database Configuration
+    parser.add_argument("--database-enabled",
+                        help="Enable the database connection, and all features which require it.",
+                        dest='database_enabled',
+                        action='store_true')
+    parser.add_argument("--database-disabled",
+                        help="Disable the database connection, and all features which require it.",
+                        dest="database_enabled",
+                        action="store_false")
+    parser.set_defaults(database_enabled=True)
+
+    parser.add_argument("--database-uri",
+                        help="URI of the MongoDB database server",
+                        default=urllib.parse.urlparse("mongodb://127.0.0.1:27017"),
+                        type=urllib.parse.urlparse)
+
     args = parser.parse_args()
 
     global discord_api_token
     global twitter_api_tokens
     discord_api_token = args.discord_api_token
     twitter_api_tokens = args.twitter_api_tokens
 
+    global database_enabled
+    global database_uri
+    database_enabled = args.database_enabled
+    database_uri = args.database_uri
+
 
 populate_config_from_command_line()

src/db/__init__.py

@@ -0,0 +1,17 @@
+import config
+from .internals import DatabaseInternals
+
+db_internals = DatabaseInternals()
+
+if config.database_enabled:
+    db_internals.connect()
+
+
+def test() -> bool:
+    """
+    Functions as a "ping" to the databse to ensure that there is an available connection
+    :return: True if the test succeeds
+    """
+    test_db = db_internals.get_db("test")
+    test_db.list_collection_names()
+    return True

src/db/internals.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+import pymongo as mongo
+import pymongo.database
+
+import config
+
+
+class DatabaseInternals:
+    """
+    Class for managing all database internals that do not need to be exposed to the command programmer.
+    """
+
+    def __init__(self):
+        self.client: Optional[mongo.MongoClient] = None
+
+    def connect(self) -> None:
+        """
+        Create a client connection to a MongoDB database
+        """
+        if self.client is None:
+            self.client = mongo.MongoClient(config.database_uri.geturl())
+
+    def get_db(self, db_name: str) -> mongo.database.Database:
+        return self.client[db_name]

src/memebot.py

@@ -7,6 +7,7 @@
 
 import commands
 import config
+import db
 from lib import constants
 
 
@@ -35,6 +36,8 @@ def __init__(self, **args):
 
         self.twitter_url_pattern = re.compile(r'https:/{2}twitter\.com/([0-9a-zA-Z_]+|i/web)/status/[0-9]+(\?s=\d+)?')
 
+        db.test()
+
     async def on_ready(self) -> None:
         """
         Determines what the bot does as soon as it is logged into discord