Webdriver (ThePornDatabase#34)

* initial end-to-end webdriver test

Co-authored-by: 4c0d3r <nope>

.github/workflows/ci.yml

@@ -40,8 +40,21 @@ jobs:
           elif [ "$RUNNER_OS" == "macOS" ]; then
               env HOMEBREW_NO_AUTO_UPDATE=1 brew install ffmpeg
           fi
-        shell: bash 
+        shell: bash
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+          node-version: '16'
+
+      - name: Install Node packages
+        run: yarn install
+
+      - name: Build Assets
+        run: yarn run build
+
       - run: poetry install
+
       - run: poetry run pytest
 
   linting:
@@ -93,10 +106,25 @@ jobs:
       - name: install poetry
         uses: abatilo/[email protected]        
         with:
-          poetry-version: '1.1.13'  
+          poetry-version: '1.1.13'
+
       - run: poetry install
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+          node-version: '16'
+
+      - name: Install Node packages
+        run: yarn install
+
+      - name: Build Assets
+        run: yarn run build
+
       - run: poetry run pytest --cov
+
       - run: poetry run coverage xml
+
       - uses: codecov/codecov-action@v2
         with:
           files: ./coverage.xml

.gitignore

@@ -12,5 +12,6 @@ creds.sh
 init
 .python-version
 node_modules
+.env
 namer/web/public/assets
 namer/web/templates

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:impish as base
+FROM ubuntu:jammy as base
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -22,12 +22,25 @@ RUN apt-get update \
        systemd-sysv \
        python3-pip \
        python3-dev \
-       python3.9-venv \
+       python3.10-venv \
        curl \
+       wget \
+       gnupg2 \
+       xvfb \
     && rm -rf /var/lib/apt/lists/* \
     && rm -Rf /usr/share/doc && rm -Rf /usr/share/man \
     && apt-get clean
 
+ENV DISPLAY=:99
+ARG CHROME_VERSION="google-chrome-stable"
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
+  && apt-get update -qqy \
+  && apt-get -qqy install \
+    ${CHROME_VERSION:-google-chrome-stable} \
+  && rm /etc/apt/sources.list.d/google-chrome.list \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN curl -fsSL https://deb.nodesource.com/setup_lts.x | bash -
 RUN apt-get install -y --no-install-recommends \
@@ -39,16 +52,16 @@ RUN npm install --global yarn
 
 RUN mkdir /work/
 COPY . /work
-RUN cd /work/ \
-    && export PATH="/root/.local/bin:$PATH" \
-    && rm -rf /work/namer/__pycache__/ || true \
+ENV PATH="/root/.local/bin:$PATH"
+WORKDIR /work
+RUN rm -rf /work/namer/__pycache__/ || true \
     && rm -rf /work/test/__pycache__/ || true \
     && poetry install \
-    && poetry run pytest \
-    && poetry run flakeheaven lint \
     && yarn install \
     && yarn run build \
+    && poetry run flakeheaven lint \
     && poetry build
+RUN  ( Xvfb :99 & cd /work/ && poetry run pytest )
 
 FROM base
 COPY --from=build /work/dist/namer-*.tar.gz /
@@ -66,4 +79,3 @@ ENV GIT_HASH=$GIT_HASH
 ENV PROJECT_VERSION=$PROJECT_VERSION
 EXPOSE 6980
 ENTRYPOINT ["python3", "-m", "namer", "watchdog"]
-

docker_build.sh

@@ -2,11 +2,11 @@
 BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 CLEAN=$(git diff-index --quiet HEAD; echo $?)
 version=$(cat pyproject.toml | grep -m1 "version = " | sed 's/.* = //' | sed 's/"//g')
-if [[ "${CLEAN}" == "0" ]]; then
+#if [[ "${CLEAN}" == "0" ]]; then
   export BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
   export GIT_HASH=$(git rev-parse --verify HEAD)
   export repo=theporndatabase
   docker build . --build-arg "BUILD_DATE=${BUILD_DATE}" --build-arg "GIT_HASH=${GIT_HASH}" --build-arg "PROJECT_VERSION=${version}" -t "ghcr.io/${repo}/namer:${version}" -t "ghcr.io/${repo}/namer:latest"
-else
-  echo Not building for dirty code.
-fi
+#else
+#  echo Not building for dirty code.
+#fi

namer/watchdog.py

@@ -129,30 +129,54 @@ def __processing_thread(self):
         self.__command_queue.task_done()
 
     def __init__(self, namer_config: NamerConfig):
+        self.__started = False
+        self.__stopped = False
         self.__namer_config = namer_config
         self.__src_path = namer_config.watch_dir
         self.__event_observer = PollingObserver()
         self.__webserver: Optional[WebServer] = None
         self.__command_queue: Queue = Queue()
         self.__worker_thread: Thread = Thread(target=self.__processing_thread, daemon=True)
         self.__event_handler = MovieEventHandler(namer_config, self.__command_queue)
+        self.__background_thread: Optional[Thread] = None
 
     def run(self):
         """
         Checks for new files in 3 second intervals,
         needed if running in docker as events aren't properly passed in.
         """
-        self.start()
-        if self.__namer_config.web is True:
-            self.__webserver = WebServer(self.__namer_config, command_queue=self.__command_queue)
-            if self.__webserver:
+        if not self.__started:
+            self.__starting = True
+            self.start()
+            if self.__namer_config.web is True:
+                self.__webserver = WebServer(self.__namer_config, command_queue=self.__command_queue)
                 self.__webserver.start()
-        try:
-            while True:
-                schedule.run_pending()
-                time.sleep(3)
-        except KeyboardInterrupt:
-            self.stop()
+            try:
+                while True and not self.__stopped:
+                    schedule.run_pending()
+                    time.sleep(3)
+                self.stop()
+            except KeyboardInterrupt:
+                self.stop()
+            self.__started = False
+            self.__stopped = False
+
+    def __enter__(self):
+        self.__background_thread = Thread(target=self.run)
+        self.__background_thread.start()
+        tries = 0
+        while self.get_web_port() is None and tries < 20:
+            time.sleep(0.2)
+            tries += 1
+        if self.get_web_port is None:
+            raise RuntimeError("application did not get assigned a port within 4 seconds.")
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.stop()
+        if self.__background_thread is not None:
+            self.__background_thread.join()
+            self.__background_thread = None
 
     def start(self):
         """
@@ -186,14 +210,20 @@ def stop(self):
         stops a background thread to check for files.
         Waits for the event processor to complete.
         """
-        logger.info("exiting")
-        self.__event_observer.stop()
-        self.__event_observer.join()
+        if not self.__stopped:
+            self.__stopped = True
+            logger.info("exiting")
+            self.__event_observer.stop()
+            self.__event_observer.join()
+            if self.__webserver is not None:
+                self.__webserver.stop()
+            self.__command_queue.put(None)
+            self.__command_queue.join()
+            logger.info("exited")
+
+    def get_web_port(self) -> Optional[int]:
         if self.__webserver is not None:
-            self.__webserver.stop()
-        self.__command_queue.put(None)
-        self.__command_queue.join()
-        logger.info("exited")
+            return self.__webserver.get_effective_port()
 
     def __schedule(self):
         self.__event_observer.schedule(self.__event_handler, str(self.__src_path), recursive=True)

namer/web/actions.py

@@ -101,12 +101,11 @@ def delete_file(file_name_str: str, config: NamerConfig) -> bool:
 
 
 def read_failed_log_file(name: str, config: NamerConfig) -> str:
-    name = Path(name)
-    file_name = config.failed_dir / f'{name.stem}_namer.log'
+    file_name = config.failed_dir / name
+    file_name = file_name.parent / (file_name.stem + '_namer.log')
     data = ''
     if file_name.is_file():
         data = file_name.read_text('UTF-8')
-
     return data
 
 

namer/web/server.py

@@ -3,7 +3,7 @@
 """
 from queue import Queue
 from threading import Thread
-from typing import Union
+from typing import Optional, Union
 
 from flask import Flask
 from flask_compress import Compress
@@ -60,3 +60,9 @@ def stop(self):
         """
         if self.__server:
             self.__server.close()
+
+    def get_effective_port(self) -> Optional[int]:
+        port = None
+        if hasattr(self.__server, "effective_port"):
+            port = self.__server.effective_port
+        return port