Version 2.0

Added nox
Added Threading
Adding Linting & Typing

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+per-file-ignores = tests/*:S101
+application-import-names = src,tests
+import-order-style = google
+select = B,B9,BLK,C,E,F,I,S,W, D, A
+ignore = E203,E501,W503,ANN101,D100,D107,D401,S404,S607,S603
+max-line-length = 120

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/type-lint.yml

@@ -0,0 +1,27 @@
+name: "Typing and Linting"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes nox
+    - name: Initialize Nox
+      uses: excitedleigh/[email protected]
+
+    - name: Runs nox
+      run: nox

.gitignore

@@ -3,3 +3,5 @@
 *.iml
 out
 gen
+
+__pycache__

.mypy.ini

@@ -0,0 +1,4 @@
+[mypy]
+
+[mypy-nox.*,pytest,pyfilemovr.*]
+ignore_missing_imports = True

dev-requirements.txt

@@ -0,0 +1 @@
+nox~=2021.10.1

noxfile.py

@@ -0,0 +1,38 @@
+import nox
+from nox.sessions import Session
+
+locations = "pyfilemovr", "run.py", "noxfile.py"
+python_versions = ["3.9"]
+nox.options.sessions = "lint", "mypy"
+
+
+@nox.session(python=python_versions)
+def lint(session: Session) -> None:
+    """Lint code using flake8."""
+    args = session.posargs or locations
+    session.install(
+        "flake8",
+        "flake8-annotations",
+        "flake8-bandit",
+        "flake8-black",
+        "flake8-bugbear",
+        "flake8-docstrings",
+        "flake8-import-order",
+    )
+    session.run("flake8", *args)
+
+
+@nox.session(python=python_versions)
+def black(session: Session) -> None:
+    """Format code using black."""
+    args = session.posargs or locations
+    session.install("black")
+    session.run("black", *args)
+
+
+@nox.session(python=python_versions)
+def mypy(session: Session) -> None:
+    """Check typing with mypy."""
+    args = session.posargs or locations
+    session.install("mypy")
+    session.run("mypy", *args)

pyfilemovr/pyfilemovr.py

@@ -0,0 +1,115 @@
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+import hashlib
+import os
+import shutil
+from typing import Any, Generator, List, Optional, Union
+
+
+class PyFileMovr:
+    """Main Application."""
+
+    def __init__(
+        self,
+        file_input: str,
+        output: str,
+        extension: Optional[str],
+        duplicate: Optional[bool],
+        debug: Optional[bool],
+    ) -> None:
+        self.input = file_input
+        self.output = output
+        self.extension = extension
+
+        if duplicate is None:
+            self.duplicate = False
+        else:
+            self.duplicate = duplicate
+
+        if debug is None:
+            self.debug = False
+        else:
+            self.debug = debug
+
+        if duplicate is True:
+            self.hash_list: List[str] = []
+
+    @staticmethod
+    def walk_through_files(path: str, file_extension: Optional[str]) -> Generator:
+        """Walk through all files in all sub-dirs."""
+        try:
+            for (dirpath, _, filenames) in os.walk(path):
+                for filename in filenames:
+                    if file_extension is not None:
+                        if filename.endswith(file_extension):
+                            yield os.path.join(dirpath, filename)
+                    else:
+                        yield os.path.join(dirpath, filename)
+        except Exception as e:
+            print("Error:" + str(e))
+
+    @staticmethod
+    def get_date_time() -> str:
+        """Get current date_time."""
+        now = datetime.now()
+        dt = now.strftime("%d/%m/%Y %H:%M:%S")
+        return dt
+
+    @staticmethod
+    def hash_byte_str_iter(bytes_iter: Union[Any]) -> str:
+        """Get hash hex from string."""
+        hl = hashlib.sha256()
+        for block in bytes_iter:
+            hl.update(block)
+        return hl.hexdigest()
+
+    @staticmethod
+    def file_as_block_iter(file: Any, block_size: int = 65536) -> Generator:
+        """Get hash from file in blocks."""
+        with file:
+            block = file.read(block_size)
+            while len(block) > 0:
+                yield block
+                block = file.read(block_size)
+
+    def move_file(self, file: str) -> None:
+        """Move files to output directory."""
+        try:
+            shutil.move(file, self.output)
+        except shutil.Error as err:
+            print("Error:" + str(err))
+
+    def handle_queue(self, queue: List) -> None:
+        """Handle the queue using threads."""
+        with ThreadPoolExecutor(max_workers=12) as worker:
+            worker.map(self.move_file, queue)
+
+    def run(self) -> None:
+        """Run Application."""
+        print(
+            "PyFileMovr - Created by: math280h - Found at: https://github.com/math280h/PyFileMovr\n"
+        )
+
+        if self.debug:
+            print("Input path:", self.input, " Output path:", self.output, "\n")
+
+        queue: List[str] = []
+
+        for file in self.walk_through_files(self.input, file_extension=self.extension):
+            print("Current file: {}".format(file))
+            if self.debug:
+                print("     Hash List:", self.hash_list)
+
+            if self.duplicate is True:
+                fh = self.hash_byte_str_iter(self.file_as_block_iter(open(file, "rb")))
+                if self.debug:
+                    print("     File hash:", fh)
+                if fh not in self.hash_list:
+                    self.hash_list.append(fh)
+                else:
+                    print("     Skipped: Duplicate")
+                    continue
+            queue.append(file)
+            print("     Successfully moved file")
+
+        self.handle_queue(queue)

run.py

@@ -0,0 +1,38 @@
+import argparse
+
+from pyfilemovr.pyfilemovr import PyFileMovr
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Move all files from one destination to the other"
+    )
+    parser.add_argument(
+        "-i", "--input", type=str, help="Destination to move files from"
+    )
+    parser.add_argument("-o", "--output", type=str, help="Destination to move files to")
+    parser.add_argument(
+        "-e",
+        "--extension",
+        type=str,
+        help="Only move files with this extension (Default: *)",
+    )
+    parser.add_argument(
+        "-d",
+        "--duplicates",
+        type=bool,
+        help="If true all file hashes will be compared and only the "
+        "first in a series of duplicates will be moved ("
+        "Default: False)",
+    )
+    parser.add_argument(
+        "--debug", type=bool, help="Toggles debug mode (Default: False)"
+    )
+    args = parser.parse_args()
+
+    if args.input is None or args.output is None:
+        exit(1)
+
+    app = PyFileMovr(
+        args.input, args.output, args.extension, args.duplicates, args.debug
+    )
+    app.run()