Empirical search for the optimal way

empirical_delivery/casync.py

@@ -0,0 +1,50 @@
+from .empirical_delivery import EmpiricalDelivery
+from util.exec import run
+from util.index_name import validate_index_name, IncorrectIndexFileName
+from typing import List
+import logging
+
+
+class Casync(EmpiricalDelivery):
+    def __init__(self, cache_store: str, index_store: str):
+        self._cache_store = cache_store
+        self._index_store = index_store
+
+    def make_chunking(
+        self,
+        source,
+        chunk_size="16:64:256",
+    ):
+        try:
+            self._index_store = validate_index_name(self._index_store, source)
+        except IncorrectIndexFileName as e:
+            logging.error(
+                "Exception when validating index store file name: {}".format(e.message)
+            )
+            exit(1)
+        run(
+            [
+                "casync",
+                "make",
+                self._index_store,
+                source,
+                "--chunk-size",
+                chunk_size,
+                "--store",
+                self._cache_store,
+            ]
+        )
+
+    def deliver(self, local_cache_store, output):
+        run(
+            [
+                "casync",
+                "extract",
+                self._index_store,
+                output,
+                "--store",
+                self._cache_store,
+                "--cache",
+                local_cache_store,
+            ]
+        )

empirical_delivery/desync.py

@@ -0,0 +1,71 @@
+from .empirical_delivery import EmpiricalDelivery
+from util.exec import run
+from util.index_name import validate_index_name, IncorrectIndexFileName
+import os
+from typing import List
+import logging
+
+
+class Desync(EmpiricalDelivery):
+    def __init__(self, cache_store: str, index_store: str):
+        self._cache_store = cache_store
+        self._index_store = index_store
+
+    def _make_chunking_dir(self, source, chunk_size):
+        run(
+            [
+                "desync",
+                "tar",
+                "-i",
+                "-s",
+                self._cache_store,
+                "-m",
+                chunk_size,
+                self._index_store,
+                source,
+            ]
+        )
+
+    def make_chunking(self, source, chunk_size="16:64:256"):
+        try:
+            self._index_store = validate_index_name(self._index_store, source)
+        except IncorrectIndexFileName as e:
+            logging.error(
+                "Exception when validating index store file name: {}".format(e.message)
+            )
+            exit(1)
+        if os.path.isdir(source):
+            self._make_chunking_dir(source, chunk_size)
+            return
+        run(
+            [
+                "desync",
+                "make",
+                "-m",
+                chunk_size,
+                "-s",
+                self._cache_store,
+                self._index_store,
+                source,
+            ]
+        )
+
+    def deliver(self, local_cache_store, output):
+        untar_index_flag = []
+        command = "extract"
+        if self._index_store.endswith(".caidx"):
+            command = "untar"
+            untar_index_flag = ["-i", "--no-same-owner"]
+        run(
+            [
+                "desync",
+                command,
+                "-s",
+                self._cache_store,
+                "-c",
+                local_cache_store,
+                self._index_store,
+                output,
+            ]
+            + untar_index_flag
+        )

empirical_delivery/empirical_delivery.py

@@ -0,0 +1,10 @@
+from abc import ABC
+
+
+class EmpiricalDelivery(ABC):
+
+    def make_chunking(self, source, chunk_size="16:64:256"):
+        pass
+
+    def deliver(self, local_cache_store, output):
+        pass

main.py

@@ -0,0 +1,52 @@
+from empirical_delivery.desync import Desync
+from empirical_delivery.casync import Casync
+import argparse
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help="subcommand help")
+    desync_parser = subparsers.add_parser("desync")
+    desync_parser.add_argument("command")
+    desync_parser.add_argument("index")
+    desync_parser.add_argument("source")
+    desync_parser.add_argument("-s", "--store", required=True)
+    desync_parser.add_argument("-c", "--cache")
+    desync_parser.add_argument("-m", "--chunk-size", default="16:64:256")
+    desync_parser.set_defaults(func=desync)
+
+    casync_parser = subparsers.add_parser("casync")
+    casync_parser.add_argument("command")
+    casync_parser.add_argument("index")
+    casync_parser.add_argument("source")
+    casync_parser.add_argument("-s", "--store", required=True)
+    casync_parser.add_argument("-c", "--cache")
+    casync_parser.add_argument("-m", "--chunk-size", default="16384:65536:262144")
+    casync_parser.set_defaults(func=casync)
+
+    return parser.parse_args()
+
+
+def desync(args):
+    desync_tranfer = Desync(args.store, args.index)
+    if args.command == "make":
+        desync_tranfer.make_chunking(args.source, args.chunk_size)
+    elif args.command == "extract":
+        desync_tranfer.deliver(args.cache, args.source)
+
+
+def casync(args):
+    casync_tranfer = Casync(args.store, args.index)
+    if args.command == "make":
+        casync_tranfer.make_chunking(args.source, args.chunk_size)
+    elif args.command == "extract":
+        casync_tranfer.deliver(args.cache, args.source)
+
+
+def main():
+    args = parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

util/exec.py

@@ -0,0 +1,17 @@
+import logging
+import subprocess
+import sys
+from typing import List
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+
+def run(command: List[str]):
+    logger.info(" ".join(command))
+    try:
+        subprocess.check_output(command)
+    except subprocess.CalledProcessError as e:
+        logger.error("Return code: {}".format(e.returncode))
+        exit(e.returncode)
+    logger.info("OK")

util/index_name.py

@@ -0,0 +1,25 @@
+import logging
+import os
+
+
+class IncorrectIndexFileName(ValueError):
+    def __init__(self, message):
+        self.message = message
+
+
+def validate_index_name(index_file_name, file):
+    if os.path.isdir(file):
+        if index_file_name.endswith(".caibx"):
+            raise IncorrectIndexFileName(
+                "{} is directory! Index must be .caidx file".format(file)
+            )
+        elif not index_file_name.endswith(".caidx"):
+            index_file_name += ".caidx"
+    else:
+        if index_file_name.endswith(".caidx"):
+            raise IncorrectIndexFileName(
+                "{} is not a directory! Index must be .caibx file".format(file)
+            )
+        elif not index_file_name.endswith(".caibx"):
+            index_file_name += ".caibx"
+    return index_file_name