diff --git a/empirical_delivery/casync.py b/empirical_delivery/casync.py new file mode 100644 index 0000000..3ec93e5 --- /dev/null +++ b/empirical_delivery/casync.py @@ -0,0 +1,50 @@ +from .empirical_delivery import EmpiricalDelivery +from util.exec import run +from util.index_name import validate_index_name, IncorrectIndexFileName +from typing import List +import logging + + +class Casync(EmpiricalDelivery): + def __init__(self, cache_store: str, index_store: str): + self._cache_store = cache_store + self._index_store = index_store + + def make_chunking( + self, + source, + chunk_size="16:64:256", + ): + try: + self._index_store = validate_index_name(self._index_store, source) + except IncorrectIndexFileName as e: + logging.error( + "Exception when validating index store file name: {}".format(e.message) + ) + exit(1) + run( + [ + "casync", + "make", + self._index_store, + source, + "--chunk-size", + chunk_size, + "--store", + self._cache_store, + ] + ) + + def deliver(self, local_cache_store, output): + run( + [ + "casync", + "extract", + self._index_store, + output, + "--store", + self._cache_store, + "--cache", + local_cache_store, + ] + ) diff --git a/empirical_delivery/desync.py b/empirical_delivery/desync.py new file mode 100644 index 0000000..0120ea5 --- /dev/null +++ b/empirical_delivery/desync.py @@ -0,0 +1,71 @@ +from .empirical_delivery import EmpiricalDelivery +from util.exec import run +from util.index_name import validate_index_name, IncorrectIndexFileName +import os +from typing import List +import logging + + +class Desync(EmpiricalDelivery): + def __init__(self, cache_store: str, index_store: str): + self._cache_store = cache_store + self._index_store = index_store + + def _make_chunking_dir(self, source, chunk_size): + run( + [ + "desync", + "tar", + "-i", + "-s", + self._cache_store, + "-m", + chunk_size, + self._index_store, + source, + ] + ) + + def make_chunking(self, source, chunk_size="16:64:256"): + try: + self._index_store = validate_index_name(self._index_store, source) + except IncorrectIndexFileName as e: + logging.error( + "Exception when validating index store file name: {}".format(e.message) + ) + exit(1) + if os.path.isdir(source): + self._make_chunking_dir(source, chunk_size) + return + run( + [ + "desync", + "make", + "-m", + chunk_size, + "-s", + self._cache_store, + self._index_store, + source, + ] + ) + + def deliver(self, local_cache_store, output): + untar_index_flag = [] + command = "extract" + if self._index_store.endswith(".caidx"): + command = "untar" + untar_index_flag = ["-i", "--no-same-owner"] + run( + [ + "desync", + command, + "-s", + self._cache_store, + "-c", + local_cache_store, + self._index_store, + output, + ] + + untar_index_flag + ) diff --git a/empirical_delivery/empirical_delivery.py b/empirical_delivery/empirical_delivery.py new file mode 100644 index 0000000..92ca88e --- /dev/null +++ b/empirical_delivery/empirical_delivery.py @@ -0,0 +1,10 @@ +from abc import ABC + + +class EmpiricalDelivery(ABC): + + def make_chunking(self, source, chunk_size="16:64:256"): + pass + + def deliver(self, local_cache_store, output): + pass diff --git a/main.py b/main.py new file mode 100644 index 0000000..5951c82 --- /dev/null +++ b/main.py @@ -0,0 +1,52 @@ +from empirical_delivery.desync import Desync +from empirical_delivery.casync import Casync +import argparse + + +def parse_args(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help="subcommand help") + desync_parser = subparsers.add_parser("desync") + desync_parser.add_argument("command") + desync_parser.add_argument("index") + desync_parser.add_argument("source") + desync_parser.add_argument("-s", "--store", required=True) + desync_parser.add_argument("-c", "--cache") + desync_parser.add_argument("-m", "--chunk-size", default="16:64:256") + desync_parser.set_defaults(func=desync) + + casync_parser = subparsers.add_parser("casync") + casync_parser.add_argument("command") + casync_parser.add_argument("index") + casync_parser.add_argument("source") + casync_parser.add_argument("-s", "--store", required=True) + casync_parser.add_argument("-c", "--cache") + casync_parser.add_argument("-m", "--chunk-size", default="16384:65536:262144") + casync_parser.set_defaults(func=casync) + + return parser.parse_args() + + +def desync(args): + desync_tranfer = Desync(args.store, args.index) + if args.command == "make": + desync_tranfer.make_chunking(args.source, args.chunk_size) + elif args.command == "extract": + desync_tranfer.deliver(args.cache, args.source) + + +def casync(args): + casync_tranfer = Casync(args.store, args.index) + if args.command == "make": + casync_tranfer.make_chunking(args.source, args.chunk_size) + elif args.command == "extract": + casync_tranfer.deliver(args.cache, args.source) + + +def main(): + args = parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/util/exec.py b/util/exec.py new file mode 100644 index 0000000..f7129b5 --- /dev/null +++ b/util/exec.py @@ -0,0 +1,17 @@ +import logging +import subprocess +import sys +from typing import List + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + + +def run(command: List[str]): + logger.info(" ".join(command)) + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + logger.error("Return code: {}".format(e.returncode)) + exit(e.returncode) + logger.info("OK") diff --git a/util/index_name.py b/util/index_name.py new file mode 100644 index 0000000..de4808f --- /dev/null +++ b/util/index_name.py @@ -0,0 +1,25 @@ +import logging +import os + + +class IncorrectIndexFileName(ValueError): + def __init__(self, message): + self.message = message + + +def validate_index_name(index_file_name, file): + if os.path.isdir(file): + if index_file_name.endswith(".caibx"): + raise IncorrectIndexFileName( + "{} is directory! Index must be .caidx file".format(file) + ) + elif not index_file_name.endswith(".caidx"): + index_file_name += ".caidx" + else: + if index_file_name.endswith(".caidx"): + raise IncorrectIndexFileName( + "{} is not a directory! Index must be .caibx file".format(file) + ) + elif not index_file_name.endswith(".caibx"): + index_file_name += ".caibx" + return index_file_name