tsu

#!/usr/bin/env python3

# tsu - time series utility

import csv
from collections import namedtuple, OrderedDict
from datetime import datetime, timezone
from functools import partial
from itertools import chain
import json
import os
import time
from urllib.parse import urlparse, urlunparse


class Config:
    data_root_path          = "data"
    default_coll_data_path  = "data/accounts"
    export_outpath          = "export.csv"
    host_replaces           = {}


class Const:
    int_exts            = (".csv", ".tsi")
    float_exts          = (".tsf", )
    str_exts            = (".tss", )
    numeric_types       = [int, float]


# Define and catch our own errors to avoid intercepting language and
# library errors.
class TsuError(Exception):
    pass


class ValidationError(TsuError):
    pass


def warn(template, *args):
    print("WARN: " + template.format(*args))


def parse_timestamp(s):
    try:
        tsi = int(s)
    except ValueError as e:
        raise ValidationError(
            "Timestamp is not an integer: '{}'".format(s)) from e
    try:
        dt = datetime.utcfromtimestamp(tsi)
    except Exception as e:
        raise ValidationError(
            "Cannot parse datetime from timestamp: '{}'".format(tsi)) from e
    return dt


def parse_int(s):
    try:
        return int(s)
    except ValueError as e:
        raise ValidationError(
            "Value is not an integer: '{}'".format(s)) from e


def parse_float(s):
    try:
        return float(s)
    except ValueError as e:
        raise ValidationError(
            "Value is not a float: '{}'".format(s)) from e


def parse_record(rec, valtype):
    if len(rec) != 2:
        raise ValidationError(
            "Record must have exactly 2 fields: " + str(rec))
    s1, s2 = rec

    dt = parse_timestamp(s1)

    if valtype == int:
        val = parse_int(s2)
    elif valtype == float:
        val = parse_float(s2)
    else:
        if len(s2) > 0:
            val = s2
        else:
            raise ValidationError("Value is empty")
    return dt, val


def value_type(path):
    ext = os.path.splitext(path)[1]
    if ext in Const.int_exts:
        return int
    elif ext in Const.float_exts:
        return float
    elif ext in Const.str_exts:
        return str
    else:
        return None


def is_known_type(path):
    return value_type(path) is not None


def is_numeric_type(path):
    return value_type(path) in Const.numeric_types


def csv_iter(path):
    """Generator yielding rows of the csv file."""
    with open(path, newline="") as f:
        yield from csv.reader(f)


def write_csv(path, rows):
    # Overwrite existing file.
    with open(path, "w", newline="") as f:
        writer = csv.writer(f, quoting=csv.QUOTE_MINIMAL, lineterminator="\n")
        writer.writerows(rows)


def print_record(dt, val, valtype):
    if valtype == int:
        print(dt, "{:>6,}".format(val))
    else:
        print(dt, val)


def validate(path, view=False, limit=None):
    valtype = value_type(path)
    prev_dt = datetime.min
    all_rows = list(csv_iter(path))
    first = 0
    rows = all_rows
    if limit is not None:
        first = max(len(all_rows) - limit, 0)
        rows = all_rows[first:]

    # Start counting from 1 to match file line number.
    for line_num, row in enumerate(rows, start=first + 1):
        try:
            dt, val = parse_record(row, valtype)
            if dt <= prev_dt:
                raise ValidationError(
                    "Timestamp '{}' must be greater than '{}'"
                    "".format(dt, prev_dt))
            prev_dt = dt
            if view:
                print_record(dt, val, valtype)
        except ValidationError as e:
            print("Err {}:{}: {}".format(path, line_num, e))


def file_paths_sorted(root, pred, filter_root=True, error=True):
    paths = []
    if os.path.isfile(root):
        # `filter_root=False` allows a uniform interface for handling
        # two different cases. When a caller (indirectly it is the user)
        # _knows_ that root is a file that must be used regardless of
        # its type, predicate check is skipped. However, when root is a
        # directory that must be walked, the caller (user) wants the
        # discovered files to be filtered with pred.
        root_ok = pred(root)
        if (not filter_root) or root_ok:
            paths.append(root)
        if not root_ok:
            warn("Test {} failed for: {}", pred.__name__, root)

    elif os.path.isdir(root):
        for curdir, dirs, files in os.walk(root):
            for fname in files:
                if pred(fname):
                    paths.append(os.path.join(curdir, fname))

    else:
        if error:
            raise TsuError("Path is neither a file nor a directory: " + root)
        else:
            warn("Path is neither a file nor a directory: {}", root)

    return sorted(paths)


def cmd_view(args):
    limit = args.limit
    int_limit = None
    if limit is not None:
        try:
            int_limit = int(limit)
        except ValueError as e:
            raise TsuError("Limit is not an integer: '{}'"
                           "".format(limit)) from e

    root = args.path
    for fp in file_paths_sorted(root, pred=is_known_type, filter_root=False):
        print("----", fp)
        validate(fp, view=True, limit=int_limit)


# Path-related terminology used below is better explained in one place:
#
# - collection - a set of data of (mostly) the same type and structure.
#   Currently, tsu can only handle a collection that is a hierarchy of
#   time series files.
# - rpath - real path to a file or directory, used to read/write data
#   on the file system.
# - icpath - "in-collection path" that is relative to the collection
#   directory and always uses slash `/` as path separator. icpaths are
#   used to derive metadata and also in various lists for
#   mass-processing accounts and metrics. These paths are very local by
#   design for greater flexibility, i.e. they don't hardcode a
#   collection they are included in.
# - account - a group of metrics, plus account-wide metadata common for
#   all metrics.
# - account meta - information that helps to track and process account's
#   metrics, like account's URL, creation date, tags, etc.
# - account path - icpath where account's metrics are stored.
# - metric - a *single* value we track for the account. Keeping each
#   metric in its own time series makes them simple and composable.
# - metric meta - information that helps to work with the metric. Most
#   importantly it is the account the metric belongs to, which in turn
#   allows to load account meta to capture new values.


MetricMeta = namedtuple("MetricMeta", [
    "rpath",
    "name",
    "value_type",
    "platform",
    "account",
    "account_path",
    "account_meta",
])


def coll_env(args, data="none", meta="none"):
    # `data` and `meta` may be "none", "optional", or "required".
    coll_data_dir, coll_meta_path, coll_meta = None, None, None

    if (data == "optional") or (data == "required"):
        if os.path.isdir(args.colldata):
            coll_data_dir = args.colldata
            print("Collection dir exists, treating paths as relative to:",
                  coll_data_dir)
        elif data == "required":
            raise TsuError("Collection data dir is required but '{}' is not a"
                           " directory (see -c)".format(args.colldata))

    if (meta == "optional") or (meta == "required"):
        if args.collmeta:
            coll_meta_path = args.collmeta
        elif coll_data_dir is not None:
            coll_meta_path = coll_data_dir + ".json"
        elif meta == "required":
            raise TsuError("Collection metadata path is required but missing"
                           " (see -c and -m)")

        if coll_meta_path is not None:
            try:
                coll_meta_raw = load_json(coll_meta_path)
                for acc in coll_meta_raw:
                    if "metrics" in acc:
                        # Convert metrics list to a dict.
                        acc["metrics"] = index_list(acc["metrics"], key="name")
                # Convert accounts list to a dict.
                coll_meta = index_list(coll_meta_raw, key="path")
                print("Loaded {} accounts metadata from: {}"
                      "".format(len(coll_meta), coll_meta_path))
            except FileNotFoundError:
                if meta == "required":
                    raise TsuError("Could not load account meta from '{}' (see"
                                   " -m)".format(coll_meta_path))
                elif meta == "optional":
                    warn("Could not load account meta from '{}', hints will be"
                         " limited!", coll_meta_path)

    return coll_data_dir, coll_meta


def cmd_latest(args):
    coll_data_dir, _ = coll_env(args, data="required", meta="none")
    metric_icpaths = load_list(args.metriclist)
    if not metric_icpaths:
        return

    metfmt = "{:20}|{:24}|{:24}"
    headerfmt = metfmt + "|{:20}|{:7}"
    print(headerfmt.format("platform", "account", "metric", "date", "value"))
    print("{:-<20}|{:-<24}|{:-<24}|{:-<20}|{:-<7}".format("", "", "", "", ""))

    for met_icpath in metric_icpaths:
        # Resolve the in-collection path to get the real path.
        met_rpath = os.path.join(coll_data_dir, met_icpath)
        if not os.path.isfile(met_rpath):
            warn("Not a file: {}", met_rpath)
            continue
        rows = list(csv_iter(met_rpath))
        if len(rows) < 1:
            warn("No data: {}", met_rpath)
            continue
        mm = metric_meta(met_rpath, met_icpath)
        last_dt, last_val = parse_record(rows[-1], mm.value_type)
        if mm.platform and mm.account and mm.name:
            metstr = metfmt.format(mm.platform, mm.account, mm.name)
        else:
            metstr = "{:68}".format(met_icpath)
        print("{}|{:20}|{:>7,}".format(metstr, str(last_dt), last_val))


def cmd_list_accounts(args):
    # `data="optional"` allows to derive meta file path from data path.
    _, coll_meta = coll_env(args, data="optional", meta="required")

    for acc_path in coll_meta.keys():
        print(acc_path)


def cmd_validate(args):
    root = args.path
    files_checked = 0
    for fp in file_paths_sorted(root, pred=is_known_type, filter_root=False):
        validate(fp)
        files_checked += 1
    print("Checked {} files".format(files_checked))


def load_list(path):
    with open(path) as f:
        return [line.rstrip("\n") for line in f]


def index_list(list, key):
    res = OrderedDict()
    for item in list:
        ik = item.get(key)
        if ik is None:
            raise ValidationError("Missing value for key '{}' in: {}"
                                  "".format(key, item))
        if ik in res:
            raise ValidationError("Duplicate value '{}' for key '{}'"
                                  "".format(ik, key))
        res[ik] = item
    return res


def metric_meta(rpath, icpath, coll_meta=None):
    # To have `met_name` in more cases, take it from `rpath` and not
    # from `icpath`.
    _dirname, filename = os.path.split(rpath)
    met_name, _ext = os.path.splitext(filename)

    val_type = value_type(rpath)

    # Example `icpath`: `github.com/decred/dcrd/forks.csv`
    # Here `github.com` is the platform, `decred/dcrd` is the account,
    # and `forks` is the metric. Therefore, `icpath` normally has 3
    # parts (minimum), 4 for two-level accounts like on github.com, and
    # possibly more.
    # `icpath` MUST only use `/` path separator and MUST be relative to
    # collection data dir, i.e. must not include any parts above
    # platform, i.e. not be concerned in what collection it is placed.

    plat, acc, acc_path = None, None, None
    if icpath is not None:
        # todo: Test how `/` in metric paths would work on Windows.
        parts = icpath.split("/")
        icpath_valid = len(parts) >= 3
        if icpath_valid:
            plat, *acc_parts, _filename = parts
            acc = "/".join(acc_parts)
            acc_path = plat + "/" + acc
        else:
            warn("Path does not follow the expected structure"
                 " (platform/account/metric.ext): {}", icpath)

    acc_meta = None
    if (coll_meta is not None) and (acc_path is not None):
        acc_meta = coll_meta.get(acc_path)
        if acc_meta is None:
            warn("Account meta not found for acc path: {}", acc_path)

    return MetricMeta(rpath=rpath, name=met_name, value_type=val_type,
                      platform=plat, account=acc, account_path=acc_path,
                      account_meta=acc_meta)


def export_append_rows(rows, met_meta):
    if met_meta.value_type is None:
        warn("Exporting from unknown file type: {}", met_meta.rpath)

    tags = ""
    acc_meta = met_meta.account_meta
    if acc_meta:
        atags = acc_meta.get("tags")
        if atags:
            tags = " ".join(atags)
    else:
        warn("Account meta not found for acc path: {}", met_meta.account_path)

    for ts, val in csv_iter(met_meta.rpath):
        rows.append((ts, met_meta.platform, met_meta.account, met_meta.name,
                     val, tags))


def export_csv(export_icpaths, outpath, coll_data_dir, coll_meta):
    rows = []
    exported_files = 0
    skipped_paths = 0

    for export_icpath in export_icpaths:
        # Resolve the in-collection path.
        rpath = os.path.join(coll_data_dir, export_icpath)
        # If any path explicitly specified by the user is a file, try
        # exporting it regardless of its type. Otherwise, filter the
        # walked (discovered indirectly) paths and only take time series
        # files with numeric values.
        for met_rpath in file_paths_sorted(rpath, pred=is_numeric_type,
                                           filter_root=False, error=False):
            met_icpath = os.path.relpath(met_rpath, coll_data_dir)
            met_meta = metric_meta(met_rpath, met_icpath, coll_meta)
            if met_meta.platform and met_meta.account:
                export_append_rows(rows, met_meta)
                exported_files += 1
            else:
                warn("Unknown platform/account, skipping path: {}", met_rpath)
                skipped_paths += 1

    # In-place sort by timestamp in the first row cell.
    rows.sort()
    print("Exporting {} data points from {} files (skipped {} paths)"
          "".format(len(rows), exported_files, skipped_paths))

    header = ("timestamp", "platform", "account", "metric", "value", "tags")

    # Overwrite existing file.
    write_csv(outpath, chain([header], rows))
    print("File saved:", outpath)


def cmd_export_csv(args):
    # Path walking logic is mostly similar to `cmd_entry()`, with a
    # notable exception that `-c` is always required.
    coll_data_dir, coll_meta = coll_env(args, data="required", meta="required")
    if args.path:
        export_paths = [args.path]
        print("Exporting all metrics found in:", args.path)
    elif args.pathlist:
        export_paths = load_list(args.pathlist)
        print("Exporting {} paths listed in: {}"
              "".format(len(export_paths), args.pathlist))
    else:
        # If all we have is the collection dir, make it the export root.
        export_paths = [""]
        print("Exporting all metrics found in the collection:", coll_data_dir)

    export_csv(export_paths, args.outpath, coll_data_dir, coll_meta)


def load_json(path):
    with open(path) as f:
        return json.load(f)


def replace_host(url):
    # Replace host to an alternative if available.
    pu = urlparse(url)
    if pu.netloc in Config.host_replaces:
        newpu = pu._replace(netloc=Config.host_replaces[pu.netloc])
        newurl = urlunparse(newpu)
        return newurl
    return url


def make_hint(met_meta):
    parts = []
    am = met_meta.account_meta
    if am:
        aname = am.get("name")
        if aname:
            parts.append('"' + aname + '"')
        aurl = am.get("url")
        if aurl:
            parts.append(replace_host(aurl))
    hint = " ".join(parts) if parts else met_meta.rpath
    return hint


class SpecialValue:
    pass


class InputCancel(SpecialValue):
    pass


def optional_input(prompt, input):
    """Read a string, cancel if blank line entered twice.

    Use the passed input function for reading. Return an InputCancel
    instance to signal cancellation by the user.
    """
    try:
        s = input(prompt)
        if s == "":
            s = input("Enter blank again to skip or a value to continue: ")
            if s == "":
                return InputCancel()
        return s
    except EOFError:
        print("(Got EOF)")
        return InputCancel()


class Command(SpecialValue):
    pass


class TimestampCommand(Command):
    pass


def command_input(prompt, input):
    """Read a string and optionally map it to a Command instance.

    Use the passed input function for reading. If the read value starts
    with colon (:), translate it to an appropriate Command instance.
    """
    inp = input(prompt)
    if isinstance(inp, str) and inp.startswith(":"):
        if inp == ":t":
            return TimestampCommand()
        else:
            raise ValidationError("Unknown command '{}'".format(inp))
    else:
        return inp


def confirmed_input(prompt, confirm_prompt, input):
    """Read inputs until two subsequent inputs match.

    Use the passed input function for reading. If the input function
    returns a SpecialValue instance, return it as is without a
    confirmation input.
    """
    inp = input(prompt)
    if isinstance(inp, SpecialValue):
        return inp

    prev = inp
    # Keep collecting inputs until (a) two inputs match OR (b) input is
    # canceled.
    while True:
        cur = input(confirm_prompt)
        if (cur == prev) or isinstance(cur, SpecialValue):
            return cur
        prev = cur


def validated_input(prompt, converter, printer, input):
    while True:
        try:
            inp = input(prompt)
            if isinstance(inp, SpecialValue):
                return inp
            return converter(inp)
        except ValidationError as e:
            printer("Error: {}".format(str(e)))


def parse_date(date_string, format):
    # Extend `datetime.strptime` formats with `%s`, inspired by the
    # `date` program from GNU coreutils.
    if format == "%s":
        return parse_timestamp(date_string)
    else:
        try:
            return datetime.strptime(date_string, format)
        except ValueError as e:
            raise ValidationError(
                "Cannot parse datetime from: '{}'".format(date_string)) from e


def datetime_converter(s):
    dt = None
    for f in ["%d %b %Y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%s"]:
        try:
            dt = parse_date(s, f)
            break
        except ValidationError:
            pass

    if not dt:
        raise ValidationError("Unrecognized date value: " + s)

    return dt


def datetime_greater(other):
    def validator(s):
        dt = datetime_converter(s)
        if dt > other:
            return dt
        else:
            raise ValidationError("Timestamp must be greater than "
                                  + str(other))
    return validator


def make_input(prefix, typedesc, converter, printer):
    # Use function composition to build a powerful input function.
    # Input functions will run (and handle user's input) in the
    # following order: optional_input -> command_input ->
    # validated_input -> confirmed_input.
    cancelable = partial(optional_input, input=input)
    command = partial(command_input, input=cancelable)
    validated = partial(validated_input, converter=converter, printer=printer,
                        input=command)
    prompt = prefix + "Enter {} value: ".format(typedesc)
    confirm_prompt = prefix + "Confirm {} value: ".format(typedesc)
    confirmed = partial(confirmed_input, prompt, confirm_prompt, validated)
    return confirmed


def timestamp_input(prefix, min_ts, printer):
    converter = datetime_greater(min_ts) if min_ts else datetime_converter
    input = make_input(prefix, "timestamp", converter, printer)
    while True:
        inp = input()
        if isinstance(inp, Command):
            printer("Error: commands are not allowed while entering timestamp")
        else:
            return inp


def identity(x):
    return x


def make_prompt_prefix(met_meta):
    prefix = met_meta.name
    if met_meta.account:
        prefix = met_meta.account + "/" + met_meta.name
    return prefix + ": "


def entry_file(met_meta):
    valtype = met_meta.value_type
    # Read all to know the length.
    rows = list(csv_iter(met_meta.rpath))
    last_dt, last_val = None, None
    if len(rows) > 0:
        # Read last value.
        last_dt, last_val = parse_record(rows[-1], valtype)
        # Calculate "freshness" of the last value.
        acc_meta = met_meta.account_meta
        if acc_meta:
            # Allow missing key. Default to update once a day.
            interval = acc_meta.get("update_interval", 86400)
            now_ts = int(datetime.now(timezone.utc).timestamp())
            last_ts = int(last_dt.replace(tzinfo=timezone.utc).timestamp())
            age = now_ts - last_ts
            if age < 0:
                warn("Skipping '{}': Last record timestamp ({}) is in the"
                     " future OR your clock is wrong. Please fix this file"
                     " manually.".format(met_meta.rpath, last_dt))
                return
            elif age < interval:
                print("Skipping '{}': Metric is fresh (updated {:.1f} days ago < {:.1f})"
                      "".format(met_meta.rpath, age / 86400, interval / 86400))
                return

    prompt_prefix = make_prompt_prefix(met_meta)
    printer = lambda s: print(prompt_prefix + s)
    printer("Capture the value for " + make_hint(met_meta))
    if last_dt is not None:
        # Extra output for a sanity check.
        printer("Last record is: time {}, value {}".format(last_dt, last_val))

    if valtype == int:
        input = make_input(prompt_prefix, "integer", parse_int, printer)
    elif valtype == float:
        input = make_input(prompt_prefix, "float", parse_float, printer)
    elif valtype == str:
        input = make_input(prompt_prefix, "string", identity, printer)
    else:
        # Use `Exception` over `TsuError` to force traceback printing.
        raise Exception("Unexpected value type: " + str(valtype))

    dt_custom = None
    while True: # main_entry_loop
        inp = input()
        dt = dt_custom if dt_custom else datetime.utcnow().replace(microsecond=0)
        if isinstance(inp, InputCancel):
            printer("Skipping")
            return
        elif isinstance(inp, TimestampCommand):
            dt_inp = timestamp_input(prompt_prefix, last_dt, printer)
            if isinstance(dt_inp, datetime):
                printer("Using timestamp {} for current metric only"
                        "".format(dt_inp))
                dt_custom = dt_inp
            elif isinstance(dt_inp, InputCancel):
                printer("Timestamp entry canceled")
        elif isinstance(inp, SpecialValue):
            # Use `Exception` over `TsuError` to force a traceback.
            raise Exception("Unexpected SpecialValue: " + str(inp))
        else:
            val = inp
            break # main_entry_loop

    if valtype == int and last_val:
        delta = " ({:+})".format(val - last_val)
    else:
        delta = ""

    ts = int(dt.replace(tzinfo=timezone.utc).timestamp())
    rows.append((ts, val))
    write_csv(met_meta.rpath, rows)
    printer("Saved: time {}, value {}{}".format(dt, val, delta))


def entry_path(path, coll_data_dir, coll_meta):
    if coll_data_dir is None:
        rpath = path
    else:
        # Resolve the in-collection path.
        rpath = os.path.join(coll_data_dir, path)

    # Unlike the other places, here we DO filter out the root path.
    # If it is a file, and not of a numeric type, we won't know how to
    # properly add data to it.
    for met_rpath in file_paths_sorted(rpath, pred=is_numeric_type,
                                       filter_root=True, error=False):
        if coll_data_dir is None:
            met_icpath = None
        else:
            met_icpath = os.path.relpath(met_rpath, coll_data_dir)
        met_meta = metric_meta(met_rpath, met_icpath, coll_meta)

        acc_enabled, met_enabled = True, True
        acc_meta = met_meta.account_meta
        if acc_meta:
            # Allow missing key. Update is enabled by default.
            acc_enabled = acc_meta.get("update_enabled", True)
            if acc_enabled:
                acc_metrics = acc_meta.get("metrics")
                if acc_metrics:
                    acc_met_meta = acc_metrics.get(met_meta.name)
                    if acc_met_meta:
                        # Allow missing key. Update is enabled by default.
                        met_enabled = acc_met_meta.get("update_enabled", True)

        if acc_enabled and met_enabled:
            entry_file(met_meta)
        else:
            level = "Account" if (not acc_enabled) else "Metric"
            print("Skipping '{}': {} update disabled".format(met_rpath, level))


def cmd_entry(args):
    """Enter data manually in an interactive session.

    You will be prompted to enter data points one by one for each
    applicable file. After collecting the input value, UTC timestamp
    will be generated and appended to the end of file together with the
    new value. Make sure your system clock is accurate.

    To protect from errors, you will be prompted to enter each value
    twice.

    To skip entering current value, enter a blank line twice or hit
    Ctrl-D (Ctl-Z+Return on Windows).

    To enter the timestamp manually, enter ':t' instead of the value.

    To quit the data entry session, hit Ctrl-C.
    """
    # Use `None` as a special value to say we don't know where the
    # collection starts, can't do any path math, and have to trust that
    # the user knows correct paths and can survive without (some) hints.
    coll_data_dir, coll_meta = None, None

    # Only process `-c` and `-m` if the `path` arg is NOT set. In other
    # words, `path` arg disables the effects of `-c` and `-m`.
    if args.path is None:
        coll_data_dir, coll_meta = coll_env(args,
                                            data="optional",
                                            meta="optional")

    if args.path:
        paths = [args.path]
        print("Walking real path '{}', hints disabled".format(args.path))
    elif args.pathlist:
        # `pathlist` items MUST be relative to the collection data dir.
        if coll_data_dir is None:
            raise TsuError("pathlist requires a valid collection data dir (see"
                           " -c)")
        paths = load_list(args.pathlist)
        print("Walking {} paths listed in: {}"
              "".format(len(paths), args.pathlist))
    elif coll_data_dir is not None:
        # If all we have is the collection dir, make it the entry root.
        paths = [""]
        print("Walking collection dir:", coll_data_dir)
    else:
        raise TsuError("I don't know what path(s) to walk. Set the path arg,"
                       " or -p, or -c.")

    replaces = args.replaces
    if replaces:
        if not len(replaces) % 2 == 0:
            raise TsuError("List of host replaces must have an even length.")
        it = iter(replaces)
        try:
            while True:
                k, v = next(it), next(it)
                Config.host_replaces[k] = v
        except StopIteration:
            pass

    print("Interactive data entry mode.")
    print("Make sure your system time is accurate (UTC):",
          format(datetime.utcfromtimestamp(int(time.time()))))

    for path in paths:
        entry_path(path, coll_data_dir, coll_meta)


def make_arg_parser():
    import argparse

    file_types_str = ", ".join(Const.int_exts +
                               Const.float_exts +
                               Const.str_exts)
    parser = argparse.ArgumentParser(
        description="Time series utility. Supported file types: "
                     + file_types_str)

    # `-c` and `-m` mean that only ONE collection is handled at a time.

    parser.add_argument(
        "-c", "--colldata",
        default=Config.default_coll_data_path,
        help="Path to collection data directory. When this dir exists, it"
             " changes how paths are treated by some commands. (default: {})"
             "".format(Config.default_coll_data_path))

    parser.add_argument(
        "-m", "--collmeta",
        help="Path to collection metadata JSON file (default is -c plus a"
             " .json extension)")

    subparsers = parser.add_subparsers(dest="command", title="commands")

    listaccs = subparsers.add_parser(
        "listaccs", aliases=["la"],
        help="List accounts in the JSON collection metadata file. -m is"
             " required if its default does not exist.")
    listaccs.set_defaults(func=cmd_list_accounts)

    validate = subparsers.add_parser(
        "validate", aliases=["val"],
        help="Validate time series files")
    validate.add_argument(
        "path", nargs="?",
        default=Config.data_root_path,
        help="Path to search time series files (default: {})"
             "".format(Config.data_root_path))
    validate.set_defaults(func=cmd_validate)

    view = subparsers.add_parser(
        "view", aliases=["v"],
        help="View one or more time series files")
    view.add_argument(
        "path",
        help="Path to view. If directory, view all files of known types")
    view.add_argument(
        "-l", "--limit",
        help="Max number of records to output for each file")
    view.set_defaults(func=cmd_view)

    latest = subparsers.add_parser(
        "latest",
        help="Print latest values for metrics listed in a file. -c is required"
             " if its default does not exist.")
    latest.add_argument(
        "metriclist",
        help="List file with metrics to report, one per line. Paths must be"
             " files and must be *relative* to the collection data dir (-c).")
    latest.set_defaults(func=cmd_latest)

    export = subparsers.add_parser(
        "export",
        help="Export data into a single CSV file. Both -c and -m are required"
             " if their defaults do not exist. If neither 'path' nor -p are"
             " set, -c will be attempted as a root dir to walk.",
        description="Export data from arbitrary tree of time series files into"
                     " a single file, overwriting it.")
    export_src = export.add_mutually_exclusive_group(required=False)
    export_src.add_argument(
        "path", nargs="?",
        help="Path to export data points from. It may be file or directory and"
             " must be *relative* to the collection data dir (-c).")
    export_src.add_argument(
        "-p", "--pathlist",
        help="List file with paths to export data points from, one per line."
             " Paths may be files or directories and must be *relative* to the"
             " collection data dir (-c).")
    export.add_argument(
        "--outpath",
        default=Config.export_outpath,
        help="Output path to save (default: {})".format(Config.export_outpath))
    export.set_defaults(func=cmd_export_csv)

    entry = subparsers.add_parser(
        "entry", aliases=["e"],
        help="Enter data manually. If -c is in effect (i.e. exists and not"
             " disabled by the 'path' arg), it changes path computation and"
             " entry hints. If -m exists, it improves hints. If neither 'path'"
             " nor -p are set, -c will be attempted as a root dir to walk.",
        description=cmd_entry.__doc__)
    entry_target = entry.add_mutually_exclusive_group(required=False)
    entry_target.add_argument(
        "path", nargs="?",
        help="File or directory path to enter data in. If directory, the"
             " program will prompt for data entry for each (numeric) time"
             " series file found in it. The path must be a real path (NOT"
             " relative to -c) and if set, it *disables* the -c path math and"
             " entry hints.")
    entry_target.add_argument(
        "-p", "--pathlist",
        help="List file with paths to enter data in, one path per line."
             " Paths may be files or directories and must be *relative* to the"
             " collection data dir (-c).")
    entry.add_argument(
        "-r", "--replaces", nargs="*",
        metavar="HOST",
        help="Host replacement pairs separated by space")
    entry.set_defaults(func=cmd_entry)

    return parser


def main():
    parser = make_arg_parser()
    args = parser.parse_args()

    if args.command:
        try:
            args.func(args)
        except (TsuError, FileNotFoundError) as e:
            print("Error:", e)
        except KeyboardInterrupt:
            print("\nAborting")
        except BrokenPipeError:
            # Silence error when e.g. piping into `less` and quitting
            # it before consuming everything from the pipe.
            pass
    else:
        parser.print_usage()


if __name__ == "__main__":
    main()