move epy_conv to cli, and reorganize conversion utilities in epygram.…

…formats.conversion

pyproject.toml

@@ -53,6 +53,7 @@ epy_ddhlfa_plot = "epygram.cli.ddhlfa_plot:main"
 epy_domain_maker = "epygram.cli.domain_maker:main"
 epy_what_the_grib = "epygram.cli.what_the_grib:main"
 epy_fa_sp2gp = "epygram.cli.fa_sp2gp:main"
+epy_convert = "epygram.cli.convert:main"
 
 [tool.setuptools]
 include-package-data = true

src/epygram/cli/__init__.py

@@ -32,7 +32,7 @@ def get_args():
                                      epilog='End of help for: %(prog)s (EPyGrAM-' + epygram.__version__ + ')')
     parser.add_argument('command',
             help=' '.join(['Command to be executed: will call the "main" function from epygram.cli.<command> module.',
-                           'Available commands are: {}.'.format(commands),
-                           'Each command is then auto-documented with option -h.']))
+                           'Each command is then auto-documented: `epygram <command> -h`.']),
+            choices=commands)
     return parser.parse_args()
 

src/epygram/cli/convert.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) Météo France (2014-)
+# This software is governed by the CeCILL-C license under French law.
+# http://www.cecill.info
+
+import argparse
+
+import footprints
+from bronx.syntax.parsing import str2dict
+
+import epygram
+from epygram.extra import griberies
+from epygram.formats.conversion.functions import batch_convert
+from .args_catalog import (add_arg_to_parser,
+                                      files_management, fields_management,
+                                      misc_options, runtime_options,
+                                      operational_options, output_options)
+
+epylog = footprints.loggers.getLogger(__name__)
+
+
+def main():
+    epygram.init_env()
+    args = get_args()
+    if args.verbose:
+        epylog.setLevel('INFO')
+    else:
+        epylog.setLevel('WARNING')
+    batch_convert(args.filenames,
+                  args.output_format,
+                  # technical
+                  threads_number=args.threads_number,
+                  progressmode=args.progressmode,
+                  # instructions common to formats
+                  fieldseed=args.fieldseed,
+                  subzone=args.subzone,
+                  grib_short_fid=args.grib_short_fid,
+                  # GRIB specifics
+                  suite=args.suite,
+                  typeofgeneratingprocess=args.typeOfGeneratingProcess,
+                  default_packing=args.default_packing,
+                  other_grib_options=args.other_GRIB_options,
+                  # netCDF specifics
+                  compression=args.nc_comp,
+                  flatten_horizontal_grids=args.flatten,
+                  # GeoPoints specifics
+                  order=args.order,
+                  lonlat_precision=args.lonlat_precision,
+                  precision=args.precision,
+                  llv=args.llv,
+                  columns=args.geopoints_cols,
+                  no_header=args.geopoints_noheader
+                  )
+
+
+def get_args():
+
+    # 1. Parse arguments
+    ####################
+    parser = argparse.ArgumentParser(description='An EPyGrAM tool for converting file formats. \
+                                                  Spectral fields are converted into gridpoints.',
+                                     epilog='End of help for: %(prog)s (EPyGrAM-' + epygram.__version__ + ')')
+
+    add_arg_to_parser(parser, files_management['several_files'])
+    flds = parser.add_mutually_exclusive_group()
+    add_arg_to_parser(flds, fields_management['field'])
+    add_arg_to_parser(flds, fields_management['list_of_fields'])
+    add_arg_to_parser(parser, misc_options['LAMzone'])
+    add_arg_to_parser(parser, output_options['output_format'])
+    # compression/precision options
+    add_arg_to_parser(parser, fields_management['netCDF_compression'])
+    add_arg_to_parser(parser, fields_management['GRIB2_packing'])
+    add_arg_to_parser(parser, output_options['GeoPoints_lonlat_precision'])
+    add_arg_to_parser(parser, output_options['GeoPoints_precision'])
+    # GRIB specifics
+    add_arg_to_parser(parser, operational_options['suite'])
+    add_arg_to_parser(parser, operational_options['typeOfGeneratingProcess'])
+    add_arg_to_parser(parser, operational_options['numod'])
+    add_arg_to_parser(parser, output_options['GRIB_other_options'])
+    # GeoPoints specific
+    add_arg_to_parser(parser, misc_options['array_flattening_order'])
+    cols = parser.add_mutually_exclusive_group()
+    add_arg_to_parser(cols, output_options['GeoPoints_llv'])
+    add_arg_to_parser(cols, output_options['GeoPoints_columns'])
+    add_arg_to_parser(parser, output_options['GeoPoints_noheader'])
+    # netCDF
+    add_arg_to_parser(parser, misc_options['flatten_horizontal_grids'])
+    # others
+    add_arg_to_parser(parser, output_options['GRIB_short_fid'])
+    add_arg_to_parser(parser, runtime_options['threads_number'])
+    status = parser.add_mutually_exclusive_group()
+    add_arg_to_parser(status, runtime_options['verbose'])
+    add_arg_to_parser(status, runtime_options['percentage'])
+
+    args = parser.parse_args()
+
+    # 2. Initializations
+    ####################
+
+    # 2.1 options
+    if args.zone in ('C', 'CI'):
+        args.subzone = args.zone
+    else:
+        args.subzone = None
+    if args.GRIB2_packing is not None:
+        args.default_packing = str2dict(args.GRIB2_packing, try_convert=int)
+    else:
+        args.default_packing = griberies.defaults.GRIB2_keyvalue[5]
+    if args.GRIB_other_options is not None:
+        args.other_GRIB_options = str2dict(args.GRIB_other_options, try_convert=int)
+    else:
+        args.other_GRIB_options = {}
+    if args.numod is not None:
+        args.other_GRIB_options['generatingProcessIdentifier'] = args.numod
+    if args.geopoints_cols is not None:
+        args.args.geopoints_cols = [c.strip() for c in args.geopoints_cols.split(',')]
+    assert args.filenames != [], \
+           "must supply one or several filenames."
+    args.threads_number = min(args.threads_number, len(args.filenames))
+    args.progressmode = None
+    if args.verbose:
+        args.progressmode = 'verbose'
+    elif args.percentage:
+        if threads_number > 1:
+            args.progressmode = None
+        else:
+            args.progressmode = 'percentage'
+
+    # 2.2 list of fields to be processed
+    if args.field is not None:
+        args.fieldseed = [args.field]
+    elif args.listoffields is not None:
+        listfile = epygram.containers.File(filename=args.listoffields)
+        with open(listfile.abspath, 'r') as listfile:
+            args.fieldseed = [line.replace('\n', '').strip() for line in listfile.readlines()]
+    else:
+        args.fieldseed = None
+    return args
+

src/epygram/formats/__init__.py

@@ -22,7 +22,7 @@
 from footprints import proxy as fpx
 from bronx.system.unistd import stderr_redirected
 
-from epygram import config, epygramError
+from .. import config, epygramError
 from . import fafields
 
 __all__ = []
@@ -131,40 +131,5 @@ def resource(filename, openmode, fmt=None, **kwargs):
     return fpx.dataformat(filename=filename, openmode=openmode, format=fmt,
                           **kwargs)
 
+from . import conversion
 
-def fid_converter(initial_fid, initial_fmt, target_fmt,
-                  grib_short_fid=False):
-    """
-    Creates and returns the fid in format *target_fmt* from an *initial_fid* in
-    *initial_fmt*.
-
-    *grib_short_fid* condense GRIB fid in string.
-    """
-    if initial_fmt == 'generic' and target_fmt == 'GRIB2':
-        target_fid = copy.copy(initial_fid)
-    elif initial_fmt == 'GRIB' and target_fmt in ('netCDF', 'GeoPoints'):
-        # TODO: ? this is very basic !
-        if grib_short_fid:
-            target_fid = '-'.join([str(initial_fid[k])
-                                   for k in sorted(initial_fid.keys())])
-        else:
-            target_fid = str(initial_fid).replace(' ', '').replace("'", "").replace("{", "_")
-            """
-            FIXME: doesn't work
-            try:
-                from .GRIB import namesgribdef
-                fid = copy.copy(initial_fid)
-                fid.pop('name', None)
-                fid.pop('shortName', None)
-                fid.pop('editionNumber', None)
-                fid.pop('tablesVersion', None)
-                cfVarName = namesgribdef.cfVarName(fid,
-                                                   'grib{}'.format(initial_fid['editionNumber']))
-                if len(cfVarName) == 1:
-                    target_fid = list(cfVarName.keys())[0]
-            except Exception:
-                pass"""
-    else:
-        raise NotImplementedError("this kind of conversion.")
-
-    return target_fid

src/epygram/formats/conversion/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) Météo France (2014-)
+# This software is governed by the CeCILL-C license under French law.
+# http://www.cecill.info
+
+from . import functions
+from . import converters
+