Merge branch 'develop' into feature/modernisation_nearest

definitions/boot.def

@@ -12,7 +12,8 @@ constant internalVersion=30 : hidden;
 meta checkInternalVersion check_internal_version(internalVersion) : hidden;
 
 # ECC-806: Local concepts precedence order
-transient preferLocalConcepts = 0 : hidden;
+preferLocalConceptsEnvVar = getenv("ECCODES_GRIB_PREFER_LOCAL_CONCEPTS","0") : hidden;
+transient preferLocalConcepts = preferLocalConceptsEnvVar : hidden;
 
 constant defaultTypeOfLevel="unknown" : hidden;
 

definitions/create_def.pl

@@ -38,7 +38,8 @@ sub create_cfName {
     attribute.id=grib.attribute_id and
     centre.id=grib_encoding.centre_id and
     units.id=param.units_id and
-    param.id=cf.grib1_ecmwf order by
+    param.id=cf.grib1_ecmwf and
+    grib_encoding.is_legacy=0 order by
     edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -98,6 +99,83 @@ sub create_cfName {
     close(TAR);
 }
 
+sub create_cfName_legacy {
+    my $p; my %seen; 
+    my ($key) = "cfName";
+    my $field = "cf.name";
+
+    my $query= <<"EOF";
+    select $field,force128,edition,
+    centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName from
+    param,grib_encoding,grib,attribute,centre,units,cf where
+    param.hide_def=0 and
+#    param.retired=0 and
+    grib_encoding.id=grib.encoding_id and
+    param.id=grib_encoding.param_id and
+    attribute.id=grib.attribute_id and
+    centre.id=grib_encoding.centre_id and
+    units.id=param.units_id and
+    param.id=cf.grib1_ecmwf and
+    grib_encoding.is_legacy=1 order by
+    edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
+EOF
+
+    my $qh=$dbh->prepare($query);
+    $qh->execute();
+
+    # file containing the list of grib api parameters files we want to tar and 
+    # distribute to users for them to download and update their list of parameter
+    # to the latest
+    #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
+    #$tarfilesflag=1;
+
+    while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
+    {
+        if ($centre eq "wmo" ) { $conceptDir=""; }
+        else { $conceptDir="/localConcepts/$centre"; }
+
+        if ($filebase ne "$basedir/grib$edition$conceptDir") {
+            if ($filebase) {
+                print $out "}\n";
+                close $out;
+            }
+            $filebase="$basedir/grib$edition$conceptDir";
+            mkpath($filebase);
+
+            print TAR "grib$edition$conceptDir/$key.legacy.def\n";
+            #system("cp -f $filebase/$key.def $filebase/$key.def.orig");
+            open($out,"> $filebase/$key.legacy.def")
+                or die "unable to open $filebase/$key.legacy.def";
+            print $out "# Automatically generated by $0, do not edit\n";
+            $p=();
+        }
+        if ($p ne $paramId || exists($seen{$attribute}) ) {
+           if ($p) { print $out "\t}\n"; }
+           print $out "#$name\n" ;
+           print $out "\'".$keyval."\' = {\n" ;
+           $p=$paramId;
+           %seen=();
+        }
+        $seen{$attribute}=1;
+        print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
+        # we need to allow strings in the attribute_value field 
+        # for the moment we apply a patch here
+        if ($attribute =~ /stepType/ ) {
+            $value="\"accum\"";
+        }
+        if ($value eq '') {
+            $value="missing()";
+        }
+        print $out "\t $attribute = $value ;\n" ;
+    }
+    if ($filebase) {
+       print $out "}\n";
+       close $out;
+    }
+
+    close(TAR);
+}
+
 sub create_def {
     my $p; my %seen; 
     my ($key) =@_;
@@ -117,7 +195,8 @@ sub create_def {
         param.id=grib_encoding.param_id and
         attribute.id=grib.attribute_id and
         centre.id=grib_encoding.centre_id and
-        units.id=param.units_id
+        units.id=param.units_id and
+        grib_encoding.is_legacy=0
         order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -183,29 +262,27 @@ sub create_def {
     close(TAR);
 }
 
-# See ECC-1886
-sub create_cfVarName {
+sub create_def_legacy {
     my $p; my %seen; 
     my ($key) =@_;
     my $field=$key;
 
-    #if ($key =~ /paramId/) { $field="param.id"; } 
-    #if ($key =~ /name/) { $field="param.name"; } 
-    #if ($key =~ /units/) { $field="units.name"; } 
-    if ($key =~ /cfVarName/) { $field="cfVarName"; }
+    if ($key =~ /paramId/) { $field="param.id"; } 
+    if ($key =~ /name/) { $field="param.name"; } 
+    if ($key =~ /units/) { $field="units.name"; } 
 
     my $query= <<"EOF";
         select $field,force128,edition,
         centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
         from param,grib_encoding,grib,attribute,centre,units where
         param.hide_def=0 and
-        param.retired=0 and
+#        param.retired=0 and
         grib_encoding.id=grib.encoding_id and
         param.id=grib_encoding.param_id and
         attribute.id=grib.attribute_id and
         centre.id=grib_encoding.centre_id and
-        units.id=param.units_id
-        and cfVarName IS NOT NULL
+        units.id=param.units_id and
+        grib_encoding.is_legacy=1
         order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -237,10 +314,10 @@ sub create_cfVarName {
             #copy("$filebase/$key.def","$filebase/$key.def.bkp") 
             #  or die ("unable to copy $filebase/$key.def");
 
-            print TAR "grib$edition$conceptDir/$key.def\n";
+            print TAR "grib$edition$conceptDir/$key.legacy.def\n";
             #system("cp -f $filebase/$key.def $filebase/$key.def.orig");
-            open($out,"> $filebase/$key.def")
-                or die "unable to open $filebase/$key.def";
+            open($out,"> $filebase/$key.legacy.def")
+                or die "unable to open $filebase/$key.legacy.def";
             print $out "# Automatically generated by $0, do not edit\n";
             $p=();
         }
@@ -270,7 +347,7 @@ sub create_cfVarName {
 
     close(TAR);
 }
-
+ 
 sub create_paramId_def {
     my $p; my %seen; 
 
@@ -281,7 +358,8 @@ sub create_paramId_def {
     grib_encoding.id=grib.encoding_id and
     param.id=grib_encoding.param_id and
     attribute.id=grib.attribute_id and
-    centre.id=grib_encoding.centre_id
+    centre.id=grib_encoding.centre_id and
+    grib_encoding.is_legacy=0
     order by edition,centre_id,param.o,param.id,attribute.o";
     my $qh=$dbh->prepare($query);
     $qh->execute();
@@ -355,13 +433,15 @@ sub create_def_old {
 }
 
 create_def("paramId");
+create_def_legacy("paramId");
 create_def("shortName");
+create_def_legacy("shortName");
 create_def("name");
+create_def_legacy("name");
 create_def("units");
-# Note: The cfVarName.def files are smaller than the rest. We only store the
-#       minimum set necessary. See ECC-1886
-create_cfVarName("cfVarName");
+create_def_legacy("units");
 create_cfName("cfName");
+create_cfName_legacy("cfName");
 
 # #create_paramId_def();
 

definitions/grib1/local.98.16.def

@@ -29,7 +29,7 @@ transient hourOfEndOfOverallTimeInterval=23;
 transient minuteOfEndOfOverallTimeInterval=59;
 transient secondOfEndOfOverallTimeInterval=59;
 
-transient indicatorOfUnitForTimeRange=3;
+transient indicatorOfUnitForTimeRange=3; # month
 transient lengthOfTimeRange=1;
 unsigned[1] averagingPeriod : dump ;
 

definitions/grib1/local.98.19.def

@@ -12,6 +12,7 @@ alias perturbationNumber=number;
 
 unsigned[1] ensembleSize  : dump;
 alias totalNumber=ensembleSize;
+alias numberOfForecastsInEnsemble=ensembleSize;
 
 meta quantile sprintf("%s:%s",number,ensembleSize);
 

definitions/grib1/section.1.def

@@ -334,3 +334,11 @@ meta md5Product md5(offsetSection1,section1Length,gridDefinition,section1Flags,d
 
 # ECC-1806
 concept_nofail paramIdForConversion(zero, "paramIdForConversion.def", conceptsDir2, conceptsDir1) : long_type,read_only;
+
+# ECC-1954
+if (productDefinitionTemplateNumber == 0 && defined(perturbationNumber) && defined(numberOfForecastsInEnsemble) && numberOfForecastsInEnsemble > 0) {
+  alias productDefinitionTemplateNumber = one; # ensemble instant
+}
+if (productDefinitionTemplateNumber == 8 && defined(perturbationNumber) && defined(numberOfForecastsInEnsemble) && numberOfForecastsInEnsemble > 0) {
+  alias productDefinitionTemplateNumber = eleven; # ensemble interval
+}

definitions/grib2/local/ecmf/section4_extras.def

@@ -5,7 +5,7 @@ if (centre is "ecmf" or datasetForLocal is "era6") {
  concept modelName(unknown, "modelNameConcept.def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
 
  if (modelName isnot "unknown") {
-    concept modelVersion(unknown, "modelVersionConcept_[modelName].def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
+    concept modelVersion(unknown, "modelVersionConcept.[modelName].def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
     # alias ls.model = modelName;
     # alias mars.model = modelName;
  }

definitions/grib2/localConcepts/ecmf/modelVersionConcept.IFS.def

@@ -0,0 +1,18 @@
+'cy50r3' = { generatingProcessIdentifier = 163; }
+'cy50r2' = { generatingProcessIdentifier = 162; }
+'cy50r1' = { generatingProcessIdentifier = 161; }
+'cy49r3' = { generatingProcessIdentifier = 160; }
+'cy49r2' = { generatingProcessIdentifier = 159; }
+'cy49r1' = { generatingProcessIdentifier = 158; }
+'cy48r3' = { generatingProcessIdentifier = 157; }
+'climatedt' = { generatingProcessIdentifier = 156; }
+'cy48r2' = { generatingProcessIdentifier = 155; }
+'cy48r1' = { generatingProcessIdentifier = 154; }
+'cy47r3' = { generatingProcessIdentifier = 153; }
+'cy47r2' = { generatingProcessIdentifier = 152; }
+'cy47r1' = { generatingProcessIdentifier = 151; }
+'cy46r1' = { generatingProcessIdentifier = 150; }
+'cy45r1' = { generatingProcessIdentifier = 149; }
+'cy43r3' = { generatingProcessIdentifier = 148; }
+'cy43r1' = { generatingProcessIdentifier = 147; }
+'cy41r2' = { generatingProcessIdentifier = 146; }