Fixed provenance logging of all python diagnostics by removing 'plot_file' entry

esmvaltool/diag_scripts/autoassess/land_surface_permafrost/permafrost.py

@@ -16,7 +16,7 @@
 from . import permafrost_koven_sites
 
 
-def get_provenance_record(plot_file, caption, run):
+def get_provenance_record(caption, run):
     """Create a provenance record describing the diagnostic data and plot."""
     record = {
         'caption': caption,
@@ -32,7 +32,6 @@ def get_provenance_record(plot_file, caption, run):
             'legates90tac',
             'koven13jclim',
         ],
-        'plot_file': plot_file,
         'ancestors': run,
     }
 
@@ -194,9 +193,7 @@ def permafrost_area(soiltemp, airtemp, landfrac, run):
     plot_file = 'pf_extent_asia_' + run['runid']
     caption = 'Permafrost extent & zero degree isotherm ({})'.format(
         run['runid'])
-    provenance_record = get_provenance_record(plot_file,
-                                              caption,
-                                              run)
+    provenance_record = get_provenance_record(caption, run)
     cfg = {}
     cfg['run_dir'] = run['out_dir']
     with ProvenanceLogger(cfg) as provenance_logger:

esmvaltool/diag_scripts/autoassess/land_surface_soilmoisture/soilmoisture.py

@@ -11,7 +11,7 @@
 logger = logging.getLogger(__name__)
 
 
-def get_provenance_record(plot_file, caption, run):
+def get_provenance_record(caption, run):
     """Create a provenance record describing the diagnostic data and plot."""
     record = {
         'caption': caption,
@@ -27,7 +27,6 @@ def get_provenance_record(plot_file, caption, run):
             'dorigo17rse',
             'gruber19essd',
         ],
-        'plot_file': plot_file,
         'ancestors': run,
     }
 
@@ -128,7 +127,7 @@ def land_sm_top(run):
     # record provenance
     plot_file = "Autoassess soilmoisture metrics"
     caption = 'Autoassess soilmoisture MedAbsErr for {}'.format(str(seasons))
-    provenance_record = get_provenance_record(plot_file, caption, run)
+    provenance_record = get_provenance_record(caption, run)
     cfg = {}
     cfg['run_dir'] = run['out_dir']
     # avoid rewriting provenance when running the plot diag

esmvaltool/diag_scripts/autoassess/land_surface_surfrad/surfrad.py

@@ -11,7 +11,7 @@
 from esmvaltool.diag_scripts.shared._supermeans import get_supermean
 
 
-def get_provenance_record(plot_file, caption, run):
+def get_provenance_record(caption, run):
     """Create a provenance record describing the diagnostic data and plot."""
     record = {
         'caption': caption,
@@ -26,7 +26,6 @@ def get_provenance_record(plot_file, caption, run):
             'loeb19jclim',
             'kato18ebaf',
         ],
-        'plot_file': plot_file,
         'ancestors': run,
     }
 
@@ -111,7 +110,7 @@ def land_surf_rad(run):
     # record provenance
     plot_file = "Autoassess Surface Radiation metrics"
     caption = '{} MedAbsErr for {}'.format(str(rad_fld), str(rad_seasons))
-    provenance_record = get_provenance_record(plot_file, caption, run)
+    provenance_record = get_provenance_record(caption, run)
     cfg = {}
     cfg['run_dir'] = run['out_dir']
     # avoid rewriting provenance when running the plot diag

esmvaltool/diag_scripts/climate_metrics/create_barplot.py

@@ -267,11 +267,11 @@ def main(cfg):
     caption = f"{metadata['long_name']} for multiple datasets."
     provenance_record = get_provenance_record(caption, all_files)
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['bar'],
     })
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 if __name__ == '__main__':

esmvaltool/diag_scripts/climate_metrics/create_scatterplot.py

@@ -148,11 +148,11 @@ def main(cfg):
         cube.long_name, '' if project is None else f' for {project}')
     provenance_record = get_provenance_record(caption, [input_file])
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['scatter'],
     })
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 if __name__ == '__main__':

esmvaltool/diag_scripts/climate_metrics/ecs.py

@@ -428,11 +428,10 @@ def plot_gregory_plot(cfg, dataset_name, tas_cube, rtnt_cube, reg_stats):
         f"including linear regression to calculate ECS for {dataset_name} "
         f"({project}).")
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['scatter'],
     })
 
-    return (netcdf_path, provenance_record)
+    return (netcdf_path, plot_path, provenance_record)
 
 
 def set_default_cfg(cfg):
@@ -529,13 +528,14 @@ def main(cfg):
         reg = stats.linregress(tas_cube.data, rtnt_cube.data)
 
         # Plot Gregory plots
-        (path, provenance_record) = plot_gregory_plot(cfg, dataset_name,
-                                                      tas_cube, rtnt_cube, reg)
+        (path, plot_path, provenance_record) = plot_gregory_plot(
+            cfg, dataset_name, tas_cube, rtnt_cube, reg)
 
         # Provenance
         provenance_record['ancestors'] = ancestor_files
         with ProvenanceLogger(cfg) as provenance_logger:
             provenance_logger.log(path, provenance_record)
+            provenance_logger.log(plot_path, provenance_record)
 
         # Save data
         if cfg.get('read_external_file') and dataset_name in ecs:

esmvaltool/diag_scripts/climate_metrics/feedback_parameters.py

@@ -766,11 +766,11 @@ def _write_provenance(netcdf_path, plot_path, caption, ancestors, cfg,
     provenance_record = _get_provenance_record(caption)
     provenance_record.update({
         'ancestors': ancestors,
-        'plot_file': plot_path,
         **kwargs,
     })
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def calculate_ecs(input_data, cfg, description=None):

esmvaltool/diag_scripts/climate_metrics/tcr.py

@@ -170,7 +170,7 @@ def _get_mmm_anomaly(cubes, ancestors, cfg):
 def _plot(cfg, cube, dataset_name, tcr):
     """Create scatterplot of temperature anomaly vs. time."""
     if not cfg.get('plot', True):
-        return (None, None)
+        return (None, None, None)
     logger.debug("Plotting temperature anomaly vs. time for '%s'",
                  dataset_name)
     (_, axes) = plt.subplots()
@@ -215,11 +215,10 @@ def _plot(cfg, cube, dataset_name, tcr):
         f"(TCR) defined as the 20 year average temperature anomaly centered "
         f"at the time of CO2 doubling (vertical dashed lines).")
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['times'],
     })
 
-    return (netcdf_path, provenance_record)
+    return (netcdf_path, plot_path, provenance_record)
 
 
 def calculate_tcr(cfg):
@@ -239,12 +238,13 @@ def calculate_tcr(cfg):
                     anomaly_cube.units)
 
         # Plot
-        (path, provenance_record) = _plot(cfg, anomaly_cube, dataset_name,
-                                          new_tcr)
+        (path, plot_path, provenance_record) = _plot(cfg, anomaly_cube,
+                                                     dataset_name, new_tcr)
         if path is not None:
             provenance_record['ancestors'] = ancestors[dataset_name]
             with ProvenanceLogger(cfg) as provenance_logger:
                 provenance_logger.log(path, provenance_record)
+                provenance_logger.log(plot_path, provenance_record)
 
     return tcr
 

esmvaltool/diag_scripts/crem/ww09_esmvaltool.py

@@ -218,25 +218,22 @@ def main(cfg):
 
     # plot results
 
-    if cfg['write_plots']:
-        plotname = os.path.join(
-            cfg['plot_dir'],
-            'ww09_metric_multimodel.' + cfg['output_file_type'],
-        )
-        logger.debug("Plotting results to %s", plotname)
+    plotname = os.path.join(
+        cfg['plot_dir'],
+        'ww09_metric_multimodel.' + cfg['output_file_type'],
+    )
+    logger.debug("Plotting results to %s", plotname)
 
-        plt.figure()
-        ypos = np.arange(nummod)
-        plt.barh(ypos, crems, align='center')
-        plt.yticks(ypos, models)
-        plt.xlabel('Cloud Regime Error Metric')
+    plt.figure()
+    ypos = np.arange(nummod)
+    plt.barh(ypos, crems, align='center')
+    plt.yticks(ypos, models)
+    plt.xlabel('Cloud Regime Error Metric')
 
-        # draw observational uncertainties (dashed red line)
-        plt.plot([0.96, 0.96], [-0.5, nummod - 0.5], 'r--')
+    # draw observational uncertainties (dashed red line)
+    plt.plot([0.96, 0.96], [-0.5, nummod - 0.5], 'r--')
 
-        plt.savefig(plotname, bbox_inches='tight')
-
-        provenance_record['plot_file'] = plotname
+    plt.savefig(plotname, bbox_inches='tight')
 
     # save results to netcdf
 
@@ -282,6 +279,7 @@ def main(cfg):
 
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(oname, provenance_record)
+        provenance_logger.log(plotname, provenance_record)
 
 
 def read_and_check(srcfilename, varname, lons2, lats2, time2):

esmvaltool/diag_scripts/cvdp/cvdp_wrapper.py

@@ -271,7 +271,6 @@ def _get_provenance_record(filename, ancestors):
             'statistics': [_get_stat(filename)],
             'domain': 'global',
             'plot_type': _get_plot_type(filename),
-            'plot_file': filename,
             'authors': [
                 'phillips_adam',
             ],

esmvaltool/diag_scripts/emergent_constraints/cox18nature.py

@@ -254,11 +254,12 @@ def plot_temperature_anomaly(cfg, tas_cubes, lambda_cube, obs_name):
     legend = _get_line_plot_legend()
 
     # Save plot
-    provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
+    plot_path = _save_fig(cfg, filename, legend)
 
     # Write provenance
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def plot_psi(cfg, psi_cubes, lambda_cube, obs_name):
@@ -301,11 +302,12 @@ def plot_psi(cfg, psi_cubes, lambda_cube, obs_name):
     legend = _get_line_plot_legend()
 
     # Save plot
-    provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
+    plot_path = _save_fig(cfg, filename, legend)
 
     # Write provenance
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def plot_emergent_relationship(cfg, psi_cube, ecs_cube, lambda_cube, obs_cube):
@@ -371,11 +373,12 @@ def plot_emergent_relationship(cfg, psi_cube, ecs_cube, lambda_cube, obs_cube):
     legend = AXES.legend(loc='upper left')
 
     # Save plot
-    provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
+    plot_path = _save_fig(cfg, filename, legend)
 
     # Write provenance
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def plot_pdf(cfg, ecs_lin, ecs_pdf, ecs_cube, obs_name):
@@ -416,11 +419,12 @@ def plot_pdf(cfg, ecs_lin, ecs_pdf, ecs_cube, obs_name):
     legend = AXES.legend(loc='upper left')
 
     # Save plot
-    provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
+    plot_path = _save_fig(cfg, filename, legend)
 
     # Write provenance
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def plot_cdf(cfg, ecs_lin, ecs_pdf, ecs_cube, obs_name):
@@ -473,11 +477,12 @@ def plot_cdf(cfg, ecs_lin, ecs_pdf, ecs_cube, obs_name):
     legend = AXES.legend(loc='upper left')
 
     # Save plot
-    provenance_record['plot_file'] = _save_fig(cfg, filename, legend)
+    plot_path = _save_fig(cfg, filename, legend)
 
     # Write provenance
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 def get_ecs_range(cfg, ecs_lin, ecs_pdf):

esmvaltool/diag_scripts/ipcc_ar5/ch09_fig09_14.py

@@ -393,15 +393,13 @@ def main(config):
     """
     load_data(config)
     data = prepare_data(config)
-    if config['write_plots']:
-        plot_path = produce_plots(config, data)
+    plot_path = produce_plots(config, data)
     ancestor_files = list(config['input_data'].keys())
     provenance_record = get_provenance_record(ancestor_files)
-    if config['write_plots']:
-        provenance_record['plot_file'] = plot_path
     netcdf_path = write_data(config, data)
     with ProvenanceLogger(config) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 if __name__ == '__main__':

esmvaltool/diag_scripts/ipcc_ar5/ch09_fig09_42a.py

@@ -208,11 +208,11 @@ def main(cfg):
     ancestor_files.append(ecs_filepath)
     provenance_record = get_provenance_record(project, ancestor_files)
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['scatter'],
     })
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 if __name__ == '__main__':

esmvaltool/diag_scripts/ipcc_ar5/ch09_fig09_42b.py

@@ -290,11 +290,11 @@ def main(cfg):
     ancestor_files = [ecs_file, tcr_file]
     provenance_record = get_provenance_record(project, ancestor_files)
     provenance_record.update({
-        'plot_file': plot_path,
         'plot_types': ['scatter'],
     })
     with ProvenanceLogger(cfg) as provenance_logger:
         provenance_logger.log(netcdf_path, provenance_record)
+        provenance_logger.log(plot_path, provenance_record)
 
 
 if __name__ == '__main__':