Add file encoding (and some read modes) at open file step (#2219)

Co-authored-by: Bouwe Andela <[email protected]>

doc/gensidebar.py

@@ -10,13 +10,13 @@
 def _write_if_changed(fname, contents):
     """Write/update file only if changed."""
     try:
-        with open(fname, "r") as stream:
+        with open(fname, "r", encoding="utf-8") as stream:
             old_contents = stream.read()
     except IOError:
         old_contents = ""
 
     if old_contents != contents:
-        with open(fname, "w") as stream:
+        with open(fname, "w", encoding="utf-8") as stream:
             stream.write(contents)
 
 

esmvalcore/_citation.py

@@ -101,7 +101,8 @@ def _save_citation_bibtex(product_name, tags, json_urls):
             entries.add(cmip_citation)
     citation_entries.extend(sorted(entries))
 
-    with open(f'{product_name}_citation.bibtex', 'w') as file:
+    with open(f'{product_name}_citation.bibtex',
+              'w', encoding='utf-8') as file:
         file.write('\n'.join(citation_entries))
 
 
@@ -126,7 +127,8 @@ def _save_citation_info_txt(product_name, info_urls, other_info):
                      for t in sorted(other_info))
 
     if lines:
-        with open(f'{product_name}_data_citation_info.txt', 'w') as file:
+        with open(f'{product_name}_data_citation_info.txt',
+                  'w', encoding='utf-8') as file:
             file.write('\n'.join(lines) + '\n')
 
 
@@ -196,7 +198,7 @@ def _collect_bibtex_citation(tag):
     """Collect information from bibtex files."""
     bibtex_file = DIAGNOSTICS.references / f'{tag}.bibtex'
     if bibtex_file.is_file():
-        entry = bibtex_file.read_text()
+        entry = bibtex_file.read_text(encoding='utf-8')
     else:
         entry = ''
         logger.warning(

esmvalcore/_main.py

@@ -64,10 +64,10 @@ def parse_resume(resume, recipe):
         resume[i] = Path(os.path.expandvars(resume_dir)).expanduser()
 
     # Sanity check resume directories:
-    current_recipe = recipe.read_text()
+    current_recipe = recipe.read_text(encoding='utf-8')
     for resume_dir in resume:
         resume_recipe = resume_dir / 'run' / recipe.name
-        if current_recipe != resume_recipe.read_text():
+        if current_recipe != resume_recipe.read_text(encoding='utf-8'):
             raise ValueError(f'Only identical recipes can be resumed, but '
                              f'{resume_recipe} is different from {recipe}')
     return resume

esmvalcore/_task.py

@@ -119,7 +119,7 @@ def _log_resource_usage():
         """Write resource usage to file."""
         process = psutil.Process(pid)
         start_time = time.time()
-        with open(filename, 'w') as file:
+        with open(filename, 'w', encoding='utf-8') as file:
             for msg, max_mem in _get_resource_usage(process, start_time,
                                                     children):
                 file.write(msg)
@@ -219,7 +219,7 @@ def _ncl_type(value):
                          'end if\n'.format(var_name=var_name))
             lines.append(_py2ncl(value, var_name))
 
-    with open(filename, mode) as file:
+    with open(filename, mode, encoding='utf-8') as file:
         file.write('\n'.join(lines))
         file.write('\n')
 
@@ -301,7 +301,7 @@ def __init__(self, prev_preproc_dir, preproc_dir, name):
 
         # Reconstruct output
         prev_metadata_file = prev_preproc_dir / 'metadata.yml'
-        with prev_metadata_file.open('rb') as file:
+        with prev_metadata_file.open('r', encoding='utf-8') as file:
             prev_metadata = yaml.safe_load(file)
 
         products = set()
@@ -323,7 +323,7 @@ def _run(self, _):
 
         # Write metadata to file
         self._metadata_file.parent.mkdir(parents=True)
-        with self._metadata_file.open('w') as file:
+        with self._metadata_file.open('w', encoding='utf-8') as file:
             yaml.safe_dump(metadata, file)
 
         return [str(self._metadata_file)]
@@ -609,7 +609,7 @@ def _collect_provenance(self):
 
         logger.debug("Collecting provenance from %s", provenance_file)
         start = time.time()
-        table = yaml.safe_load(provenance_file.read_text())
+        table = yaml.safe_load(provenance_file.read_text(encoding='utf-8'))
 
         ignore = (
             'auxiliary_data_dir',

esmvalcore/cmor/table.py

@@ -150,7 +150,7 @@ def _read_cmor_tables(cfg_file: Path, mtime: float) -> dict[str, CMORTable]:
         cfg_developer = yaml.safe_load(file)
     cwd = os.path.dirname(os.path.realpath(__file__))
     var_alt_names_file = os.path.join(cwd, 'variable_alt_names.yml')
-    with open(var_alt_names_file, 'r') as yfile:
+    with open(var_alt_names_file, 'r', encoding='utf-8') as yfile:
         alt_names = yaml.safe_load(yfile)
 
     cmor_tables: dict[str, CMORTable] = {}
@@ -415,7 +415,7 @@ def _get_cmor_path(cmor_tables_path):
             'CMOR tables not found in {}'.format(cmor_tables_path))
 
     def _load_table(self, json_file):
-        with open(json_file) as inf:
+        with open(json_file, encoding='utf-8') as inf:
             raw_data = json.loads(inf.read())
             if not self._is_table(raw_data):
                 return
@@ -465,7 +465,7 @@ def _load_coordinates(self):
         self.coords = {}
         for json_file in glob.glob(
                 os.path.join(self._cmor_folder, '*coordinate*.json')):
-            with open(json_file) as inf:
+            with open(json_file, encoding='utf-8') as inf:
                 table_data = json.loads(inf.read())
                 for coord_name in table_data['axis_entry'].keys():
                     coord = CoordinateInfo(coord_name)
@@ -477,7 +477,7 @@ def _load_controlled_vocabulary(self):
         self.institutes = {}
         for json_file in glob.glob(os.path.join(self._cmor_folder,
                                                 '*_CV.json')):
-            with open(json_file) as inf:
+            with open(json_file, encoding='utf-8') as inf:
                 table_data = json.loads(inf.read())
                 try:
                     exps = table_data['CV']['experiment_id']
@@ -846,7 +846,7 @@ def _load_table(self, table_file, table_name=''):
         self._read_table_file(table_file, table)
 
     def _read_table_file(self, table_file, table=None):
-        with open(table_file) as self._current_table:
+        with open(table_file, 'r', encoding='utf-8') as self._current_table:
             self._read_line()
             while True:
                 key, value = self._last_line_read
@@ -1057,7 +1057,7 @@ def get_variable(self, table, short_name, derived=False):
         return self.tables['custom'].get(short_name, None)
 
     def _read_table_file(self, table_file, table=None):
-        with open(table_file) as self._current_table:
+        with open(table_file, 'r', encoding='utf-8') as self._current_table:
             self._read_line()
             while True:
                 key, value = self._last_line_read

esmvalcore/config/_config.py

@@ -43,7 +43,7 @@ def _load_extra_facets(project, extra_facets_dir):
         config_file_paths = config_path.glob(f"{project.lower()}-*.yml")
         for config_file_path in sorted(config_file_paths):
             logger.debug("Loading extra facets from %s", config_file_path)
-            with config_file_path.open() as config_file:
+            with config_file_path.open(encoding='utf-8') as config_file:
                 config_piece = yaml.safe_load(config_file)
             if config_piece:
                 _deep_update(config, config_piece)

esmvalcore/config/_config_object.py

@@ -225,7 +225,7 @@ def _read_config_file(config_file):
     if not config_file.exists():
         raise IOError(f'Config file `{config_file}` does not exist.')
 
-    with open(config_file, 'r') as file:
+    with open(config_file, 'r', encoding='utf-8') as file:
         cfg = yaml.safe_load(file)
 
     return cfg

esmvalcore/config/_diagnostics.py

@@ -83,7 +83,7 @@ def from_file(cls, filename: str):
         """Load the reference tags used for provenance recording."""
         if os.path.exists(filename):
             logger.debug("Loading tags from %s", filename)
-            with open(filename) as file:
+            with open(filename, 'r', encoding='utf-8') as file:
                 tags = cls(yaml.safe_load(file))
                 tags.source_file = filename
                 return tags

esmvalcore/config/_esgf_pyclient.py

@@ -110,7 +110,7 @@ def read_config_file():
         if mode & stat.S_IRWXG or mode & stat.S_IRWXO:
             logger.warning("Correcting unsafe permissions on %s", CONFIG_FILE)
             os.chmod(CONFIG_FILE, stat.S_IRUSR | stat.S_IWUSR)
-        with CONFIG_FILE.open() as file:
+        with CONFIG_FILE.open(encoding='utf-8') as file:
             cfg = yaml.safe_load(file)
     else:
         logger.info(

esmvalcore/config/_logging.py

@@ -87,7 +87,7 @@ def configure_logging(
 
     cfg_file = Path(cfg_file).absolute()
 
-    with open(cfg_file) as file_handler:
+    with open(cfg_file, 'r', encoding='utf-8') as file_handler:
         cfg = yaml.safe_load(file_handler)
 
     if output_dir is None:

esmvalcore/esgf/_download.py

@@ -52,7 +52,7 @@ def compute_speed(size, duration):
 def load_speeds():
     """Load average download speeds from HOSTS_FILE."""
     try:
-        content = HOSTS_FILE.read_text()
+        content = HOSTS_FILE.read_text(encoding='utf-8')
     except FileNotFoundError:
         content = '{}'
     speeds = yaml.safe_load(content)
@@ -94,7 +94,7 @@ def atomic_write(filename):
     filename.parent.mkdir(parents=True, exist_ok=True)
     with NamedTemporaryFile(prefix=f"{filename}.") as file:
         tmp_file = file.name
-    with open(tmp_file, 'w') as file:
+    with open(tmp_file, 'w', encoding='utf-8') as file:
         yield file
     shutil.move(tmp_file, filename)
 

esmvalcore/experimental/recipe.py

@@ -70,7 +70,8 @@ def name(self):
     def data(self) -> dict:
         """Return dictionary representation of the recipe."""
         if self._data is None:
-            self._data = yaml.safe_load(open(self.path, 'r'))
+            with open(self.path, 'r', encoding='utf-8') as yaml_file:
+                self._data = yaml.safe_load(yaml_file)
         return self._data
 
     def _load(self, session: Session) -> RecipeEngine:

esmvalcore/experimental/recipe_info.py

@@ -72,7 +72,7 @@ def _repr_html_(self) -> str:
     @classmethod
     def from_yaml(cls, path: str):
         """Return instance of 'RecipeInfo' from a recipe in yaml format."""
-        data = yaml.safe_load(open(path, 'r'))
+        data = yaml.safe_load(Path(path).read_text(encoding='utf-8'))
         return cls(data, filename=path)
 
     @property

esmvalcore/experimental/recipe_output.py

@@ -200,7 +200,7 @@ def write_html(self):
         template = get_template('recipe_output_page.j2')
         html_dump = self.render(template=template)
 
-        with open(filename, 'w') as file:
+        with open(filename, 'w', encoding='utf-8') as file:
             file.write(html_dump)
 
         logger.info("Wrote recipe output to:\nfile://%s", filename)
@@ -225,11 +225,11 @@ def render(self, template=None):
 
     def read_main_log(self) -> str:
         """Read log file."""
-        return self.session.main_log.read_text()
+        return self.session.main_log.read_text(encoding='utf-8')
 
     def read_main_log_debug(self) -> str:
         """Read debug log file."""
-        return self.session.main_log_debug.read_text()
+        return self.session.main_log_debug.read_text(encoding='utf-8')
 
 
 class OutputFile():

esmvalcore/preprocessor/_io.py

@@ -495,7 +495,7 @@ def write_metadata(products, write_ncl=False):
 
         output_filename = os.path.join(output_dir, 'metadata.yml')
         output_files.append(output_filename)
-        with open(output_filename, 'w') as file:
+        with open(output_filename, 'w', encoding='utf-8') as file:
             yaml.safe_dump(metadata, file)
         if write_ncl:
             output_files.append(_write_ncl_metadata(output_dir, metadata))

setup.py

@@ -183,7 +183,7 @@ def run(self):
 
 def read_authors(filename):
     """Read the list of authors from .zenodo.json file."""
-    with Path(filename).open() as file:
+    with Path(filename).open(encoding='utf-8') as file:
         info = json.load(file)
         authors = []
         for author in info['creators']:
@@ -194,7 +194,7 @@ def read_authors(filename):
 
 def read_description(filename):
     """Read the description from .zenodo.json file."""
-    with Path(filename).open() as file:
+    with Path(filename).open(encoding='utf-8') as file:
         info = json.load(file)
         return info['description']
 
@@ -203,7 +203,7 @@ def read_description(filename):
     name='ESMValCore',
     author=read_authors('.zenodo.json'),
     description=read_description('.zenodo.json'),
-    long_description=Path('README.md').read_text(),
+    long_description=Path('README.md').read_text(encoding='utf-8'),
     long_description_content_type='text/markdown',
     url='https://www.esmvaltool.org',
     download_url='https://github.com/ESMValGroup/ESMValCore',

tests/integration/cmor/_fixes/icon/test_icon.py

@@ -2066,7 +2066,7 @@ def test_get_path_from_facet(path, description, output, tmp_path):
 
     # Create empty dummy file
     output = output.format(tmp_path=tmp_path)
-    with open(output, 'w'):
+    with open(output, 'w', encoding='utf-8'):
         pass
 
     out_path = fix._get_path_from_facet('test_path', description=description)

tests/integration/esgf/test_search_download.py

@@ -116,7 +116,7 @@ def test_mock_search(variable, mocker):
         # Skip cases where the raw search results were too large to save.
         pytest.skip(f"Raw search results in {raw_results} not available.")
 
-    with raw_results.open('r') as file:
+    with raw_results.open('r', encoding='utf-8') as file:
         search_results = [
             FileResult(json=j, context=None) for j in json.load(file)
         ]

tests/integration/recipe/test_recipe.py

@@ -1251,7 +1251,7 @@ def simulate_diagnostic_run(diagnostic_task):
     plot_file = create_test_image('test', cfg)
     provenance = os.path.join(cfg['run_dir'], 'diagnostic_provenance.yml')
     os.makedirs(cfg['run_dir'])
-    with open(provenance, 'w') as file:
+    with open(provenance, 'w', encoding='utf-8') as file:
         yaml.safe_dump({diagnostic_file: record, plot_file: record}, file)
 
     diagnostic_task._collect_provenance()

tests/integration/test_citation.py

@@ -33,7 +33,7 @@ def test_references(tmp_path, monkeypatch):
 
     _write_citation_files(filename, provenance)
     citation_file = tmp_path / 'output_citation.bibtex'
-    citation = citation_file.read_text()
+    citation = citation_file.read_text(encoding='utf-8')
     assert citation == '\n'.join([ESMVALTOOL_PAPER, fake_bibtex])
 
 
@@ -84,7 +84,7 @@ def test_cmip6_data_citation(tmp_path, monkeypatch):
         \tdoi = {{{doi}}},
         }}
         """).lstrip()
-    assert citation_file.read_text() == '\n'.join(
+    assert citation_file.read_text(encoding='utf-8') == '\n'.join(
         [ESMVALTOOL_PAPER, fake_bibtex_entry])
 
 
@@ -114,4 +114,4 @@ def test_cmip6_data_citation_url(tmp_path):
         f"- {CMIP6_URL_STEM}/cmip6?input={fake_url_prefix}",
         '',
     ])
-    assert citation_url.read_text() == text
+    assert citation_url.read_text(encoding='utf-8') == text

tests/integration/test_diagnostic_run.py

@@ -56,7 +56,7 @@ def arguments(*args):
 
 def check(result_file):
     """Check the results."""
-    result = yaml.safe_load(result_file.read_text())
+    result = yaml.safe_load(result_file.read_text(encoding='utf-8'))
 
     required_keys = {
         'input_files',
@@ -80,7 +80,7 @@ def check(result_file):
         import yaml
         import shutil
 
-        with open("settings.yml", 'r') as file:
+        with open("settings.yml", 'r', encoding='utf-8') as file:
             settings = yaml.safe_load(file)
 
         shutil.copy("settings.yml", settings["setting_name"])

tests/integration/test_local.py

@@ -10,7 +10,9 @@
 from esmvalcore.local import LocalFile, _get_output_file, find_files
 
 # Load test configuration
-with open(os.path.join(os.path.dirname(__file__), 'data_finder.yml')) as file:
+with open(os.path.join(os.path.dirname(__file__),
+                       'data_finder.yml'),
+          encoding='utf-8') as file:
     CONFIG = yaml.safe_load(file)
 
 
@@ -40,7 +42,7 @@ def create_file(filename):
     if not os.path.exists(dirname):
         os.makedirs(dirname)
 
-    with open(filename, 'a'):
+    with open(filename, 'a', encoding='utf-8'):
         pass
 
 

tests/integration/test_main.py

@@ -81,7 +81,7 @@ def test_empty_run(tmp_path):
     Config.get_config_user(path=tmp_path)
     log_dir = f'{tmp_path}/esmvaltool_output'
     config_file = f"{tmp_path}/config-user.yml"
-    with open(config_file, 'r+') as file:
+    with open(config_file, 'r+', encoding='utf-8') as file:
         config = yaml.safe_load(file)
         config['output_dir'] = log_dir
         yaml.safe_dump(config, file, sort_keys=False)