Merge pull request #293 from roedoejet/dev.fix_generate_mapping

Produce valid `config-g2p.yaml` when running `g2p generate-mapping`
roedoejet · Oct 5, 2023 · 3d293d4 · 3d293d4
2 parents 865f16c + 686232b
commit 3d293d4
Show file tree

Hide file tree

Showing 6 changed files with 151 additions and 58 deletions.
diff --git a/g2p/mappings/__init__.py b/g2p/mappings/__init__.py
@@ -318,18 +318,37 @@ def mapping_to_file(self, output_path: str = GEN_DIR, file_type: str = "json"):
         with open(fn, "w", encoding="utf8", newline="\n") as f:
             self.mapping_to_stream(f, file_type)
 
-    def export_to_dict(self, mapping_type="json"):
-        model_dict = json.loads(
-            self.model_dump_json(exclude_none=True, exclude={"parent_dir": True})
-        )
+    def export_to_dict(self, mapping_type="json", config_only=False):
+        """Export a mapping to a dictionary, optionally including only the
+        configuration (and not the rules or alignments) in the case
+        where we are just writing the config file.
+        """
+        if config_only:
+            model_dict = self.model_dump(
+                mode="json",
+                exclude_none=True,
+                exclude={
+                    "parent_dir": True,
+                    "rules": True,
+                    "processed": True,
+                    "alignments": True,
+                    "abbreviations": True,
+                },
+            )
+        else:
+            model_dict = self.model_dump(
+                mode="json",
+                exclude_none=True,
+                exclude={"parent_dir": True},
+            )
         model_dict["rules_path"] = f"{self.in_lang}_to_{self.out_lang}.{mapping_type}"
         return model_dict
 
     def config_to_file(
         self,
         output_path: str = os.path.join(GEN_DIR, "config-g2p.yaml"),
     ):
-        """Write config to file"""
+        """Write configuration to file."""
         add_config = False
         if os.path.isdir(output_path):
             output_path = os.path.join(output_path, "config-g2p.yaml")
@@ -339,7 +358,7 @@ def config_to_file(
         else:
             LOGGER.warning(f"writing mapping config to file at {output_path}")
         fn = output_path
-        config_template = self.export_to_dict()
+        config_template = self.export_to_dict(config_only=True)
         # Serialize piece-by-piece, which is why this is a list of type dict and not type Mapping
         # If config file exists already, just add the mapping.
         to_export = None
@@ -359,14 +378,22 @@ def config_to_file(
                 existing_data.mappings.append(config_template)
             to_export = {
                 "mappings": [
-                    x.export_to_dict() if isinstance(x, Mapping) else x
+                    x.export_to_dict(config_only=True) if isinstance(x, Mapping) else x
                     for x in existing_data.mappings
-                ]
+                ],
             }
         else:
             to_export = {"mappings": [config_template]}
         with open(fn, "w", encoding="utf8", newline="\n") as f:
-            yaml.dump(to_export, f, Dumper=IndentDumper, default_flow_style=False)
+            yaml.dump(
+                to_export,
+                f,
+                Dumper=IndentDumper,
+                default_flow_style=False,
+                # do not write strings as unreadable \u escapes! (see
+                # https://stackoverflow.com/questions/10648614/dump-in-pyyaml-as-utf-8)
+                allow_unicode=True,
+            )
 
 
 MAPPINGS_AVAILABLE: List[Mapping] = [