better error recovery for file encoding and bad json

beet/contrib/auto_yaml.py

@@ -10,7 +10,7 @@
 
 from typing import Any, ClassVar, Tuple, Type
 
-from beet import Context, Drop, JsonFileBase, NamespaceFile, Pack, YamlFile
+from beet import Context, Drop, JsonFileBase, NamespaceFile, Pack, YamlFile, NamespaceFileScope
 
 
 def beet_default(ctx: Context):
@@ -54,7 +54,7 @@ def create_namespace_handler(
     """Create handler that turns yaml namespace files into json."""
 
     class AutoYamlNamespaceHandler(YamlFile):
-        scope: ClassVar[Tuple[str, ...]] = namespace_scope
+        scope: ClassVar[NamespaceFileScope] = namespace_scope
         extension: ClassVar[str] = namespace_extension
 
         model = file_type.model

beet/contrib/messages.py

@@ -9,9 +9,9 @@
 import json
 import re
 from dataclasses import dataclass
-from typing import Any, ClassVar, Optional, Tuple, cast
+from typing import Any, ClassVar, Optional, cast
 
-from beet import Context, JsonFile
+from beet import Context, JsonFile, NamespaceFileScope
 from beet.core.utils import TextComponent
 
 PATH_REGEX = re.compile(r"\w+")
@@ -20,7 +20,7 @@
 class Message(JsonFile):
     """Class representing a message file."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("messages",)
+    scope: ClassVar[NamespaceFileScope] = ("messages",)
     extension: ClassVar[str] = ".json"
 
 

beet/contrib/optifine.py

@@ -10,9 +10,10 @@
 ]
 
 
-from typing import ClassVar, Tuple, Union
+from typing import ClassVar, Union
 
 from beet import Context, JsonFile, PngFile, ResourcePack, TextFile
+from beet.library.base import NamespaceFileScope
 
 
 def beet_default(ctx: Context):
@@ -35,33 +36,33 @@ def optifine(pack: Union[Context, ResourcePack]):
 class JsonEntityModel(JsonFile):
     """Class representing a json entity model."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("optifine", "cem")
+    scope: ClassVar[NamespaceFileScope] = ("optifine", "cem")
     extension: ClassVar[str] = ".jem"
 
 
 class JsonPartModel(JsonFile):
     """Class representing a json part model."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("optifine", "cem")
+    scope: ClassVar[NamespaceFileScope] = ("optifine", "cem")
     extension: ClassVar[str] = ".jpm"
 
 
 class OptifineProperties(TextFile):
     """Class representing optifine properties."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("optifine",)
+    scope: ClassVar[NamespaceFileScope] = ("optifine",)
     extension: ClassVar[str] = ".properties"
 
 
 class OptifineTexture(PngFile):
     """Class representing an optifine texture."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("optifine",)
+    scope: ClassVar[NamespaceFileScope] = ("optifine",)
     extension: ClassVar[str] = ".png"
 
 
 class ShaderProperties(TextFile):
     """Class representing shader properties."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("shaders",)
+    scope: ClassVar[NamespaceFileScope] = ("shaders",)
     extension: ClassVar[str] = ".properties"

beet/contrib/rename_files.py

@@ -31,6 +31,7 @@
     ResourcePack,
     TemplateManager,
     configurable,
+    get_output_scope,
 )
 from beet.contrib.find_replace import RenderSubstitutionOption, TextSubstitutionOption
 
@@ -115,7 +116,8 @@ def handle_filename_for_namespace_file(
     ):
         dest = self.substitute(filename)
         file_type = type(file_instance)
-        prefix = "".join(f"{d}/" for d in file_type.scope)
+        scope = get_output_scope(file_type.scope, pack.pack_format)
+        prefix = "".join(f"{d}/" for d in scope)
 
         _, namespace, path = filename.split("/", 2)
 

beet/contrib/unknown_files.py

@@ -7,22 +7,22 @@
 ]
 
 
-from typing import ClassVar, Tuple
+from typing import ClassVar
 
-from beet import BinaryFile, Context
+from beet import BinaryFile, Context, NamespaceFileScope
 
 
 class UnknownAsset(BinaryFile):
     """Class representing an unknown file in resource packs."""
 
-    scope: ClassVar[Tuple[str, ...]] = ()
+    scope: ClassVar[NamespaceFileScope] = ()
     extension: ClassVar[str] = ""
 
 
 class UnknownData(BinaryFile):
     """Class representing an unknown file in data packs."""
 
-    scope: ClassVar[Tuple[str, ...]] = ()
+    scope: ClassVar[NamespaceFileScope] = ()
     extension: ClassVar[str] = ""
 
 

beet/contrib/worldgen.py

@@ -26,9 +26,9 @@
 ]
 
 
-from typing import ClassVar, Tuple, Union
+from typing import ClassVar, Union
 
-from beet import Context, DataPack, JsonFile, TagFile
+from beet import Context, DataPack, JsonFile, TagFile, NamespaceFileScope
 
 
 def beet_default(ctx: Context):
@@ -67,140 +67,140 @@ def worldgen(pack: Union[Context, DataPack]):
 class Dimension(JsonFile):
     """Class representing a dimension."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("dimension",)
+    scope: ClassVar[NamespaceFileScope] = ("dimension",)
     extension: ClassVar[str] = ".json"
 
 
 class DimensionType(JsonFile):
     """Class representing a dimension type."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("dimension_type",)
+    scope: ClassVar[NamespaceFileScope] = ("dimension_type",)
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenBiome(JsonFile):
     """Class representing a biome."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "biome")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "biome")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenConfiguredCarver(JsonFile):
     """Class representing a worldgen carver."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "configured_carver")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "configured_carver")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenConfiguredFeature(JsonFile):
     """Class representing a worldgen feature."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "configured_feature")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "configured_feature")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenDensityFunction(JsonFile):
     """Class representing a density function."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "density_function")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "density_function")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenNoise(JsonFile):
     """Class representing a worldgen noise."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "noise")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "noise")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenNoiseSettings(JsonFile):
     """Class representing worldgen noise settings."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "noise_settings")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "noise_settings")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenPlacedFeature(JsonFile):
     """Class representing a placed feature."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "placed_feature")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "placed_feature")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenProcessorList(JsonFile):
     """Class representing a worldgen processor list."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "processor_list")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "processor_list")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenStructure(JsonFile):
     """Class representing a worldgen structure feature."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "structure")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "structure")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenStructureSet(JsonFile):
     """Class representing a worldgen structure set."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "structure_set")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "structure_set")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenConfiguredSurfaceBuilder(JsonFile):
     """Class representing a worldgen surface builder."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "configured_surface_builder")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "configured_surface_builder")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenTemplatePool(JsonFile):
     """Class representing a worldgen template pool."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "template_pool")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "template_pool")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenWorldPreset(JsonFile):
     """Class representing a worldgen world preset."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "world_preset")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "world_preset")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenFlatLevelGeneratorPreset(JsonFile):
     """Class representing a worldgen flat level generator preset."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("worldgen", "flat_level_generator_preset")
+    scope: ClassVar[NamespaceFileScope] = ("worldgen", "flat_level_generator_preset")
     extension: ClassVar[str] = ".json"
 
 
 class WorldgenBiomeTag(TagFile):
     """Class representing a biome tag."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("tags", "worldgen", "biome")
+    scope: ClassVar[NamespaceFileScope] = ("tags", "worldgen", "biome")
 
 
 class WorldgenStructureTag(TagFile):
     """Class representing a worldgen structure feature tag."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("tags", "worldgen", "structure")
+    scope: ClassVar[NamespaceFileScope] = ("tags", "worldgen", "structure")
 
 
 class WorldgenStructureSetTag(TagFile):
     """Class representing a worldgen structure set tag."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("tags", "worldgen", "structure_set")
+    scope: ClassVar[NamespaceFileScope] = ("tags", "worldgen", "structure_set")
 
 
 class WorldgenConfiguredCarverTag(TagFile):
     """Class representing a worldgen carver tag."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("tags", "worldgen", "configured_carver")
+    scope: ClassVar[NamespaceFileScope] = ("tags", "worldgen", "configured_carver")
 
 
 class WorldgenPlacedFeatureTag(TagFile):
     """Class representing a worldgen placed feature tag."""
 
-    scope: ClassVar[Tuple[str, ...]] = ("tags", "worldgen", "placed_feature")
+    scope: ClassVar[NamespaceFileScope] = ("tags", "worldgen", "placed_feature")

beet/core/file.py

@@ -23,6 +23,9 @@
 
 import io
 import json
+
+import chardet
+import pyjson5
 import shutil
 from copy import deepcopy
 from dataclasses import dataclass, replace
@@ -436,14 +439,41 @@ def deserialize(self, content: Union[ValueType, str]) -> ValueType:
 
     @classmethod
     def from_path(cls, path: FileSystemPath, start: int, stop: int) -> str:
-        with open(path, "r", encoding="utf-8") as f:
-            if start > 0:
-                f.seek(start)
-            return f.read(stop - start) if stop >= -1 else f.read()
+        # We assume that the file is utf-8 since its most common
+        # In the event the file fails to be parsed in utf-8, we'll attempt to detect
+        # the correct encoding and retry. If that fails than another exception will be raised
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                if start > 0:
+                    f.seek(start)
+                return f.read(stop - start) if stop >= -1 else f.read()
+        except ValueError:
+            with open(path, "rb") as f:
+                if start > 0:
+                    f.seek(start)
+                file_bytes = f.read(stop - start) if stop >= -1 else f.read()
+
+                encoding = chardet.detect(file_bytes)["encoding"]
+
+                if encoding == None:
+                    encoding = "utf-8"
+
+                return file_bytes.decode(encoding=encoding)
+
 
     @classmethod
     def from_zip(cls, origin: ZipFile, name: str) -> str:
-        return origin.read(name).decode()
+        file_bytes = origin.read(name)
+
+        try:
+            return file_bytes.decode()
+        except UnicodeDecodeError:
+            encoding = chardet.detect(file_bytes)["encoding"]
+
+            if encoding == None:
+                encoding = "utf-8"
+
+            return file_bytes.decode(encoding=encoding)
 
     def dump_path(self, path: FileSystemPath, raw: str) -> None:
         with open(
@@ -634,7 +664,7 @@ def __post_init__(self):
         if not self.encoder:  # type: ignore
             self.encoder = dump_json
         if not self.decoder:  # type: ignore
-            self.decoder = json.loads
+            self.decoder = pyjson5.loads
 
 
 @dataclass(eq=False, repr=False)