From ee7fcebd5fb5de98f93033c7f6692558116c3d8d Mon Sep 17 00:00:00 2001 From: "Dheeraj Agarwal (ML)" Date: Fri, 23 Jun 2023 15:54:30 +0530 Subject: [PATCH] fix pycache folder being added instead of encoding info files --- tiktoken/registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tiktoken/registry.py b/tiktoken/registry.py index 52d8ec2d..241747c2 100644 --- a/tiktoken/registry.py +++ b/tiktoken/registry.py @@ -27,8 +27,9 @@ def _find_constructors() -> None: # - it's a separate top-level package because namespace subpackages of non-namespace # packages don't quite do what you want with editable installs plugin_mods = pkgutil.iter_modules(tiktoken_ext.__path__, tiktoken_ext.__name__ + ".") - - for _, mod_name, _ in plugin_mods: + filtered_plugin_mods = [(loader, mod_name, ispkg) for loader, mod_name, ispkg in plugin_mods if not mod_name.endswith('__pycache__')] + + for _, mod_name, _ in filtered_plugin_mods: mod = importlib.import_module(mod_name) try: constructors = mod.ENCODING_CONSTRUCTORS