From 0c675843f522dd1c3497de6efc03b3d663d251bb Mon Sep 17 00:00:00 2001
From: SimFG <bang.fu@zilliz.com>
Date: Fri, 26 May 2023 22:24:55 +0800
Subject: [PATCH] Improve the class doc format

Signed-off-by: SimFG <bang.fu@zilliz.com>
---
 docs/.readthedocs.yaml                        |   5 +
 docs/_exts/docgen.py                          | 157 ------------------
 docs/_exts/docgen2.py                         | 106 ++++++++++++
 docs/_exts/{indexCon.py => index_con.py}      |   0
 docs/_templates/function.rst                  |  10 +-
 docs/_templates/index.rst                     |   2 +-
 docs/conf.py                                  |  35 ++--
 docs/references/index.rst                     |  13 +-
 docs/requirements.txt                         |   4 -
 examples/README.md                            |   2 +-
 gptcache/__init__.py                          |   3 +
 gptcache/adapter/api.py                       |   7 +-
 gptcache/adapter/openai.py                    |  14 +-
 gptcache/core.py                              |   8 +-
 .../similarity_evaluation.py                  |   1 +
 setup.py                                      |  21 ++-
 16 files changed, 182 insertions(+), 206 deletions(-)
 create mode 100644 docs/.readthedocs.yaml
 delete mode 100644 docs/_exts/docgen.py
 create mode 100644 docs/_exts/docgen2.py
 rename docs/_exts/{indexCon.py => index_con.py} (100%)

diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml
new file mode 100644
index 00000000..db4f5b92
--- /dev/null
+++ b/docs/.readthedocs.yaml
@@ -0,0 +1,5 @@
+version: 2
+
+python:
+   version: 3.8
+   system_packages: true
\ No newline at end of file
diff --git a/docs/_exts/docgen.py b/docs/_exts/docgen.py
deleted file mode 100644
index 5345e979..00000000
--- a/docs/_exts/docgen.py
+++ /dev/null
@@ -1,157 +0,0 @@
-import inspect
-import os
-import sys
-
-from jinja2 import Environment, FileSystemLoader, select_autoescape
-
-
-class DocGen:
-    def __init__(self, output_dir="references", skip_list=[]):
-        _default_skip_list = [
-            "__builtins__",
-            "__cached__",
-            "__doc__",
-            "__file__",
-            "__loader__",
-            "__name__",
-            "__package__",
-            "__spec__",
-        ]
-
-        self.OUTPUT = os.path.abspath(output_dir)
-        self.skip_list = _default_skip_list + skip_list
-        self.expand_func_list = []
-        self.add_list = [
-            "adapter.api",
-            "adapter.openai",
-            "adapter.diffusers",
-            "adapter.dolly",
-            "adapter.langchain_models",
-            "adapter.replicate",
-            "adapter.stability_sdk",
-        ]
-        self.add_root_list = [
-            "report",
-            "session",
-        ]
-
-    @staticmethod
-    def title_bar(input_str):
-        return "=" * len(input_str)
-
-    @staticmethod
-    def section_bar(input_str):
-        return "-" * len(input_str)
-
-    @staticmethod
-    def cap(input_str):
-        return "GPTCache" if input_str == "gptcache" else str.join(" ", [i.capitalize() for i in input_str.split("_")])
-
-    def generate(self, lib_name):
-        # Set the output directory
-        env = Environment(
-            loader=FileSystemLoader("_templates"), autoescape=select_autoescape()
-        )
-
-        # Add custom filters
-        env.filters["title_bar"] = DocGen.title_bar
-        env.filters["section_bar"] = DocGen.section_bar
-        env.filters["cap"] = DocGen.cap
-
-        # Add the target path to the system path
-        sys.path.insert(0, os.path.abspath(".."))
-
-        # Import the library
-        try:
-            lib = __import__(lib_name)
-        except ImportError:
-            print(f"Can't import {lib_name}")
-            return
-
-        for x in self.add_list:
-            sub_lib_name = lib_name + "." + x
-            try:
-                __import__(sub_lib_name)
-            except ImportError:
-                print(f"Can't import {sub_lib_name}")
-                continue
-
-        # Get the modules, functions, and classes
-        modules = [
-            x
-            for x in inspect.getmembers(lib)
-            if inspect.ismodule(x[1]) and str.startswith(x[1].__name__, f"{lib_name}.")
-        ]
-        functions = [x for x in inspect.getmembers(lib) if inspect.isfunction(x[1])]
-        classes = [x for x in inspect.getmembers(lib) if inspect.isclass(x[1])]
-
-        cf_combined = classes + functions
-        classes_method = [
-            (f"{y[0]}.{x[0]}", x) for y in classes for x in inspect.getmembers(y[1]) if
-            inspect.isfunction(x[1]) and not x[0].startswith("_")
-        ]
-        cf_combined.extend(classes_method)
-        cf_combined.sort(key=lambda x: x[0])
-        cf_combined = [
-            x + (f"{lib_name}.{x[0]}",)
-            for x in cf_combined
-            if x[0] not in self.skip_list
-        ]
-
-        # Render the index templates and write rendered output to files
-        index_temp = env.get_template("index.rst")
-
-        with open(os.path.join(self.OUTPUT, "index.rst"), "w") as f:
-            t = index_temp.render({"modules": [(lib_name, "")] + modules})
-            f.write(t)
-
-        # Render the function templates and write rendered output to files
-        func_temp = env.get_template("function.rst")
-
-        with open(os.path.join(self.OUTPUT, f"{lib_name}.rst"), "w") as f:
-            t = func_temp.render({"module_name": lib_name, "funcs": cf_combined})
-
-            f.write(t)
-
-        # Iterate the modules, render the function templates and write rendered output to files
-        for module in modules:
-            module_name = module[0]
-
-            if module_name in self.expand_func_list:
-                ms_func = [
-                    x for x in inspect.getmembers(module[1]) if inspect.isfunction(x[1])
-                ]
-                ms.extend(ms_func)
-            elif module_name not in self.add_root_list:
-                ms = [
-                    x for x in inspect.getmembers(module[1]) if inspect.ismodule(x[1])
-                ]
-            else:
-                ms = []
-
-            classes = [
-                x for x in inspect.getmembers(module[1]) if inspect.isclass(x[1]) and x[1].__module__.startswith(f"{lib_name}.{module_name}")
-            ]
-            ms.extend(classes)
-            class_method = [
-                (f"{y[0]}.{x[0]}", x) for y in classes for x in inspect.getmembers(y[1]) if inspect.isfunction(x[1]) and not x[0].startswith("_")
-            ]
-            ms.extend(class_method)
-
-            ms = [
-                x + (f"{lib_name}.{module_name}.{x[0]}",)
-                for x in ms
-                if x[0] not in self.skip_list
-            ]
-            t = func_temp.render({"module_name": module[0], "funcs": ms})
-
-            with open(os.path.join(self.OUTPUT, f"{module_name}.rst"), "w") as f:
-                f.write(t)
-
-
-if __name__ == "__main__":
-    gen = DocGen(skip_list=[
-        "processor.context",
-        "adapter.adapter",
-    ])
-    gen.generate("gptcache")
diff --git a/docs/_exts/docgen2.py b/docs/_exts/docgen2.py
new file mode 100644
index 00000000..20497def
--- /dev/null
+++ b/docs/_exts/docgen2.py
@@ -0,0 +1,106 @@
+import os
+import sys
+
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+_default_skip_file_list = ["__init__.py"]
+_default_skip_dir_list = ["__pycache__"]
+
+
+class DocGen:
+    def __init__(self, lib_name="gptcache", source_dir="../gptcache", output_dir="references", skip_list=[]):
+        self.lib_name = lib_name
+        self.output_dir = os.path.abspath(output_dir)
+        self.source_dir = os.path.abspath(source_dir)
+        self.skip_list = skip_list
+
+    @staticmethod
+    def title_bar(input_str):
+        return "=" * len(input_str)
+
+    @staticmethod
+    def section_bar(input_str):
+        return "-" * len(input_str)
+
+    @staticmethod
+    def get_filename(input_str):
+        if input_str == "gptcache":
+            return input_str
+        input_str = os.path.splitext(input_str)[1][1:]
+        return input_str
+
+    @staticmethod
+    def cap(input_str):
+        input_str = DocGen.get_filename(input_str)
+        if input_str == "gptcache":
+            return "GPTCache"
+        return str.join(" ", [i.capitalize() for i in input_str.split("_")])
+
+    def model_name(self, input_str: str):
+        return self.lib_name + input_str[len(self.source_dir):].replace("/", ".")
+
+    def get_module_and_libs(self, module_dir, is_root):
+        module = self.model_name(module_dir)
+        libs = []
+        for file in os.listdir(module_dir):
+            if os.path.isfile(os.path.join(module_dir, file)) and file not in _default_skip_file_list:
+                libs.append(module + "." + os.path.splitext(file)[0])
+            if not is_root:
+                if os.path.isdir(os.path.join(module_dir, file)) and file not in _default_skip_dir_list:
+                    _, child_libs = self.get_module_and_libs(os.path.join(module_dir, file), False)
+                    libs.extend(child_libs)
+        if len(libs) > 0:
+            sorted(libs)
+            return module, libs
+        return "", []
+
+    def generate(self):
+        # Set the output directory
+        env = Environment(
+            loader=FileSystemLoader(os.path.join(self.output_dir, "../_templates")), autoescape=select_autoescape()
+        )
+
+        # Add custom filters
+        env.filters["title_bar"] = DocGen.title_bar
+        env.filters["section_bar"] = DocGen.section_bar
+        env.filters["cap"] = DocGen.cap
+
+        # Add the target path to the system path
+        sys.path.insert(0, os.path.abspath(".."))
+
+        # Load the modules
+        modules = []
+        libs = []
+
+        a, b = self.get_module_and_libs(self.source_dir, True)
+        if a:
+            modules.append(a)
+            libs.append(b)
+
+        for file in os.listdir(self.source_dir):
+            tmp_dir = os.path.join(self.source_dir, file)
+            if os.path.isdir(tmp_dir) and file not in _default_skip_dir_list:
+                a, b = self.get_module_and_libs(tmp_dir, False)
+                if a:
+                    modules.append(a)
+                    libs.append(b)
+
+        # Render the index templates and write rendered output to files
+        index_temp = env.get_template("index.rst")
+
+        with open(os.path.join(self.output_dir, "index.rst"), "w") as f:
+            t = index_temp.render({"modules": [DocGen.get_filename(module) for module in modules]})
+            f.write(t)
+
+        # Render the function templates and write rendered output to files
+        func_temp = env.get_template("function.rst")
+
+        for index, module in enumerate(modules):
+            with open(os.path.join(self.output_dir, f"{DocGen.get_filename(module)}.rst"), "w") as f:
+                t = func_temp.render({"module_name": module, "funcs": [(DocGen.get_filename(lib), lib) for lib in libs[index]]})
+                f.write(t)
+
+
+# if __name__ == "__main__":
+#     gen = DocGen(source_dir="/Users/derek/fubang/gptcache/gptcache", output_dir="/Users/derek/fubang/gptcache/docs/references")
+#     gen.generate()
\ No newline at end of file
diff --git a/docs/_exts/indexCon.py b/docs/_exts/index_con.py
similarity index 100%
rename from docs/_exts/indexCon.py
rename to docs/_exts/index_con.py
diff --git a/docs/_templates/function.rst b/docs/_templates/function.rst
index 00310310..9b40de0a 100644
--- a/docs/_templates/function.rst
+++ b/docs/_templates/function.rst
@@ -4,9 +4,11 @@
 .. contents:: Index
 
 {% for func in funcs  -%}
-{{module_name}}.{{func[0]}}
-{{ func[2] | section_bar }}
-.. automodule:: {{func[2]}}
-    :members:
+{{func[0]}}
+{{ func[0] | section_bar }}
+.. automodule:: {{func[1]}}
+   :members:
+   :undoc-members:
+   :show-inheritance:
 
 {% endfor %}
\ No newline at end of file
diff --git a/docs/_templates/index.rst b/docs/_templates/index.rst
index 3dc77429..c0a23847 100644
--- a/docs/_templates/index.rst
+++ b/docs/_templates/index.rst
@@ -6,5 +6,5 @@
    :caption: Contents:
 
 {% for module in modules %}
-   {{ module[0] }} 
+   {{ module }}
 {%- endfor -%}
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 1af74a1a..013b3763 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,8 +16,8 @@
 sys.path.insert(0, os.path.abspath("."))
 sys.path.insert(0, os.path.abspath(".."))
 sys.path.insert(0, os.path.abspath("_exts"))
-from _exts.docgen import DocGen
-from _exts.indexCon import IndexCon
+from _exts.index_con import IndexCon
+from _exts.docgen2 import DocGen
 
 
 # -- Project information -----------------------------------------------------
@@ -51,7 +51,6 @@
     "sphinx_copybutton",
     "sphinx_panels",
     "sphinx_toolbox.collapse",
-    "sphinxcontrib.autodoc_pydantic",
 ]
 
 intersphinx_mapping = {
@@ -59,18 +58,19 @@
     "numpy": ("https://numpy.org/devdocs/", None),
     "python": ("https://docs.python.org/3", None),
 }
+
 autodoc_member_order = "bysource"
-autodoc_mock_imports = ["httpx"]
-autodoc_inherit_docstrings = False
-
-autodoc_pydantic_model_show_json = False
-autodoc_pydantic_field_list_validators = False
-autodoc_pydantic_config_members = False
-autodoc_pydantic_model_show_config_summary = False
-autodoc_pydantic_model_show_validator_members = False
-autodoc_pydantic_model_show_field_summary = False
-autodoc_pydantic_model_members = False
-autodoc_pydantic_model_undoc_members = False
+autodoc_mock_imports = [
+    "httpx",
+    "protobuf",
+    "minigpt4",
+    "llama_cpp",
+    "transformers",
+    "diffusers",
+    "stability_sdk",
+    "paddle",
+    "paddlenlp",
+]
 
 source_suffix = {
     ".rst": "restructuredtext",
@@ -89,9 +89,6 @@
     "_build",
     "Thumbs.db",
     ".DS_Store",
-    "references/client*",
-    "references/config*",
-    "references/core*",
 ]
 
 
@@ -131,7 +128,7 @@
 # -- Preactions --------------------------------------------------------------
 
 # Prepare docs
-docgen = DocGen(output_dir="references")
-docgen.generate("gptcache")
+docgen = DocGen()
+docgen.generate()
 
 IndexCon("../README.md", "index.rst")
diff --git a/docs/references/index.rst b/docs/references/index.rst
index 0abbc6a2..8bc34293 100644
--- a/docs/references/index.rst
+++ b/docs/references/index.rst
@@ -7,14 +7,9 @@
 
 
    gptcache
-   adapter
-   client
-   config
-   core
+   processor
    embedding
+   utils
+   adapter
    manager
-   processor
-   report
-   session
-   similarity_evaluation
-   utils
\ No newline at end of file
+   similarity_evaluation
\ No newline at end of file
diff --git a/docs/requirements.txt b/docs/requirements.txt
index bccfc1b1..8cd2b710 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -7,11 +7,7 @@ autodoc_pydantic
 myst_nb
 sphinx-autobuild
 sphinx_book_theme
-sphinx_rtd_theme
-sphinx-typlog-theme
 sphinx-panels
 sphinx_copybutton
-pydata-sphinx-theme
 m2r2
 sphinx_toolbox
-protobuf==3.20.0
diff --git a/examples/README.md b/examples/README.md
index c1238d72..db3a7517 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -595,7 +595,7 @@ config:
     similarity_threshold: 0.8
     # Set other config here
 ```
-- model_src: The model source.
+- model_source: The model source.
 - model_config: The model name, model config, api key.
 - data_dir: The cache directory.
 - manager: The cache storage and vector storage.
diff --git a/gptcache/__init__.py b/gptcache/__init__.py
index 9aab8b7a..9ce7038e 100644
--- a/gptcache/__init__.py
+++ b/gptcache/__init__.py
@@ -1,3 +1,6 @@
+"""gptcache version"""
+__version__ = "0.1.28"
+
 from gptcache.client import Client
 from gptcache.config import Config
 from gptcache.core import Cache
diff --git a/gptcache/adapter/api.py b/gptcache/adapter/api.py
index 0cd5f883..3efabc6d 100644
--- a/gptcache/adapter/api.py
+++ b/gptcache/adapter/api.py
@@ -29,6 +29,7 @@
     OnnxModelEvaluation,
     ExactMatchEvaluation,
     KReciprocalEvaluation,
+    SimilarityEvaluation,
 )
 from gptcache.utils import import_ruamel
 
@@ -126,6 +127,7 @@ def init_similar_cache(
     pre_func: Callable = get_prompt,
     embedding: Optional[BaseEmbedding] = None,
     data_manager: Optional[DataManager] = None,
+    evaluation: Optional[SimilarityEvaluation] = None,
     post_func: Callable = temperature_softmax,
     config: Config = Config(),
 ):
@@ -141,6 +143,8 @@ def init_similar_cache(
     :type embedding: BaseEmbedding
     :param data_manager: data manager object
     :type data_manager: DataManager
+    :param evaluation: similarity evaluation object
+    :type evaluation: SimilarityEvaluation
     :param post_func: post-processing of the cached result list, the most similar result is taken by default
     :type post_func: Callable[[List[Any]], Any]
     :param config: cache configuration, the core is similar threshold
@@ -164,7 +168,8 @@ def init_similar_cache(
             data_dir=data_dir,
             vector_params={"dimension": embedding.dimension},
         )
-    evaluation = SearchDistanceEvaluation()
+    if not evaluation:
+        evaluation = SearchDistanceEvaluation()
     cache_obj = cache_obj if cache_obj else cache
     cache_obj.init(
         pre_embedding_func=pre_func,
diff --git a/gptcache/adapter/openai.py b/gptcache/adapter/openai.py
index 8fd19538..7a8d0b7b 100644
--- a/gptcache/adapter/openai.py
+++ b/gptcache/adapter/openai.py
@@ -286,14 +286,14 @@ class Moderation(openai.Moderation):
     Example:
         .. code-block:: python
 
-        from gptcache.adapter import openai
-        from gptcache.adapter.api import init_similar_cache
-        from gptcache.processor.pre import get_openai_moderation_input
+            from gptcache.adapter import openai
+            from gptcache.adapter.api import init_similar_cache
+            from gptcache.processor.pre import get_openai_moderation_input
 
-        init_similar_cache(pre_func=get_openai_moderation_input)
-        openai.Moderation.create(
-            input="I want to kill them.",
-        )
+            init_similar_cache(pre_func=get_openai_moderation_input)
+            openai.Moderation.create(
+                input="I want to kill them.",
+            )
     """
 
     @classmethod
diff --git a/gptcache/core.py b/gptcache/core.py
index 74e45f77..313801e9 100644
--- a/gptcache/core.py
+++ b/gptcache/core.py
@@ -47,10 +47,12 @@ def init(
         self,
         cache_enable_func=cache_all,
         pre_embedding_func=last_content,
+        pre_func=None,
         embedding_func=string_embedding,
         data_manager: DataManager = get_data_manager(),
         similarity_evaluation=ExactMatchEvaluation(),
         post_process_messages_func=temperature_softmax,
+        post_func=None,
         config=Config(),
         next_cache=None,
     ):
@@ -58,20 +60,22 @@ def init(
 
         :param cache_enable_func: a function to enable cache, defaults to ``cache_all``
         :param pre_embedding_func: a function to preprocess embedding, defaults to ``last_content``
+        :param pre_func: a function to preprocess embedding, same as ``pre_embedding_func``
         :param embedding_func: a function to extract embeddings from requests for similarity search, defaults to ``string_embedding``
         :param data_manager: a ``DataManager`` module, defaults to ``get_data_manager()``
         :param similarity_evaluation: a module to calculate embedding similarity, defaults to ``ExactMatchEvaluation()``
         :param post_process_messages_func: a function to post-process messages, defaults to ``temperature_softmax`` with a default temperature of 0.0
+        :param post_func: a function to post-process messages, same as ``post_process_messages_func``
         :param config: a module to pass configurations, defaults to ``Config()``
         :param next_cache: customized method for next cache
         """
         self.has_init = True
         self.cache_enable_func = cache_enable_func
-        self.pre_embedding_func = pre_embedding_func
+        self.pre_embedding_func = pre_func if pre_func else pre_embedding_func
         self.embedding_func = embedding_func
         self.data_manager: DataManager = data_manager
         self.similarity_evaluation = similarity_evaluation
-        self.post_process_messages_func = post_process_messages_func
+        self.post_process_messages_func = post_func if post_func else post_process_messages_func
         self.config = config
         self.next_cache = next_cache
 
diff --git a/gptcache/similarity_evaluation/similarity_evaluation.py b/gptcache/similarity_evaluation/similarity_evaluation.py
index 7089e82e..a1aea1ad 100644
--- a/gptcache/similarity_evaluation/similarity_evaluation.py
+++ b/gptcache/similarity_evaluation/similarity_evaluation.py
@@ -9,6 +9,7 @@ class SimilarityEvaluation(metaclass=ABCMeta):
 
     Example:
         .. code-block:: python
+
             from gptcache import cache
             from gptcache.similarity_evaluation import SearchDistanceEvaluation
 
diff --git a/setup.py b/setup.py
index 5a220809..5cfeda12 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,14 @@
+import codecs
+import os
+import re
 from typing import List
 
 import setuptools
 from setuptools import find_packages
 
+here = os.path.abspath(os.path.dirname(__file__))
+
+
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
@@ -15,10 +21,23 @@ def parse_requirements(file_name: str) -> List[str]:
         ]
 
 
+def read(*parts):
+    with codecs.open(os.path.join(here, *parts), "r") as fp:
+        return fp.read()
+
+
+def find_version(*file_paths):
+    version_file = read(*file_paths)
+    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+    if version_match:
+        return version_match.group(1)
+    raise RuntimeError("Unable to find version string.")
+
+
 setuptools.setup(
     name="gptcache",
     packages=find_packages(),
-    version="0.1.27",
+    version=find_version("gptcache", "__init__.py"),
     author="SimFG",
     author_email="bang.fu@zilliz.com",
     description="GPTCache, a powerful caching library that can be used to speed up and lower the cost of chat "