Merge pull request #309 from StampyAI/autoposting-least-recently-edited

Autoposting least recently edited

README.md

@@ -74,6 +74,7 @@ Not required:
 - `STOP_ON_ERROR`: Dockerfile/`runstampy` only, unset `BOT_REBOOT` only. If defined, will only restart Stampy when he gets told to reboot, returning exit code 42. Any other exit code will cause the script to just stop.
 - `BE_SHY`: Stamp won't respond when the message isn't specifically to him.
 - `CHANNEL_WHITELIST`: channels Stampy is allowed to respond to messages in
+- `NOT_ROB_SERVER`: If `True`, Rob Miles server-specific stuff is disabled. Servers other than Rob Miles Discord Server and Stampy Test Server should set it to `1`. Otherwise some errors are likely to occur.
 
 Specific modules (excluding LLM stuff):
 

api/coda.py

@@ -14,12 +14,12 @@
     parse_question_row,
     QuestionRow,
     QuestionStatus,
-    DEFAULT_DATE,
 )
 from config import ENVIRONMENT_TYPE, coda_api_token
 from utilities import is_in_testing_mode, Utilities
 from utilities.discordutils import DiscordUser
 from utilities.serviceutils import ServiceMessage
+from utilities.time_utils import DEFAULT_DATE
 from utilities.utilities import fuzzy_contains, get_user_handle, shuffle_df
 
 if TYPE_CHECKING:
@@ -57,19 +57,8 @@ def __init__(self):
         self.class_name = "Coda API"
         self.log = get_logger()
         self.last_question_id: Optional[str] = None
-        self.questions_df = pd.DataFrame(
-            columns=[
-                "id",
-                "title",
-                "url",
-                "status",
-                "tags",
-                "last_asked_on_discord",
-                "alternate_phrasings",
-                "row",
-            ]
-        )
-
+        # pylint:disable=no-member
+        self.questions_df = pd.DataFrame(columns=list(QuestionRow.__required_keys__))  # fmt:skip
         if is_in_testing_mode():
             return
 
@@ -170,15 +159,17 @@ def update_questions_cache(self) -> tuple[list[QuestionRow], list[QuestionRow]]:
             if row["id"] not in self.questions_df.index:
                 new_questions.append(row)
             else:
-                df_row = self.questions_df.loc[row["id"]]
-                df_row["last_asked_on_discord"] = row["last_asked_on_discord"]
-                df_row["status"] = row["status"]
-                df_row["tags"].clear()
-                df_row["tags"].extend(row["tags"])
-                df_row["alternate_phrasings"].clear()
-                df_row["alternate_phrasings"].extend(row["alternate_phrasings"])
-                df_row["title"] = row["title"]
-                df_row["url"] = row["url"]
+                self.questions_df.at[row["id"], "title"] = row["title"]
+                self.questions_df.at[row["id"], "url"] = row["url"]
+                self.questions_df.at[row["id"], "status"] = row["status"]
+
+                self.questions_df.at[row["id"], "tags"].clear()
+                self.questions_df.at[row["id"], "tags"].extend(row["tags"])
+                self.questions_df.at[row["id"], "alternate_phrasings"].clear()
+                self.questions_df.at[row["id"], "alternate_phrasings"].extend(row["alternate_phrasings"])  # fmt:skip
+
+                self.questions_df.at[row["id"], "doc_last_edited"] = row["doc_last_edited"]  # fmt:skip
+                self.questions_df.at[row["id"], "last_asked_on_discord"] = row["last_asked_on_discord"]  # fmt:skip
 
         deleted_question_ids = sorted(
             set(self.questions_df.index.tolist()) - question_ids
@@ -230,9 +221,8 @@ def get_questions_by_gdoc_links(self, urls: list[str]) -> list[QuestionRow]:
             )
         ]
 
-        questions_queried = cast(
-            list[QuestionRow], questions_df_queried.to_dict(orient="records")
-        )
+        questions_queried = self.q_df_to_rows(questions_df_queried)
+
         # If some links were not recognized, refresh cache and look into the new questions
         if len(questions_df_queried) < len(urls):
             new_questions, _ = self.update_questions_cache()
@@ -276,7 +266,7 @@ def update_question_status(
             question["row"], make_updated_cells({"Status": status})
         )
         # update local cache
-        self.questions_df.loc[question["id"]]["status"] = status
+        self.questions_df.at[question["id"], "status"] = status
 
     def update_question_last_asked_date(
         self, question: QuestionRow, current_time: datetime
@@ -290,7 +280,7 @@ def update_question_last_asked_date(
             make_updated_cells({"Last Asked On Discord": current_time.isoformat()}),
         )
         # update local cache
-        self.questions_df.loc[question["id"]]["last_asked_on_discord"] = current_time
+        self.questions_df.at[question["id"], "last_asked_on_discord"] = current_time
 
     def _reset_dates(self) -> None:
         """Reset all questions' dates (util, not to be used by Stampy)"""
@@ -306,8 +296,8 @@ def update_question_tags(self, question: QuestionRow, new_tags: list[str]) -> No
         self.doc.get_table(self.STAMPY_ANSWERS_API_ID).update_row(
             question["row"], make_updated_cells({"Tags": new_tags})
         )
-        self.questions_df.loc[question["id"]]["tags"].clear()
-        self.questions_df.loc[question["id"]]["tags"].extend(new_tags)
+        self.questions_df.at[question["id"], "tags"].clear()
+        self.questions_df.at[question["id"], "tags"].extend(new_tags)
         self.last_question_id = question["id"]
 
     # Alternate phrasings
@@ -318,10 +308,8 @@ def update_question_altphr(
         self.doc.get_table(self.STAMPY_ANSWERS_API_ID).update_row(
             question["row"], make_updated_cells({"Alternate Phrasings": new_alt_phrs})
         )
-        self.questions_df.loc[question["id"]]["alternate_phrasings"].clear()
-        self.questions_df.loc[question["id"]]["alternate_phrasings"].extend(
-            new_alt_phrs
-        )
+        self.questions_df.at[question["id"], "alternate_phrasings"].clear()
+        self.questions_df.at[question["id"], "alternate_phrasings"].extend(new_alt_phrs)
         self.last_question_id = question["id"]
 
     ###############
@@ -398,7 +386,7 @@ async def query_for_questions(
         if least_recently_asked_unpublished:
             questions_df = questions_df.query("status != 'Live on site'")
         elif status is not None:
-            questions_df = questions_df.query("status == @status")
+            questions_df = questions_df.query(f"status == '{status}'")
 
         # if tag was specified, filter for questions having that tag
         questions_df = filter_on_tag(questions_df, tag)
@@ -419,8 +407,7 @@ async def query_for_questions(
         questions_df = questions_df.sort_values("last_asked_on_discord", ascending=False).iloc[: min(limit, 5)]  # fmt:skip
         if questions_df.empty:
             return []
-        questions = questions_df.to_dict(orient="records")
-        return cast(list[QuestionRow], questions)
+        return self.q_df_to_rows(questions_df)
 
     ResponseText = ResponseWhy = str
 
@@ -549,6 +536,10 @@ def get_all_statuses(self) -> list[str]:
             raise AssertionError(msg)
         return sorted(coda_status_vals)
 
+    @staticmethod
+    def q_df_to_rows(questions_df: pd.DataFrame) -> list[QuestionRow]:
+        return cast(list[QuestionRow], questions_df.to_dict(orient="records"))
+
 
 def filter_on_tag(questions_df: pd.DataFrame, tag: Optional[str]) -> pd.DataFrame:
     if tag is None:
@@ -564,6 +555,5 @@ def get_least_recently_asked_on_discord(
     questions: pd.DataFrame,
 ) -> pd.DataFrame:
     """Get all questions with oldest date and shuffle them"""
-    # pylint:disable=unused-variable
     oldest_date = questions["last_asked_on_discord"].min()
-    return questions.query("last_asked_on_discord == @oldest_date")
+    return questions.query(f"last_asked_on_discord == '{oldest_date}'")

api/utilities/coda_utils.py

@@ -5,17 +5,7 @@
 
 from codaio import Cell, Row
 
-
-DEFAULT_DATE = datetime(1, 1, 1, 0)
-
-
-def adjust_date(date_str: str) -> datetime:
-    """If date is in isoformat, parse it.
-    Otherwise, assign earliest date possible.
-    """
-    if not date_str:
-        return DEFAULT_DATE
-    return datetime.fromisoformat(date_str.split("T")[0])
+from utilities.time_utils import adjust_date
 
 
 def parse_question_row(row: Row) -> QuestionRow:
@@ -29,6 +19,7 @@ def parse_question_row(row: Row) -> QuestionRow:
     # remove empty strings
     tags = [tag for tag in row_dict["Tags"].split(",") if tag]
     last_asked_on_discord = adjust_date(row_dict["Last Asked On Discord"])
+    doc_last_edited = adjust_date(row_dict["Doc Last Edited"])
     alternate_phrasings = [
         alt for alt in row_dict["Alternate Phrasings"].split(",") if alt
     ]
@@ -38,8 +29,9 @@ def parse_question_row(row: Row) -> QuestionRow:
         "url": url,
         "status": status,
         "tags": tags,
-        "last_asked_on_discord": last_asked_on_discord,
         "alternate_phrasings": alternate_phrasings,
+        "last_asked_on_discord": last_asked_on_discord,
+        "doc_last_edited": doc_last_edited,
         "row": row,
     }
 
@@ -62,8 +54,9 @@ class QuestionRow(TypedDict):
     url: str
     status: str
     tags: list[str]
-    last_asked_on_discord: datetime
     alternate_phrasings: list[str]
+    last_asked_on_discord: datetime
+    doc_last_edited: datetime
     row: Row
 
 
@@ -87,3 +80,9 @@ class QuestionRow(TypedDict):
     "duplicated": "Duplicate",
     "published": "Live on site",
 }
+
+REVIEW_STATUSES: set[QuestionStatus] = {
+    "Bulletpoint sketch",
+    "In progress",
+    "In review",
+}

build_help.py

@@ -1,67 +1,12 @@
-import os
 from pathlib import Path
-import re
-from textwrap import dedent
-from typing import Optional
 
-from utilities.help_utils import ModuleHelp
-
-MODULES_PATH = Path("modules/")
-
-ModuleName = Docstring = str
-
-FILE_HEADER = dedent(
-    """\
-    # Stampy Module & Command Help
-    
-    This file was auto-generated from file-level docstrings in `modules` directory. If you think it's out of sync with docstrings, re-generate it by calling `python build_help.py`. If docstrings are out of date with code, feel free to edit them or nag somebody with a `@Stampy dev` on the server.
-    
-    """
-)
-
-
-def extract_docstring(code: str) -> Optional[Docstring]:
-    if not (code.startswith('"""') and '"""' in code[3:]):
-        return
-    docstring_end_pos = code.find('"""', 3)
-    assert docstring_end_pos != -1
-    docstring = code[3:docstring_end_pos].strip()
-    assert docstring
-    return docstring
-
-
-_re_module_name = re.compile(r"(?<=class\s)\w+(?=\(Module\):)", re.I)
-
-
-def extract_module_name(code: str) -> Optional[ModuleName]:
-    if match := _re_module_name.search(code):
-        return match.group()
-
-
-def load_modules_with_docstrings() -> dict[ModuleName, Docstring]:
-    modules_with_docstrings = {}
-    for fname in os.listdir(MODULES_PATH):
-        if fname.startswith("_") or fname == "module.py":
-            continue
-        with open(MODULES_PATH / fname, "r", encoding="utf-8") as f:
-            code = f.read()
-        if (module_name := extract_module_name(code)) and (
-            docstring := extract_docstring(code)
-        ):
-            modules_with_docstrings[module_name] = docstring
-    return modules_with_docstrings
+from utilities.help_utils import build_help_md
 
 
 def main() -> None:
-    modules_with_docstrings = load_modules_with_docstrings()
-    helps = []
-    for module_name, docstring in modules_with_docstrings.items():
-        help = ModuleHelp.from_docstring(module_name, docstring)
-        if not help.empty:
-            helps.append(help.get_module_help(markdown=True))
-    help_txt = FILE_HEADER + "\n\n".join(helps)
+    help_md = build_help_md(Path("modules"))
     with open("help.md", "w", encoding="utf-8") as f:
-        f.write(help_txt)
+        f.write(help_md)
 
 
 if __name__ == "__main__":

config.py

@@ -1,5 +1,5 @@
 import os
-from typing import Literal, Optional, TypeVar, Union, cast, get_args, overload, Any
+from typing import Literal, TypeVar, Optional, Union, cast, get_args, overload
 
 import dotenv
 from structlog import get_logger
@@ -51,18 +51,19 @@ def getenv_bool(env_var: str) -> bool:
     return e != "UNDEFINED"
 
 
-Tt = TypeVar("Tt")
+# fmt:off
 @overload
 def getenv_unique_set(var_name: str) -> frozenset:...
 @overload
 def getenv_unique_set(var_name: str, default: frozenset) -> frozenset:...
 @overload
-def getenv_unique_set(var_name: str, default: Literal[None]) \
-        -> Optional[frozenset[str]]:...
+def getenv_unique_set(var_name: str, default: None) -> Optional[frozenset[str]]:...
 @overload
-def getenv_unique_set(var_name: str, default: Tt) \
-        -> Union[frozenset[str], Tt]:...
-def getenv_unique_set(var_name: str, default = frozenset()):
+def getenv_unique_set(var_name: str, default: T) -> Union[frozenset[str], T]:...
+# fmt:on
+
+
+def getenv_unique_set(var_name: str, default: T = frozenset()) -> Union[frozenset, T]:
     l = getenv(var_name, default="EMPTY_SET").split(" ")
     if l == ["EMPTY_SET"]:
         return default
@@ -217,7 +218,9 @@ def getenv_unique_set(var_name: str, default = frozenset()):
     use_helicone = getenv_bool("USE_HELICONE")
     llm_prompt = getenv("LLM_PROMPT", default=stampy_default_prompt)
     be_shy = getenv_bool("BE_SHY")
-    channel_whitelist: Optional[frozenset[str]] = getenv_unique_set("CHANNEL_WHITELIST", None)
+    channel_whitelist: Optional[frozenset[str]] = getenv_unique_set(
+        "CHANNEL_WHITELIST", None
+    )
 
 discord_token = getenv("DISCORD_TOKEN")
 database_path = getenv("DATABASE_PATH")
@@ -227,6 +230,10 @@ def getenv_unique_set(var_name: str, default = frozenset()):
 slack_app_token = getenv("SLACK_APP_TOKEN", default=None)
 slack_bot_token = getenv("SLACK_BOT_TOKEN", default=None)
 
+not_rob_server = getenv_bool("NOT_ROB_SERVER")
+is_rob_server = not not_rob_server
+
+
 # VARIABLE VALIDATION
 bot_reboot_options = frozenset(["exec", False])
 assert (

modules/HelpModule.py

@@ -15,7 +15,6 @@
 from textwrap import dedent
 
 from modules.module import IntegrationTest, Module, Response
-from utilities.help_utils import ModuleHelp
 from utilities.serviceutils import ServiceMessage
 
 
@@ -29,7 +28,6 @@ class HelpModule(Module):
 
     def __init__(self):
         super().__init__()
-        self.help = ModuleHelp.from_docstring(self.class_name, __doc__)
         self.re_help = re.compile(r"help \w+", re.I)
 
     def process_message(self, message: ServiceMessage) -> Response: