Merge pull request #9 from pingcap-inc/feat-log

feat: update logging funciton

app/config/__init__.py

@@ -21,6 +21,7 @@ def __init__(self):
         load_dotenv()
 
         self.debug = bool(os.getenv("DEBUG", "True") == "True")
+        self.log_path = os.getenv("LOG_PATH", "./transforum.log")
         self.sleep_time = int(os.getenv("SLEEP_TIME", "30"))
 
         self.open_ai_base_url = os.getenv("OPEN_AI_BASE_URL", "https://api.openai.com/v1")

app/db/__init__.py

@@ -16,6 +16,9 @@
 from app.config import conf
 from sqlalchemy.orm import Session
 from typing import Callable, Any
+from app.log import getLogger
+
+logger = getLogger(__name__)
 
 
 def get_db_url():
@@ -54,7 +57,7 @@ def wrapper(*args, **kwargs):
                 session.commit()
             except Exception as e:
                 session.rollback()
-                print(f"Exec and an error occurred: {e}")
+                logger.error(f"Exec and an error occurred: {e}")
                 raise
     return wrapper
 
@@ -68,7 +71,7 @@ def wrapper(*args, **kwargs) -> Any:
                 return result
             except Exception as e:
                 session.rollback()
-                print(f"Query and an error occurred: {e}")
+                logger.error(f"Query and an error occurred: {e}")
                 raise
     return wrapper
 

app/forum/get.py

@@ -20,6 +20,9 @@
 from app.db import db_exec
 from dataclasses import fields
 from retrying import retry
+from app.log import getLogger
+
+logger = getLogger(__name__)
 
 
 def get_topic_and_post_ids(topic_id: int) -> (CnTopics, List[int]):
@@ -95,7 +98,9 @@ def get_and_save_page_sync_progress(page: int, earliest: datetime) -> bool:
     topics = list(
         filter(lambda t: datetime.datetime.strptime(t.created_at, '%Y-%m-%dT%H:%M:%S.%fZ') > earliest, topics))
     save_page_topic_ids(topics)
-    print(f"Got {len(topics)} topics in page {page}!")
+
+    logger.info(f"Got {len(topics)} topics in page {page}!")
+
     return len(topics) != 0
 
 

app/log/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 2022 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+import logging
+from app.config import conf
+
+
+def getLogger(logger_name: str) -> logging.Logger:
+    logger = logging.getLogger(logger_name)
+    logLevel = logging.DEBUG if conf.debug else logging.INFO
+    logger.setLevel(logLevel)
+
+    # Create handlers for logging to the standard output and a file
+    stdoutHandler = logging.StreamHandler(stream=sys.stdout)
+    fileHandler = logging.FileHandler(conf.log_path)
+
+    # Set the log levels on the handlers
+    stdoutHandler.setLevel(logLevel)
+    fileHandler.setLevel(logLevel)
+
+    # Create a log format using Log Record attributes
+    fmt = logging.Formatter(
+        "%(name)s: %(asctime)s | %(levelname)s | %(filename)s:%(lineno)s | %(process)d >>> %(message)s"
+    )
+
+    # Set the log format on each handler
+    stdoutHandler.setFormatter(fmt)
+    fileHandler.setFormatter(fmt)
+
+    # Add each handler to the Logger object
+    logger.addHandler(stdoutHandler)
+    logger.addHandler(fileHandler)
+
+    return logger

app/service/__init__.py

@@ -17,24 +17,27 @@
 from app.service.translate import translate_task, task_error, translate_or_update_first_page
 import threading
 from app.config import conf
+from app.log import getLogger
 
+logger = getLogger(__name__)
 
-def thread_loop():
-    while True:
-        try:
-            incremental_loop()
-        except Exception as e:
-            print(f"[Error {datetime.datetime.now()}] translate_or_update_first_page() {e}")
-
-        if conf.sleep_time != 0:
-            time.sleep(conf.sleep_time)
-
-
-def incremental_loop():
-    translate_or_update_first_page()
-
-
-thread = threading.Thread(target=thread_loop)
-thread.setDaemon(True)
-thread.start()
-print(f"Sync task is running now.")
+#
+# def thread_loop():
+#     while True:
+#         try:
+#             incremental_loop()
+#         except Exception as e:
+#             logger.error(f"[Error {datetime.datetime.now()}] translate_or_update_first_page() {e}")
+#
+#         if conf.sleep_time != 0:
+#             time.sleep(conf.sleep_time)
+#
+#
+# def incremental_loop():
+#     translate_or_update_first_page()
+#
+#
+# thread = threading.Thread(target=thread_loop)
+# thread.setDaemon(True)
+# thread.start()
+# logger.info(f"Sync task is running now.")

app/service/translate.py

@@ -25,10 +25,9 @@
 from datetime import datetime
 from retrying import retry
 import re
-import logging
+from app.log import getLogger
 
-
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
 class Operator(Enum):
@@ -42,7 +41,7 @@ def translate_topic(topic_id: int):
     topic, post_ids = get_topic_and_post_ids(topic_id)
     topic_op = translate_and_save_topic(topic)
 
-    print(f"Start to deal with topic {topic.id}")
+    logger.info(f"Start to deal with topic {topic.id}")
 
     if topic_op == Operator.Create:
         posts = [get_post(post_id) for post_id in post_ids]
@@ -67,7 +66,7 @@ def translate_topic(topic_id: int):
                 if posts[i].accepted_answer:
                     client.solve_solution(posts[i].en_id)
 
-        print(f"Create topic {topic.id} successfully, en id: {topic.en_id}")
+        logger.info(f"Create topic {topic.id} successfully, en id: {topic.en_id}")
     else:
         if topic_op == Operator.Update:
             update_topic(topic)
@@ -80,11 +79,11 @@ def translate_topic(topic_id: int):
                 update_post(topic, posts[i])
             elif post_ops[i] == Operator.Create:
                 post_result = create_post(topic, posts[i])
-                print(f"post_result: {post_result}")
+                logger.debug(f"post_result: {post_result}")
                 posts[i].en_id = post_result['id']
                 update_obj(posts[i])
 
-        print(f"Update topic {topic.id} successfully")
+        logger.info(f"Update topic {topic.id} successfully")
 
     return topic
 
@@ -166,7 +165,7 @@ def find_link_from_cooked(cooked: str, sha: str) -> str:
         if len(src) == 0:
             return ""
         else:
-            print(str(src[0]))
+            logger.debug(str(src[0]))
             return str(src[0])
 
     src_set = str(src_sets[0])
@@ -200,7 +199,7 @@ def query_progress_and_update_state_to_translating(session: Session):
         .order_by(SyncProgress.cn_created_at.desc()).first()
 
     if progress is None:
-        print("All synchronized")
+        logger.debug("All synchronized")
         return None
 
     progress.translate_state = 1
@@ -219,21 +218,21 @@ def save_progress(session: Session, sync_progress, topic):
 
 def translate_task(wait_when_none: int = 2):
     sync_progress = query_progress_and_update_state_to_translating()
-    print(f"sync_progress: {sync_progress}")
+    logger.info(f"sync_progress: {sync_progress}")
 
     if sync_progress is None:
         if wait_when_none != 0:
             time.sleep(wait_when_none)
         return
 
     start_time = datetime.now()
-    print(f"[{start_time.strftime('%Y-%m-%d %H:%M:%S')}] get {sync_progress}")
+    logger.info(f"get {sync_progress}")
 
     topic = translate_topic(sync_progress.cn_topic_id)
     save_progress(sync_progress, topic)
 
     delta = datetime.now() - start_time
-    print(f"[{delta.seconds}s] merged {sync_progress}")
+    logger.info(f"[{delta.seconds}s] merged {sync_progress}")
 
     return sync_progress