From 39235e48533cd2de86521fbaaa543e6343ebbeeb Mon Sep 17 00:00:00 2001 From: Icemap Date: Mon, 24 Jun 2024 16:20:42 +0800 Subject: [PATCH] feat: update logging funciton --- app/config/__init__.py | 1 + app/db/__init__.py | 7 ++++-- app/forum/get.py | 7 +++++- app/log/__init__.py | 47 ++++++++++++++++++++++++++++++++++++++++ app/service/__init__.py | 41 +++++++++++++++++++---------------- app/service/translate.py | 23 ++++++++++---------- 6 files changed, 92 insertions(+), 34 deletions(-) create mode 100644 app/log/__init__.py diff --git a/app/config/__init__.py b/app/config/__init__.py index 2cde373..e1d0c22 100644 --- a/app/config/__init__.py +++ b/app/config/__init__.py @@ -21,6 +21,7 @@ def __init__(self): load_dotenv() self.debug = bool(os.getenv("DEBUG", "True") == "True") + self.log_path = os.getenv("LOG_PATH", "./transforum.log") self.sleep_time = int(os.getenv("SLEEP_TIME", "30")) self.open_ai_base_url = os.getenv("OPEN_AI_BASE_URL", "https://api.openai.com/v1") diff --git a/app/db/__init__.py b/app/db/__init__.py index c1b66d7..98238f6 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -16,6 +16,9 @@ from app.config import conf from sqlalchemy.orm import Session from typing import Callable, Any +from app.log import getLogger + +logger = getLogger(__name__) def get_db_url(): @@ -54,7 +57,7 @@ def wrapper(*args, **kwargs): session.commit() except Exception as e: session.rollback() - print(f"Exec and an error occurred: {e}") + logger.error(f"Exec and an error occurred: {e}") raise return wrapper @@ -68,7 +71,7 @@ def wrapper(*args, **kwargs) -> Any: return result except Exception as e: session.rollback() - print(f"Query and an error occurred: {e}") + logger.error(f"Query and an error occurred: {e}") raise return wrapper diff --git a/app/forum/get.py b/app/forum/get.py index 290a8b1..100fa47 100644 --- a/app/forum/get.py +++ b/app/forum/get.py @@ -20,6 +20,9 @@ from app.db import db_exec from dataclasses import fields from retrying import retry +from app.log import getLogger + +logger = getLogger(__name__) def get_topic_and_post_ids(topic_id: int) -> (CnTopics, List[int]): @@ -95,7 +98,9 @@ def get_and_save_page_sync_progress(page: int, earliest: datetime) -> bool: topics = list( filter(lambda t: datetime.datetime.strptime(t.created_at, '%Y-%m-%dT%H:%M:%S.%fZ') > earliest, topics)) save_page_topic_ids(topics) - print(f"Got {len(topics)} topics in page {page}!") + + logger.info(f"Got {len(topics)} topics in page {page}!") + return len(topics) != 0 diff --git a/app/log/__init__.py b/app/log/__init__.py new file mode 100644 index 0000000..05786b5 --- /dev/null +++ b/app/log/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2022 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +import logging +from app.config import conf + + +def getLogger(logger_name: str) -> logging.Logger: + logger = logging.getLogger(logger_name) + logLevel = logging.DEBUG if conf.debug else logging.INFO + logger.setLevel(logLevel) + + # Create handlers for logging to the standard output and a file + stdoutHandler = logging.StreamHandler(stream=sys.stdout) + fileHandler = logging.FileHandler(conf.log_path) + + # Set the log levels on the handlers + stdoutHandler.setLevel(logLevel) + fileHandler.setLevel(logLevel) + + # Create a log format using Log Record attributes + fmt = logging.Formatter( + "%(name)s: %(asctime)s | %(levelname)s | %(filename)s:%(lineno)s | %(process)d >>> %(message)s" + ) + + # Set the log format on each handler + stdoutHandler.setFormatter(fmt) + fileHandler.setFormatter(fmt) + + # Add each handler to the Logger object + logger.addHandler(stdoutHandler) + logger.addHandler(fileHandler) + + return logger diff --git a/app/service/__init__.py b/app/service/__init__.py index cedd151..efae907 100644 --- a/app/service/__init__.py +++ b/app/service/__init__.py @@ -17,24 +17,27 @@ from app.service.translate import translate_task, task_error, translate_or_update_first_page import threading from app.config import conf +from app.log import getLogger +logger = getLogger(__name__) -def thread_loop(): - while True: - try: - incremental_loop() - except Exception as e: - print(f"[Error {datetime.datetime.now()}] translate_or_update_first_page() {e}") - - if conf.sleep_time != 0: - time.sleep(conf.sleep_time) - - -def incremental_loop(): - translate_or_update_first_page() - - -thread = threading.Thread(target=thread_loop) -thread.setDaemon(True) -thread.start() -print(f"Sync task is running now.") +# +# def thread_loop(): +# while True: +# try: +# incremental_loop() +# except Exception as e: +# logger.error(f"[Error {datetime.datetime.now()}] translate_or_update_first_page() {e}") +# +# if conf.sleep_time != 0: +# time.sleep(conf.sleep_time) +# +# +# def incremental_loop(): +# translate_or_update_first_page() +# +# +# thread = threading.Thread(target=thread_loop) +# thread.setDaemon(True) +# thread.start() +# logger.info(f"Sync task is running now.") diff --git a/app/service/translate.py b/app/service/translate.py index e5df8a7..33ae9c9 100644 --- a/app/service/translate.py +++ b/app/service/translate.py @@ -25,10 +25,9 @@ from datetime import datetime from retrying import retry import re -import logging +from app.log import getLogger - -logger = logging.getLogger(__name__) +logger = getLogger(__name__) class Operator(Enum): @@ -42,7 +41,7 @@ def translate_topic(topic_id: int): topic, post_ids = get_topic_and_post_ids(topic_id) topic_op = translate_and_save_topic(topic) - print(f"Start to deal with topic {topic.id}") + logger.info(f"Start to deal with topic {topic.id}") if topic_op == Operator.Create: posts = [get_post(post_id) for post_id in post_ids] @@ -67,7 +66,7 @@ def translate_topic(topic_id: int): if posts[i].accepted_answer: client.solve_solution(posts[i].en_id) - print(f"Create topic {topic.id} successfully, en id: {topic.en_id}") + logger.info(f"Create topic {topic.id} successfully, en id: {topic.en_id}") else: if topic_op == Operator.Update: update_topic(topic) @@ -80,11 +79,11 @@ def translate_topic(topic_id: int): update_post(topic, posts[i]) elif post_ops[i] == Operator.Create: post_result = create_post(topic, posts[i]) - print(f"post_result: {post_result}") + logger.debug(f"post_result: {post_result}") posts[i].en_id = post_result['id'] update_obj(posts[i]) - print(f"Update topic {topic.id} successfully") + logger.info(f"Update topic {topic.id} successfully") return topic @@ -166,7 +165,7 @@ def find_link_from_cooked(cooked: str, sha: str) -> str: if len(src) == 0: return "" else: - print(str(src[0])) + logger.debug(str(src[0])) return str(src[0]) src_set = str(src_sets[0]) @@ -200,7 +199,7 @@ def query_progress_and_update_state_to_translating(session: Session): .order_by(SyncProgress.cn_created_at.desc()).first() if progress is None: - print("All synchronized") + logger.debug("All synchronized") return None progress.translate_state = 1 @@ -219,7 +218,7 @@ def save_progress(session: Session, sync_progress, topic): def translate_task(wait_when_none: int = 2): sync_progress = query_progress_and_update_state_to_translating() - print(f"sync_progress: {sync_progress}") + logger.info(f"sync_progress: {sync_progress}") if sync_progress is None: if wait_when_none != 0: @@ -227,13 +226,13 @@ def translate_task(wait_when_none: int = 2): return start_time = datetime.now() - print(f"[{start_time.strftime('%Y-%m-%d %H:%M:%S')}] get {sync_progress}") + logger.info(f"get {sync_progress}") topic = translate_topic(sync_progress.cn_topic_id) save_progress(sync_progress, topic) delta = datetime.now() - start_time - print(f"[{delta.seconds}s] merged {sync_progress}") + logger.info(f"[{delta.seconds}s] merged {sync_progress}") return sync_progress