refactor settings (#601)

* 解藕grom与cloud，新增错误回调
* 移动了一些文件的位置
* 删除watchdog
* 优化config使用
* 部分操作由操作员实现

---------

Co-authored-by: ZeYi Lin <[email protected]>

requirements.txt

@@ -21,5 +21,3 @@ peewee
 # Information collection
 psutil
 pynvml
-watchdog
-

swanlab/api/http.py

@@ -12,14 +12,13 @@
 from .info import LoginInfo, ProjectInfo, ExperimentInfo
 from .auth.login import login_by_key
 from .cos import CosClient
+from swanlab.data.modules import MediaBuffer
 from swanlab.error import NetworkError, ApiError
 from swanlab.package import get_host_api
 from swanlab.utils import FONT
 from swanlab.log import swanlog
 import requests
 
-from swanlab.data.modules import MediaBuffer
-
 
 def decode_response(resp: requests.Response) -> Union[Dict, AnyStr]:
     """

swanlab/api/upload/__init__.py

@@ -8,13 +8,10 @@
     上传相关接口
 """
 from ..http import get_http, sync_error_handler
-from .model import ColumnModel, MediaModel, ScalarModel
+from .model import ColumnModel, MediaModel, ScalarModel, FileModel
 from typing import List
-from swanlab.error import FileError, ApiError
+from swanlab.error import ApiError
 from swanlab.log import swanlog
-import json
-import yaml
-import os
 
 house_url = '/house/metrics'
 
@@ -82,33 +79,20 @@ def upload_scalar_metrics(scalar_metrics: List[ScalarModel]):
 
 
 @sync_error_handler
-def upload_files(files: List[str]):
+def upload_files(files: List[FileModel]):
     """
     上传files文件夹中的内容
     :param files: 文件列表，内部为文件绝对路径
     """
     http = get_http()
-    # 去重list
-    files = list(set(files))
-    files = {os.path.basename(x): x for x in files}
-    # 读取文件配置，生成更新信息
-    data = {}
-    for filename, filepath in files.items():
-        if filename not in _valid_files:
-            continue
-        try:
-            with open(filepath, 'r') as f:
-                if _valid_files[filename][1] == 'json':
-                    data[_valid_files[filename][0]] = json.load(f)
-                elif _valid_files[filename][1] == 'yaml':
-                    d = yaml.load(f, Loader=yaml.FullLoader)
-                    if d is None:
-                        raise FileError
-                    data[_valid_files[filename][0]] = d
-                else:
-                    data[_valid_files[filename][0]] = f.read()
-        except json.decoder.JSONDecodeError:
-            raise FileError
+    # 去重所有的FileModel，留下一个
+    if len(files) == 0:
+        return swanlog.warning("No files to upload.")
+    file_model = files[0]
+    if len(files) > 1:
+        for i in range(1, len(files) - 1):
+            file_model = FileModel.create(files[i], file_model)
+    data = file_model.to_dict()
     http.put(f'/project/{http.groupname}/{http.projname}/runs/{http.exp_id}/profile', data)
 
 

swanlab/api/upload/model.py

@@ -9,8 +9,8 @@
 """
 from enum import Enum
 from typing import List
-
 from swanlab.data.modules import MediaBuffer
+from datetime import datetime
 
 
 class ColumnModel:
@@ -125,3 +125,37 @@ def to_dict(self):
             "index": self.step,
             "epoch": self.epoch
         }
+
+
+class FileModel:
+    """
+    运行时文件信息上传模型
+    """
+
+    def __init__(self, requirements: str = None, metadata: dict = None, config: dict = None):
+        self.requirements = requirements
+        self.metadata = metadata
+        self.config = config
+        self.create_time = datetime.now()
+        """
+        主要用于去重，保留最新的文件
+        """
+
+    @classmethod
+    def create(cls, r1: "FileModel", r2: "FileModel") -> "FileModel":
+        """
+        比较两个FileModel，创建一个新的
+        如果新的newer不存在，则使用older的数据，否则使用newer的数据
+        """
+        newer, older = (r1, r2) if r1.create_time > r2.create_time else (r2, r1)
+        rq = newer.requirements if newer.requirements else older.requirements
+        md = newer.metadata if newer.metadata else older.metadata
+        cf = newer.config if newer.config else older.config
+        return cls(rq, md, cf)
+
+    def to_dict(self):
+        """
+        序列化，会删除为None的字段
+        """
+        d = {"requirements": self.requirements, "metadata": self.metadata, "config": self.config}
+        return {k: v for k, v in d.items() if v is not None}