Merge pull request #1683 from phoenix-xhuang/master

[YMIR 2.4.0]

.env

@@ -8,12 +8,12 @@ NGINX_PORT=12001
 
 # API
 APP_SECRET_KEY=replace_with_your_secret_key
-PUBLIC_DATASET_OWNER=1
 FIRST_ADMIN=[email protected]
 FIRST_ADMIN_PASSWORD=12345678
 APP_API_KEY=replace_with_your_app_api_key
 VIZ_TIMEOUT=30
 REGISTRATION_NEEDS_APPROVAL=False
+INIT_OFFICIAL_DOCKER_IMAGE=True
 
 # Database (MySQL)
 MYSQL_ROOT_PASSWORD=ymir_db_passwd
@@ -72,5 +72,5 @@ DEPLOY_MODULE_HOST_PORT=18801
 DEPLOY_MODULE_URL=${DEPLOY_MODULE_HOST_PORT}
 DEPLOY_MODULE_MYSQL_ROOT_PASSWORD=deploy_db_passwd
 
-#public image module enviroment
+# public image module enviroment
 PUBLIC_IMAGE_URL=http://pubimg.vesionbook.com:8110

dev_docs/ymir-cmd-container.md

@@ -190,7 +190,7 @@ no training data found
 | /in/assets | 必要，图像资源所在的目录，只读 |
 | /in/annotations | 必要，图像标注所在的目录 |
 
-注1. `config.yaml` 文件的固定保留键
+注1. `config.yaml` 文件的固定保留键：在任务开始时，这些键值对会根据任务的实际情况由 YMIR 系统自动填入，就算镜像作者在 training-template.yaml / mining-template.yaml / infer-template.yaml 三个模板文件中写入了其他的值，在任务开始的时候也会被系统写入的值替换掉。
 
 * `task_id`: 任务id，只含英文数字下划线，用于标识或区别任务，会出现在 `/out/monitor.txt` 中，也可以出现在日志中
 
@@ -214,6 +214,8 @@ no training data found
 
     * `图像格式` 目前只能指定为 `raw`
 
+* `object_type`: 本次训练任务的类别，2 - 目标检测，3 - 语义分割，4 - 实例分割
+
 #### 4.3.2. 输出挂载点
 
 | 路径 | 说明 |
@@ -273,6 +275,8 @@ evaluate_config: # 计算 mAP, mAR, TP, FP, FN 时使用的配置信息
 
 * `class_names`: 模型可以识别的类型名称列表
 
+* `object_type`: 任务模型类型，2 - 目标检测，3 - 语义分割，4 - 实例分割
+
 #### 4.4.2. 输出挂载点
 
 | 路径 | 说明 |

docker-compose.labelfree.yml

@@ -1,7 +1,7 @@
 version: "3.3"
 services:
   label-nginx:
-    image: labelfree/open_frontend:3.3.1
+    image: labelfree/open_frontend:4.2.0
     env_file:
       - .env
     ports:
@@ -11,7 +11,7 @@ services:
     restart: always
 
   label_api:
-    image: labelfree/open:3.3.1
+    image: labelfree/open:4.2.0
     command: gunicorn --conf /code/config/gunicorn.py  --log-config /code/config/logging.conf --chdir /code/labelfree app:app
     env_file:
       - .env
@@ -48,7 +48,7 @@ services:
     restart: always
 
   label_celery_worker:
-    image: labelfree/open:3.3.1
+    image: labelfree/open:4.2.0
     command: celery -A labelfree.app.celery worker -l info -c 4 -B
     env_file:
       - .env

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.3"
 services:
   gateway:
-    image: industryessentials/ymir-gateway:release-2.3.0
+    image: industryessentials/ymir-gateway:release-2.4.0
     env_file:
       - .env
     ports:
@@ -11,8 +11,27 @@ services:
     networks:
       - ymirnetwork
 
+  yapi:
+    image: industryessentials/ymir-backend:release-2.4.0
+    command: bash -c 'cd /yapi && supervisord -nc supervisor/supervisord.conf'
+    env_file:
+      - .env
+    volumes:
+      - ${YMIR_PATH}/ymir-data/logs:/app_logs
+    runtime: ${SERVER_RUNTIME}
+    networks:
+      - ymirnetwork
+    depends_on:
+      - backend
+    healthcheck:
+      test: curl --fail -s http://localhost:80/health || exit 1
+      interval: 60s
+      timeout: 6s
+      retries: 3
+      start_period: 30s
+
   backend:
-    image: industryessentials/ymir-backend:release-2.3.0
+    image: industryessentials/ymir-backend:release-2.4.0
     env_file:
       - .env
     runtime: ${SERVER_RUNTIME}
@@ -90,7 +109,7 @@ services:
       - ymirnetwork
 
   web:
-    image: industryessentials/ymir-web:release-2.3.0
+    image: industryessentials/ymir-web:release-2.4.0
     volumes:
       - "${YMIR_PATH}/ymir-models:/data/ymir/ymir-models"
       - "${YMIR_PATH}/ymir-assets:/data/ymir/ymir-assets"

docker_executor/sample_executor/tests/test_dataset_reader.py

@@ -58,7 +58,7 @@ def _prepare_assets(self) -> None:
             },
         }
         with open(self._custom_env_file, 'w') as f:
-            yaml.safe_dump(env_obj, f)
+            yaml.safe_dump(env_obj, f, allow_unicode=True)
 
         # training index
         with open(self._training_index_file, 'w') as f:

docker_executor/sample_executor/tests/test_env.py

@@ -62,11 +62,11 @@ def _prepare_assets(self) -> None:
             },
         }
         with open(self._custom_env_file, 'w') as f:
-            yaml.safe_dump(env_obj, f)
+            yaml.safe_dump(env_obj, f, allow_unicode=True)
 
         # executor config
         with open(self._executor_config_file, 'w') as f:
-            yaml.safe_dump(self._expected_executor_config, f)
+            yaml.safe_dump(self._expected_executor_config, f, allow_unicode=True)
 
         # training index
         with open(self._training_index_file, 'w') as f:

docker_executor/sample_executor/tests/test_monitor.py

@@ -49,7 +49,7 @@ def _prepare_env_config(self) -> None:
             },
         }
         with open(self._custom_env_file, 'w') as f:
-            yaml.safe_dump(env_obj, f)
+            yaml.safe_dump(env_obj, f, allow_unicode=True)
 
     def _deprepare_dirs(self) -> None:
         if os.path.isdir(self._test_root):

docker_executor/sample_executor/tests/test_result_writer.py

@@ -50,7 +50,7 @@ def _prepare_env_config(self) -> None:
             },
         }
         with open(self._custom_env_file, 'w') as f:
-            yaml.safe_dump(env_obj, f)
+            yaml.safe_dump(env_obj, f, allow_unicode=True)
 
     def _deprepare_dirs(self) -> None:
         if os.path.isdir(self._test_root):

docker_executor/sample_executor/ymir_exc/ymir_exc/code.py

@@ -18,6 +18,8 @@ class ExecutorReturnCode(IntEnum):
     RC_EXEC_UNKNOWN_IMAGE_FORMAT = 300003
     # Can not find model file
     RC_EXEC_MODEL_ERROR = 300004
+    # object_type in config file not provided or not supported
+    RC_EXEC_OBJECT_TYPE_ERROR = 300005
 
     # cuda, gpu and memory errors
     # CUDA and GPU mismatch

docker_executor/sample_executor/ymir_exc/ymir_exc/result_writer.py

@@ -119,7 +119,7 @@ def write_model_stage(stage_name: str,
 
     # save all
     with open(env_config.output.training_result_file, 'w') as f:
-        yaml.safe_dump(data=training_result, stream=f)
+        yaml.safe_dump(data=training_result, stream=f, allow_unicode=True)
 
 
 def write_training_result(model_names: List[str], mAP: float, classAPs: Dict[str, float], **kwargs: dict) -> None:

ymir/Dockerfile.backend

@@ -66,6 +66,8 @@ COPY ./backend/src /app
 
 COPY --from=hel-build /ymir_hel/hel_server /app/ymir_hel/hel_server
 
+COPY ./backend/src/yapi /yapi
+
 WORKDIR /app
 COPY ./backend/deploy/git.config /root/.gitconfig
 COPY ./backend/deploy/supervisor /app/supervisor

ymir/backend/src/common/common_utils/sandbox_updater.py

@@ -81,4 +81,4 @@ def _update_user_labels(label_path: str, dst_ver: str) -> None:
         label_contents = yaml.safe_load(f)
     label_contents['ymir_version'] = dst_ver
     with open(label_path, 'w') as f:
-        yaml.safe_dump(label_contents, f)
+        yaml.safe_dump(label_contents, f, allow_unicode=True)

ymir/backend/src/common/id_definition/error_codes.py

@@ -91,7 +91,6 @@ class APIErrorCode(IntEnum):
     USER_NOT_FOUND = 110201
     USER_DUPLICATED_NAME = 110202
     USER_NOT_ACCESSIBLE = 110203
-    USER_NOT_LOGGED_IN = 110204
     USER_NOT_ADMIN = 110205
     USER_NOT_ACTIVE = 110206
     USER_ROLE_NOT_ELIGIBLE = 110207
@@ -102,8 +101,6 @@ class APIErrorCode(IntEnum):
     DATASET_DUPLICATED_NAME = 110402
     DATASET_NOT_ACCESSIBLE = 110403
     DATASET_FAILED_TO_CREATE = 110404
-    DATASET_PROTECTED_TO_DELETE = 110405
-    DATASETS_NOT_IN_SAME_GROUP = 110406
     INVALID_DATASET_STRUCTURE = 110407
     DATASET_FAILED_TO_IMPORT = 110408
     INVALID_DATASET_ZIP_FILE = 110409
@@ -115,7 +112,6 @@ class APIErrorCode(IntEnum):
     MODEL_DUPLICATED_NAME = 110602
     MODEL_NOT_ACCESSIBLE = 110603
     MODEL_FAILED_TO_CREATE = 110604
-    MODEL_NOT_READY = 110605
 
     TASK_NOT_FOUND = 110701
     TASK_DUPLICATED_NAME = 110702
@@ -131,6 +127,7 @@ class APIErrorCode(IntEnum):
     INVALID_INFERENCE_RESULT_FORMAT = 110903
 
     KEYWORD_DUPLICATED = 111001
+    CLASS_NAME_NOT_FOUND = 111002
 
     DOCKER_IMAGE_DUPLICATED = 111101
     DOCKER_IMAGE_NOT_FOUND = 111102
@@ -180,6 +177,8 @@ class APIErrorCode(IntEnum):
     PREMATURE_PREDICTIONS = 113002
     PREDICTION_NOT_ACCESSIBLE = 113003
 
+    MESSAGE_NOT_FOUND = 114001
+
 
 class UpdaterErrorCode(IntEnum):
     INVALID_USER_LABEL_FILE = 170001

ymir/backend/src/common/id_definition/state.py

@@ -0,0 +1,69 @@
+from enum import IntEnum
+
+from mir.protos import mir_command_pb2 as mir_cmd_pb
+
+
+class TaskType(IntEnum):
+    training = mir_cmd_pb.TaskTypeTraining
+    mining = mir_cmd_pb.TaskTypeMining
+    label = mir_cmd_pb.TaskTypeLabel
+    filter = mir_cmd_pb.TaskTypeFilter
+    merge = mir_cmd_pb.TaskTypeMerge
+    fusion = mir_cmd_pb.TaskTypeFusion
+    infer = mir_cmd_pb.TaskTypeDatasetInfer
+
+
+class TaskState(IntEnum):
+    unknown = mir_cmd_pb.TaskStateUnknown
+    pending = mir_cmd_pb.TaskStatePending
+    running = mir_cmd_pb.TaskStateRunning
+    done = mir_cmd_pb.TaskStateDone
+    error = mir_cmd_pb.TaskStateError
+    terminate = mir_cmd_pb.TaskStateTerminate
+
+
+class ResultType(IntEnum):
+    no_result = 0
+    dataset = 1
+    model = 2
+    prediction = 3
+    docker_image = 4
+
+
+class ResultState(IntEnum):
+    processing = 0
+    ready = 1
+    error = 2
+
+
+class ObjectType(IntEnum):
+    unknown = mir_cmd_pb.OT_UNKNOWN
+    classification = mir_cmd_pb.OT_CLASS
+    object_detect = mir_cmd_pb.OT_DET_BOX
+    segmentation = mir_cmd_pb.OT_SEG
+    instance_segmentation = 4
+
+
+class AnnotationType(IntEnum):
+    gt = 1
+    pred = 2
+
+
+class DockerImageType(IntEnum):
+    unknown = 0
+    training = 1
+    mining = 2
+    infer = 9
+
+
+class DockerImageState(IntEnum):
+    pending = 1
+    done = 3
+    error = 4
+
+
+class ImportStrategy(IntEnum):
+    no_annotations = 1
+    ignore_unknown_annotations = 2
+    stop_upon_unknown_annotations = 3
+    add_unknown_annotations = 4