From 134eb723c3a7e97f822ebc0556143e97a80bb586 Mon Sep 17 00:00:00 2001 From: Jeffrey-Wang Date: Mon, 26 Jul 2021 23:37:33 +0800 Subject: [PATCH] feat: support no zip and no draco. --- Dockerfile | 2 +- README_ZH.md | 3 +- convert.sh | 2 +- server/db/redis.py | 10 ---- server/examples/python/converter_pb2.py | 26 +++++++-- server/examples/python/rpc_client.py | 6 +- server/exception/ValidateException.py | 5 -- server/rpc/converter_pb2.py | 26 +++++++-- server/rpc/protos/converter.proto | 2 + server/rpc_server.py | 30 ++++++++-- server/service/Convert.py | 76 ++++++++++++++----------- server/service/GltfPipeline.py | 16 ++++-- server/validate/Base.py | 15 ----- server/validate/Convert.py | 58 ------------------- 14 files changed, 129 insertions(+), 148 deletions(-) delete mode 100644 server/db/redis.py delete mode 100644 server/exception/ValidateException.py delete mode 100644 server/validate/Base.py delete mode 100644 server/validate/Convert.py diff --git a/Dockerfile b/Dockerfile index 7d080a0..a5e98db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# docker build . -t wj2015/3d-model-convert-to-gltf:v1.4 +# docker build . -t wj2015/3d-model-convert-to-gltf:v1.5 # startup docker by: docker run -d -p 8999:8999 wj2015/3d-model-convert-to-gltf:latest # you can debug by: docker run -it --rm -v `pwd`:/opt/3d-model-convert-to-gltf/ wj2015/3d-model-convert-to-gltf:latest /bin/bash # you can also execute `conda activate pythonocc` to enter the environment. diff --git a/README_ZH.md b/README_ZH.md index 8b90dcb..dafb14a 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -35,7 +35,8 @@ - [ ] [bug] stp 转 gltf 最终文件太大 - [x] 支持以 grpc 形式调用 - [ ] rpc 接口优化，返回具体的错误信息 -- [ ] rpc server 日志问题 +- [ ] rpc 需支持不适用 draco +- [ ] [bug]rpc server 日志问题 ## 项目用户 diff --git a/convert.sh b/convert.sh index 6098456..66bcce9 100755 --- a/convert.sh +++ b/convert.sh @@ -10,4 +10,4 @@ outPath=$( pwd ) outFile=$outPath/`basename $3` -docker run -v $inputPath:$inputPath -v $outPath:$outPath wj2015/3d-model-convert-to-gltf:v1.4 /bin/bash -c "cd $inputPath && conda run -n pythonocc python /opt/3d-model-convert-to-gltf/server/convert.py $1 $inputFile $outFile" +docker run -v $inputPath:$inputPath -v $outPath:$outPath wj2015/3d-model-convert-to-gltf:v1.5 /bin/bash -c "cd $inputPath && conda run -n pythonocc python /opt/3d-model-convert-to-gltf/server/convert.py $1 $inputFile $outFile" diff --git a/server/db/redis.py b/server/db/redis.py deleted file mode 100644 index 6d466a1..0000000 --- a/server/db/redis.py +++ /dev/null @@ -1,10 +0,0 @@ -from redis import Redis -db = None - -def connect(config): - global db - if db != None: - return db - conn = Redis(host=config['host'], port=config['port'], password=config['password'], db=config['db'], decode_responses=True) - db = conn - return db diff --git a/server/examples/python/converter_pb2.py b/server/examples/python/converter_pb2.py index bf91c3c..8524eb0 100644 --- a/server/examples/python/converter_pb2.py +++ b/server/examples/python/converter_pb2.py @@ -19,7 +19,7 @@ syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0f\x63onverter.proto\"7\n\nconvertReq\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05isBin\x18\x02 \x01(\x08\x12\x0c\n\x04\x66ile\x18\x03 \x01(\x0c\"\x1b\n\x0b\x63onvertResp\x12\x0c\n\x04\x66ile\x18\x01 \x01(\x0c\x32\x39\n\tConverter\x12,\n\rconvertToGltf\x12\x0b.convertReq\x1a\x0c.convertResp\"\x00\x62\x06proto3' + serialized_pb=b'\n\x0f\x63onverter.proto\"Y\n\nconvertReq\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05isBin\x18\x02 \x01(\x08\x12\x0c\n\x04\x66ile\x18\x03 \x01(\x0c\x12\x11\n\tneedDraco\x18\x04 \x01(\x08\x12\r\n\x05noZip\x18\x05 \x01(\x08\"\x1b\n\x0b\x63onvertResp\x12\x0c\n\x04\x66ile\x18\x01 \x01(\x0c\x32\x39\n\tConverter\x12,\n\rconvertToGltf\x12\x0b.convertReq\x1a\x0c.convertResp\"\x00\x62\x06proto3' ) @@ -54,6 +54,20 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='needDraco', full_name='convertReq.needDraco', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='noZip', full_name='convertReq.noZip', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -67,7 +81,7 @@ oneofs=[ ], serialized_start=19, - serialized_end=74, + serialized_end=108, ) @@ -98,8 +112,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=76, - serialized_end=103, + serialized_start=110, + serialized_end=137, ) DESCRIPTOR.message_types_by_name['convertReq'] = _CONVERTREQ @@ -129,8 +143,8 @@ index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=105, - serialized_end=162, + serialized_start=139, + serialized_end=196, methods=[ _descriptor.MethodDescriptor( name='convertToGltf', diff --git a/server/examples/python/rpc_client.py b/server/examples/python/rpc_client.py index 81aaf5e..5f127c9 100644 --- a/server/examples/python/rpc_client.py +++ b/server/examples/python/rpc_client.py @@ -18,11 +18,11 @@ def get_stub(target): return stub -def convert_file_and_save(target, t, source, dist, is_bin=False): +def convert_file_and_save(target, t, source, dist, is_bin=False, need_draco=True, no_zip=False): stub = get_stub(target) with open(source, 'rb') as f: response = stub.convertToGltf( - converter_pb2.convertReq(type=t, isBin=is_bin, file=f.read()) + converter_pb2.convertReq(type=t, isBin=is_bin, file=f.read(), needDraco=need_draco, noZip=no_zip) ) if response.file == b'': return False @@ -36,7 +36,7 @@ def convert_file_and_save(target, t, source, dist, is_bin=False): def run(): try: start_time = time.time() - if convert_file_and_save("127.0.0.1:8999", 'stl', '../../../assets/test.stl', 'test.glb.zip', True): + if convert_file_and_save("127.0.0.1:8999", 'stl', '../../../assets/test.stl', 'test.glb.zip', False, True, True): end_time = time.time() print("convert success", str(end_time - start_time), 's') else: diff --git a/server/exception/ValidateException.py b/server/exception/ValidateException.py deleted file mode 100644 index 6a4dd80..0000000 --- a/server/exception/ValidateException.py +++ /dev/null @@ -1,5 +0,0 @@ -from exception.BaseException import BaseException - -class ValidateException(BaseException): - def __init__(self, err="Validate error"): - BaseException.__init__(self, err) diff --git a/server/rpc/converter_pb2.py b/server/rpc/converter_pb2.py index bf91c3c..8524eb0 100644 --- a/server/rpc/converter_pb2.py +++ b/server/rpc/converter_pb2.py @@ -19,7 +19,7 @@ syntax='proto3', serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_pb=b'\n\x0f\x63onverter.proto\"7\n\nconvertReq\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05isBin\x18\x02 \x01(\x08\x12\x0c\n\x04\x66ile\x18\x03 \x01(\x0c\"\x1b\n\x0b\x63onvertResp\x12\x0c\n\x04\x66ile\x18\x01 \x01(\x0c\x32\x39\n\tConverter\x12,\n\rconvertToGltf\x12\x0b.convertReq\x1a\x0c.convertResp\"\x00\x62\x06proto3' + serialized_pb=b'\n\x0f\x63onverter.proto\"Y\n\nconvertReq\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05isBin\x18\x02 \x01(\x08\x12\x0c\n\x04\x66ile\x18\x03 \x01(\x0c\x12\x11\n\tneedDraco\x18\x04 \x01(\x08\x12\r\n\x05noZip\x18\x05 \x01(\x08\"\x1b\n\x0b\x63onvertResp\x12\x0c\n\x04\x66ile\x18\x01 \x01(\x0c\x32\x39\n\tConverter\x12,\n\rconvertToGltf\x12\x0b.convertReq\x1a\x0c.convertResp\"\x00\x62\x06proto3' ) @@ -54,6 +54,20 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='needDraco', full_name='convertReq.needDraco', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='noZip', full_name='convertReq.noZip', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), ], extensions=[ ], @@ -67,7 +81,7 @@ oneofs=[ ], serialized_start=19, - serialized_end=74, + serialized_end=108, ) @@ -98,8 +112,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=76, - serialized_end=103, + serialized_start=110, + serialized_end=137, ) DESCRIPTOR.message_types_by_name['convertReq'] = _CONVERTREQ @@ -129,8 +143,8 @@ index=0, serialized_options=None, create_key=_descriptor._internal_create_key, - serialized_start=105, - serialized_end=162, + serialized_start=139, + serialized_end=196, methods=[ _descriptor.MethodDescriptor( name='convertToGltf', diff --git a/server/rpc/protos/converter.proto b/server/rpc/protos/converter.proto index 2c5cda8..ba2acfb 100644 --- a/server/rpc/protos/converter.proto +++ b/server/rpc/protos/converter.proto @@ -11,6 +11,8 @@ message convertReq { string type = 1; bool isBin = 2; bytes file = 3; + bool needDraco = 4; + bool noZip = 5; } message convertResp { bytes file = 1; diff --git a/server/rpc_server.py b/server/rpc_server.py index 3300403..6812af0 100644 --- a/server/rpc_server.py +++ b/server/rpc_server.py @@ -12,6 +12,7 @@ from rpc import converter_pb2_grpc from service import upload from service import Convert +from exception.ConvertException import ConvertException class ConverterService(converter_pb2_grpc.ConverterServicer): @@ -20,7 +21,8 @@ def convertToGltf(self, request, context): up_service = upload.Upload() response = converter_pb2.convertResp(file=None) - print("receive request", request.type, request.isBin, ", size:", len(request.file) / 1024 / 1024, 'Mb') + print("receive request", request.type, request.isBin, ", need_draco: ", request.needDraco, ", no_zip: ", + request.noZip, " size:", len(request.file) / 1024 / 1024, 'Mb') try: # save and unzip up_service.save(request.file).unzip_save_file_with_clear_source() @@ -34,26 +36,42 @@ def convertToGltf(self, request, context): if find_path and len(find_path) > 0: # find first file source_model_path = find_path[0] + else: + raise ConvertException("can't found match source model") + else: source_model_path = up_service.get_save_path() # convert and clear source_model, then zip and response - result = model.handler(source_model_path, request.isBin) + result = model.handler(source_model_path, request.isBin, request.needDraco) # special logic, if convert to glb, only zip glb file zip_source_ext = None if request.isBin: zip_source_ext = ['glb'] - zip_path = up_service.clear_file(source_model_path).zip_source_dir(zip_source_ext).get_source_zip_path() + + if request.isBin and request.noZip: + up_service.clear_file(source_model_path) + find_path = up_service.scan_ext_file(['glb'], True) + if find_path and len(find_path) > 0: + result_file_path = find_path[0] + else: + raise ConvertException("can't found glb result file") + else: + result_file_path = up_service.clear_file(source_model_path).zip_source_dir( + zip_source_ext).get_source_zip_path() print("receive and handle ", request.type, up_service.get_save_path(), up_service.is_zip(), - 'found source file', source_model_path, 'convert result', result, 'zip path', zip_path, "zip size:", - os.path.getsize(zip_path) / 1024 / 1024, "Mb") + 'found source file', source_model_path, 'convert result', result, 'result path', result_file_path, + "zip size:", + ((os.path.getsize(result_file_path) / 1024 / 1024) if os.path.exists(result_file_path) else 'None'), + "Mb") - with open(zip_path, 'rb') as f: + with open(result_file_path, 'rb') as f: response = converter_pb2.convertResp(file=f.read()) except Exception as err: print("convert error:", err) + raise err finally: up_service.clear_save_dir() return response diff --git a/server/service/Convert.py b/server/service/Convert.py index e6c2ba4..9de4307 100644 --- a/server/service/Convert.py +++ b/server/service/Convert.py @@ -23,7 +23,7 @@ def clear_file(*file_paths): if os.path.exists(file_path): os.unlink(file_path) - def handler(self, file_path, is_bin): + def handler(self, file_path, is_bin, need_draco): if not file_path: raise ConvertException("convert file can't be empty") if not (os.path.exists(file_path) and os.path.isfile(file_path)): @@ -68,32 +68,44 @@ def check_binary(path_to_stl): return stl_assume_bytes == os.path.getsize(path_to_stl) @staticmethod - def convert_to_draco_gltf(file_path, convert_stl_path, is_bin=False, clear_stl_source=False, clear_convert_stl=False): + def convert_stl_to_gltf(file_path, convert_stl_path, is_bin=False, clear_stl_source=False, clear_convert_stl=False, + need_draco=True): # todo:: remove too many flag # 1. convert binary stl to gltf if is_bin: convert_gltf_path = file_path + '.glb' - out_convert_gltf_path = file_path + '.zip' + '.glb' + # for draco + dist_convert_gltf_path = file_path + '.draco' + '.glb' stl_to_gltf(convert_stl_path, convert_gltf_path, is_bin) + convert_gltf_bin_path = "" else: - convert_gltf_path = file_path + '.gltf' - output_path = os.path.dirname(convert_gltf_path) - out_convert_gltf_path = os.path.join(output_path, 'out.gltf') + output_path = os.path.dirname(file_path) + # for draco stl_to_gltf(convert_stl_path, output_path, is_bin) + dist_convert_gltf_path = file_path + '.gltf' + convert_gltf_path = os.path.join(output_path, 'out.gltf') + convert_gltf_bin_path = os.path.join(output_path, 'out.bin') # 2. gltf-pipeline - try: - if not gltf_pipeline(convert_gltf_path, out_convert_gltf_path): - raise ConvertException('gltf draco fail, file:' + convert_gltf_path) - finally: - if clear_stl_source: - StlModel.clear_file(file_path) - if clear_convert_stl: - StlModel.clear_file(convert_stl_path) - StlModel.clear_file(convert_gltf_path) - return out_convert_gltf_path - - def handler(self, file_path, is_bin=False): - super(StlModel, self).handler(file_path, is_bin) + if need_draco: + try: + if not gltf_pipeline(convert_gltf_path, dist_convert_gltf_path, is_bin): + raise ConvertException('gltf draco fail, file:' + convert_gltf_path) + finally: + StlModel.clear_file(convert_gltf_path) + if convert_gltf_bin_path: + StlModel.clear_file(convert_gltf_bin_path) + else: + # no draco no out_convert_gltf_path + dist_convert_gltf_path = convert_gltf_path + + if clear_stl_source: + StlModel.clear_file(file_path) + if clear_convert_stl: + StlModel.clear_file(convert_stl_path) + return dist_convert_gltf_path + + def handler(self, file_path, is_bin=False, need_draco=True): + super(StlModel, self).handler(file_path, is_bin, need_draco) # read stl file, if not binary, convert to binary convert_stl_path = file_path + '.stl' clear_convert_stl = False @@ -103,7 +115,7 @@ def handler(self, file_path, is_bin=False): clear_convert_stl = True else: convert_stl_path = file_path - return self.convert_to_draco_gltf(file_path, convert_stl_path, is_bin, False, clear_convert_stl) + return self.convert_stl_to_gltf(file_path, convert_stl_path, is_bin, False, clear_convert_stl, need_draco) class StpModel(BaseModel): @@ -111,14 +123,14 @@ def __init__(self): super(StpModel, self).__init__() self.ext = ['stp', 'step'] - def handler(self, file_path, is_bin): - super(StpModel, self).handler(file_path, is_bin) + def handler(self, file_path, is_bin, need_draco): + super(StpModel, self).handler(file_path, is_bin, need_draco) # read stp file and convert to stl convert_stl_path = file_path + '.stl' try: shapes = read_step_file(file_path) StlModel.write_by_shapes(shapes, convert_stl_path) - result = StlModel.convert_to_draco_gltf(file_path, convert_stl_path, is_bin, True) + result = StlModel.convert_stl_to_gltf(file_path, convert_stl_path, is_bin, True, True, need_draco) finally: self.clear_file(convert_stl_path) return result @@ -129,14 +141,14 @@ def __init__(self): super(IgesModel, self).__init__() self.ext = ['igs', 'iges'] - def handler(self, file_path, is_bin): - super(IgesModel, self).handler(file_path, is_bin) + def handler(self, file_path, is_bin, need_draco): + super(IgesModel, self).handler(file_path, is_bin, need_draco) # read iges file and convert to stl convert_stl_path = file_path + '.stl' try: shapes = read_iges_file(file_path) StlModel.write_by_shapes(shapes, convert_stl_path) - result = StlModel.convert_to_draco_gltf(file_path, convert_stl_path, is_bin, True) + result = StlModel.convert_stl_to_gltf(file_path, convert_stl_path, is_bin, True, True, need_draco) finally: self.clear_file(convert_stl_path) return result @@ -147,13 +159,13 @@ def __init__(self): super(ObjModel, self).__init__() self.ext = ['obj'] - def handler(self, file_path, is_bin): - super(ObjModel, self).handler(file_path, is_bin) + def handler(self, file_path, is_bin, need_draco): + super(ObjModel, self).handler(file_path, is_bin, need_draco) if is_bin: convert_gltf_path = file_path + '.glb' else: convert_gltf_path = file_path + '.gltf' - if not obj2gltf(file_path, convert_gltf_path, is_bin): + if not obj2gltf(file_path, convert_gltf_path, is_bin, need_draco): raise ConvertException('obj convert draco gltf fail, file:' + convert_gltf_path) return convert_gltf_path @@ -163,14 +175,14 @@ def __init__(self): super(FbxModel, self).__init__() self.ext = ['fbx'] - def handler(self, file_path, is_bin): - super(FbxModel, self).handler(file_path, is_bin) + def handler(self, file_path, is_bin, need_draco): + super(FbxModel, self).handler(file_path, is_bin, need_draco) if is_bin: convert_gltf_path = file_path + '.glb' else: convert_gltf_path = file_path + '.gltf' - if not fbx2gltf(file_path, convert_gltf_path, is_bin): + if not fbx2gltf(file_path, convert_gltf_path, is_bin, need_draco): raise ConvertException('fbx convert draco gltf fail, file:' + convert_gltf_path) return convert_gltf_path diff --git a/server/service/GltfPipeline.py b/server/service/GltfPipeline.py index 55bc0fa..6cb7579 100644 --- a/server/service/GltfPipeline.py +++ b/server/service/GltfPipeline.py @@ -5,25 +5,33 @@ def gltf_pipeline(input_path, out_path, is_binary=True): command = 'gltf-pipeline -i "' + input_path + '" -o "' + out_path + '" -d' if is_binary: command += ' -b' + else: + command += ' -s' + print("gltf pipeline command:", command) os.system(command) # exists mains success return os.path.exists(out_path) -def obj2gltf(input_path, out_path, is_binary=True): +def obj2gltf(input_path, out_path, is_binary=True, need_draco=True): command = 'obj2gltf -i "' + input_path + '" -o "' + out_path + '"' print('Is binary:', is_binary, 'command:', command) if is_binary: command += ' -b' + else: + command += ' -s' os.system(command) if os.path.exists(out_path): - return gltf_pipeline(out_path, out_path, is_binary) + if need_draco: + return gltf_pipeline(out_path, out_path, is_binary) + else: + return out_path else: return False -def fbx2gltf(input_path, out_path, is_binary=True): - command = 'fbx2gltf -d -i "' + input_path + '" -o "' + out_path + '"' +def fbx2gltf(input_path, out_path, is_binary=True, need_draco=True): + command = 'fbx2gltf ' + ('-d' if need_draco else '') + ' -i "' + input_path + '" -o "' + out_path + '"' if is_binary: command += ' -b' os.system(command) diff --git a/server/validate/Base.py b/server/validate/Base.py deleted file mode 100644 index dadedcc..0000000 --- a/server/validate/Base.py +++ /dev/null @@ -1,15 +0,0 @@ -from exception.ValidateException import ValidateException -def valiate_require(data, need=[]): - if not isinstance(need, list): - raise ValidateException('require validate need a list') - for field in need: - if not (field in data): - raise ValidateException(field + ' is required') -def fill_default_fields(data, fields=[], default=""): - if not isinstance(fields, list): - raise ValidateException('full default fields need a list') - for field in fields: - if not (field in data): - data[field] = default - return data - diff --git a/server/validate/Convert.py b/server/validate/Convert.py deleted file mode 100644 index 87a1447..0000000 --- a/server/validate/Convert.py +++ /dev/null @@ -1,58 +0,0 @@ -from exception.ValidateException import ValidateException -import validate.Base as Base -from aiohttp import web -import re - - -def validate_file(field): - if not isinstance(field, web.FileField): - raise ValidateException('file field must be File') - - -def validate_callback(field): - if not isinstance(field, str): - raise ValidateException('callback field must be string') - # em...,maybe need check url format - regex = re.compile(r'(http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?') - if re.search(regex, field): - raise ValidateException('callback field must be url format') - - -def validate_req_id(field): - if not isinstance(field, str): - raise ValidateException('req_id field must be str') - - -def validate_customize_data(field): - if not isinstance(field, str): - raise ValidateException('customize_data field must be string') - - -def stl(request, data): - Base.valiate_require(data, ['file']) - - validate_file(data['file']) - - -def stp(request, data): - Base.valiate_require(data, ['file']) - - validate_file(data['file']) - - -def iges(request, data): - Base.valiate_require(data, ['file']) - - validate_file(data['file']) - - -def obj(request, data): - Base.valiate_require(data, ['file']) - - validate_file(data['file']) - - -def fbx(request, data): - Base.valiate_require(data, ['file']) - - validate_file(data['file'])