From 98a39acda42a124a50f357a55a6e0c8d573c7f26 Mon Sep 17 00:00:00 2001 From: xingdi <857803137@qq.com> Date: Sun, 11 Apr 2021 15:44:53 +0800 Subject: [PATCH 1/3] :sparkles: add CLI for convert --- modelci/cli/modelhub.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/modelci/cli/modelhub.py b/modelci/cli/modelhub.py index 37218994..9b5bd9e5 100644 --- a/modelci/cli/modelhub.py +++ b/modelci/cli/modelhub.py @@ -19,6 +19,8 @@ import typer import yaml from pydantic import ValidationError +import modelci.persistence.service_ as ModelDB +from modelci.hub.manager import _generate_model_family from modelci.config import app_settings from modelci.types.models import Framework, Engine, IOShape, Task, Metric, ModelUpdateSchema @@ -242,3 +244,16 @@ def delete(model_id: str = typer.Argument(..., help='Model ID')): with requests.delete(f'{app_settings.api_v1_prefix}/model/{model_id}') as r: if r.status_code == HTTPStatus.NO_CONTENT: typer.echo(f"Model {model_id} deleted") + + +@app.command('convert') +def convert( + id: str = typer.Option(..., '-i', '--id', help='ID of model.'), +): + if ModelDB.exists_by_id(id): + model = ModelDB.get_by_id(id) + # auto execute all possible convert and return a list of save paths of every converted model + generated_dir_list = _generate_model_family(model) + typer.echo(f"Converted models are save in: {generated_dir_list}") + else: + typer.echo(f"model id: {id} does not exit in modelhub") \ No newline at end of file From 38b772865203e57f757827f5ff8c432d9d7e05f0 Mon Sep 17 00:00:00 2001 From: xingdi <857803137@qq.com> Date: Sat, 17 Apr 2021 11:28:20 +0800 Subject: [PATCH 2/3] :fix: pytorch cannot convert to onnx --- modelci/types/type_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelci/types/type_conversion.py b/modelci/types/type_conversion.py index b8471ec2..d7e4bbad 100644 --- a/modelci/types/type_conversion.py +++ b/modelci/types/type_conversion.py @@ -102,7 +102,7 @@ def model_data_type_to_np(model_dtype): def model_data_type_to_torch(model_dtype): - from modelci.types.bo import DataType + from modelci.types.models.common import DataType import torch mapper = { From e1b5c98ff6dd6acca81917827ca22d83c5b4e228 Mon Sep 17 00:00:00 2001 From: xingdi <857803137@qq.com> Date: Tue, 20 Apr 2021 18:10:08 +0800 Subject: [PATCH 3/3] logger delete :fix: add 2 arguments of convert cli :fix: make 'generate_model_family' a public function :fix:convert cli: provide args: 1. the saved path of the converted model 2. whether to register the generated model into model hub --- example/alexnet.yml | 14 +++++++++++ modelci/cli/modelhub.py | 55 ++++++++++++++++++++++++++++++++++------- modelci/hub/manager.py | 16 ++++++++---- 3 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 example/alexnet.yml diff --git a/example/alexnet.yml b/example/alexnet.yml new file mode 100644 index 00000000..89e308ee --- /dev/null +++ b/example/alexnet.yml @@ -0,0 +1,14 @@ +weight: "~/.modelci/Alexnext/pytorch-pytorch/image_classification/1.pth" +dataset: ImageNet +task: IMAGE_CLASSIFICATION +metric: + acc: 0.74 +inputs: + - name: "input" + shape: [ 1, 3, 224, 224 ] + dtype: TYPE_FP32 +outputs: + - name: "output" + shape: [ -1, 1000 ] + dtype: TYPE_FP32 +convert: false \ No newline at end of file diff --git a/modelci/cli/modelhub.py b/modelci/cli/modelhub.py index 9b5bd9e5..62299f02 100644 --- a/modelci/cli/modelhub.py +++ b/modelci/cli/modelhub.py @@ -13,6 +13,7 @@ # permissions and limitations under the License. from http import HTTPStatus from pathlib import Path +from shutil import copy2, make_archive from typing import Dict, List, Optional import requests @@ -20,11 +21,13 @@ import yaml from pydantic import ValidationError import modelci.persistence.service_ as ModelDB -from modelci.hub.manager import _generate_model_family +from modelci.hub.manager import generate_model_family from modelci.config import app_settings +from modelci.hub.utils import parse_path_plain from modelci.types.models import Framework, Engine, IOShape, Task, Metric, ModelUpdateSchema from modelci.types.models import MLModelFromYaml, MLModel +from modelci.types.models.common import ModelStatus from modelci.ui import model_view, model_detailed_view from modelci.utils import Logger from modelci.utils.misc import remove_dict_null @@ -248,12 +251,46 @@ def delete(model_id: str = typer.Argument(..., help='Model ID')): @app.command('convert') def convert( - id: str = typer.Option(..., '-i', '--id', help='ID of model.'), + id: str = typer.Option(None, '-i', '--id', help='ID of model.'), + yaml_file: Optional[Path] = typer.Option( + None, '-f', '--yaml-file', exists=True, file_okay=True, + help='Path to configuration YAML file. You should either set the `yaml_file` field or fields ' + '(`FILE_OR_DIR`, `--name`, `--framework`, `--engine`, `--version`, `--task`, `--dataset`,' + '`--metric`, `--input`, `--output`).' + ), + register: bool = typer.Option(False, '-r', '--register', is_flag=True, help='register the converted models to modelhub, default false') ): - if ModelDB.exists_by_id(id): - model = ModelDB.get_by_id(id) - # auto execute all possible convert and return a list of save paths of every converted model - generated_dir_list = _generate_model_family(model) - typer.echo(f"Converted models are save in: {generated_dir_list}") - else: - typer.echo(f"model id: {id} does not exit in modelhub") \ No newline at end of file + if id is not None and yaml_file is not None: + typer.echo("Do not use -id and -path at the same time.") + typer.Exit() + elif id is not None and yaml_file is None: + if ModelDB.exists_by_id(id): + model = ModelDB.get_by_id(id) + else: + typer.echo(f"model id: {id} does not exist in modelhub") + elif id is None and yaml_file is not None: + # get MLModel from yaml file + with open(yaml_file) as f: + model_config = yaml.safe_load(f) + model_yaml = MLModelFromYaml.parse_obj(model_config) + model_in_saved_path = model_yaml.saved_path + if model_in_saved_path != model_yaml.weight: + copy2(model_yaml.weight, model_in_saved_path) + if model_yaml.engine == Engine.TFS: + weight_dir = model_yaml.weight + make_archive(weight_dir.with_suffix('.zip'), 'zip', weight_dir) + + model_data = model_yaml.dict(exclude_none=True, exclude={'convert', 'profile'}) + model = MLModel.parse_obj(model_data) + + # auto execute all possible convert and return a list of save paths of every converted model + generated_dir_list = generate_model_family(model) + typer.echo(f"Converted models are save in: {generated_dir_list}") + if register: + model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'}) + for model_dir in generated_dir_list: + parse_result = parse_path_plain(model_dir) + engine = parse_result['engine'] + model_cvt = MLModel(**model_data, weight=model_dir, engine=engine, model_status=[ModelStatus.CONVERTED]) + ModelDB.save(model_cvt) + typer.echo(f"converted {engine} are successfully registered in Modelhub") \ No newline at end of file diff --git a/modelci/hub/manager.py b/modelci/hub/manager.py index 94c160ae..b93794ad 100644 --- a/modelci/hub/manager.py +++ b/modelci/hub/manager.py @@ -36,7 +36,7 @@ from modelci.types.bo import Task, ModelVersion, Framework, ModelBO __all__ = ['get_remote_model_weight', 'register_model', 'register_model_from_yaml', 'retrieve_model', - 'retrieve_model_by_task', 'retrieve_model_by_parent_id'] + 'retrieve_model_by_task', 'retrieve_model_by_parent_id', 'generate_model_family'] from modelci.types.models.common import Engine, ModelStatus @@ -72,7 +72,7 @@ def register_model( # generate model family if convert: - model_dir_list.extend(_generate_model_family(model)) + model_dir_list.extend(generate_model_family(model)) # register model_data = model.dict(exclude={'weight', 'id', 'model_status', 'engine'}) @@ -149,11 +149,17 @@ def register_model_from_yaml(file_path: Union[Path, str]): register_model(model, convert=model_yaml.convert, profile=model_yaml.profile) -def _generate_model_family( +def generate_model_family( model: MLModel, max_batch_size: int = -1 ): - net = load(model.saved_path) + model_weight_path = model.saved_path + if not Path(model.saved_path).exists(): + (filepath, filename) = os.path.split(model.saved_path) + os.makedirs(filepath) + with open(model.saved_path, 'wb') as f: + f.write(model.weight.__bytes__()) + net = load(model_weight_path) build_saved_dir_from_engine = partial( generate_path_plain, **model.dict(include={'architecture', 'framework', 'task', 'version'}), @@ -323,4 +329,4 @@ def retrieve_model_by_parent_id(parent_id: str) -> List[ModelBO]: if len(models) == 0: raise FileNotFoundError('Model not found!') - return models + return models \ No newline at end of file