-
Notifications
You must be signed in to change notification settings - Fork 637
栾鹏 edited this page Nov 30, 2023
·
1 revision
平台提供了sdk,可以在数据集对接,任务,推理,notebook等对接平台。在与cube-studio平台进行对接前,需要先引入对应的python包
# 如果在notebook中使用,需要先source到对应的notebook内核,才能正常使用对应到sdk
!source activate cube-studio && pip install pandas
引入基础包
import json, os, time, shutil
from cubestudio.request.model_client import Client,init
from cubestudio.dataset.dataset import Dataset
HOST = 'http://xx.xx.xx.xx'
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)
可以自动上传数据集到平台,用于训练和推理等。
执行命令需要在同一行命令中,先source环境名(base,python27,python36,python37,python38,python39,python310,cube-studio)才能pip安装到指定环境,如果不知道有哪些虚拟环境,可以conda info --envs查看。
dataset = Client(Dataset).one(name="coco")
if not dataset:
dataset = Client(Dataset).add(name='coco', version='v2014', label='coco未标注数据集', describe='来自于2014年数据,未标注的coco数据集',icon='https://pic2.zhimg.com/80/v2-399df41d8562f8f09b98d288b97c8f8d_1440w.webp')
features = json.dumps(json.load(open('coco/coco.json')),indent=4,ensure_ascii=False)
dataset = dataset.update(path='',features=features)
dataset.compress('coco.zip','coco')
dataset.upload('coco.zip',partition='20230201')
os.remove('coco.zip') if os.path.exists('coco.zip') else ''
shutil.rmtree('coco') if os.path.exists('coco') else ''
dataset.download(partition='20230201')
dataset.decompress('coco.zip','coco')
dataset.load('coco')
table = dataset.table
print(table.info)
print(table.features)
引入基础包
from cubestudio.request.model_client import Client,init
from cubestudio.train.task import InferenceService,Project
HOST = "http://xx.xx.xx.xx/"
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)
inferenceservice = Client(InferenceService).add_or_update(
service_type=f'serving',
project=Client(Project).one(name='public'),
label='sdk创建inference service',
model_name='nginx',
model_version='v202300801',
images='nginx',
resource_memory='2G',
resource_cpu='2',
resource_gpu='0',
min_replicas='1',
max_replicas='1',
ports='80',
volume_mount='kubeflow-user-workspace(pvc):/mnt',
working_dir='',
command='',
env='',
)
print(json.dumps(inferenceservice.to_dict(), indent=4))
if inferenceservice.model_status!='online':
inferenceservice.deploy()
如果是在notebook中使用需要先source到对应的notebook内核环境
# 执行命令需要在同一行命令中,先source环境名(base,python27,python36,python37,python38,python39,python310,cube-studio)才能pip安装到指定环境,如果不知道有哪些虚拟环境,可以conda info --envs查看
!source activate cube-studio && pip install pandas
引入基础包
from cubestudio.request.model_client import Client,init
from cubestudio.train.task import Job_Template,Project,Pipeline,Task
import json
HOST = 'http://xx.xx.xx.xx/'
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)
pipeline = Client(Pipeline).add_or_update(
name=f'{username}-default',
describe='sdk画布',
project=Client(Project).one(name='public')
)
job_template = Client(Job_Template).one(name="自定义镜像")
task=Client(Task).add_or_update(
name='sdk-test1',
label='sdk发起的任务',
pipeline=pipeline,
job_template=Client(Job_Template).one(name="自定义镜像"),
timeout=3600,
retry=0,
args=json.dumps(
{
"images":"ubuntu:20.04",
"command":'for i in {1..50}; do date; sleep 1; done',
"workdir":"/"
}
)
)
task.run()
task.log(follow=True)
task.stop()
引入基础包
import datetime
import json
import random
import time
from cubestudio.request.model import Model
from cubestudio.request.model_client import Client,init
from cubestudio.train.task import Notebook,Project
HOST = "http://xx.xx.xx.xx/"
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)
notebook = Client(Notebook).add_or_update(
name='sdk-test',
project=Client(Project).one(name='public'),
describe='sdk创建notebook,并直接提供给其他平台跳转使用',
images='ccr.ccs.tencentyun.com/cube-studio/notebook-enterprise:jupyter-ubuntu-machinelearning',
volume_mount='kubeflow-user-workspace(pvc):/mnt',
image_pull_policy='Always',
resource_memory='10G',
resource_cpu='10',
resource_gpu='0',
expand=json.dumps({"root":"examples/Bayesian/bayesian_network.ipynb"})
)
print(json.dumps(notebook.to_dict(),indent=4))
# if 'Running' not in notebook.status:
notebook.reset()