sdk

平台提供了sdk，可以在数据集对接，任务，推理，notebook等对接平台。在与cube-studio平台进行对接前，需要先引入对应的python包

# 如果在notebook中使用，需要先source到对应的notebook内核，才能正常使用对应到sdk
!source activate cube-studio && pip install pandas

引入基础包

import json, os, time, shutil

from cubestudio.request.model_client import Client,init
from cubestudio.dataset.dataset import Dataset

初始化客户端

HOST = 'http://xx.xx.xx.xx'
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)

数据集sdk

可以自动上传数据集到平台，用于训练和推理等。

执行命令需要在同一行命令中，先source环境名(base,python27,python36,python37,python38,python39,python310,cube-studio)才能pip安装到指定环境，如果不知道有哪些虚拟环境，可以conda info --envs查看。

定义一个数据集

dataset = Client(Dataset).one(name="coco")
if not dataset:
    dataset = Client(Dataset).add(name='coco', version='v2014', label='coco未标注数据集', describe='来自于2014年数据，未标注的coco数据集',icon='https://pic2.zhimg.com/80/v2-399df41d8562f8f09b98d288b97c8f8d_1440w.webp')

上传数据集

features = json.dumps(json.load(open('coco/coco.json')),indent=4,ensure_ascii=False)
dataset = dataset.update(path='',features=features)
dataset.compress('coco.zip','coco')
dataset.upload('coco.zip',partition='20230201')

下载数据集

os.remove('coco.zip')  if os.path.exists('coco.zip') else ''
shutil.rmtree('coco')  if os.path.exists('coco') else ''
dataset.download(partition='20230201')
dataset.decompress('coco.zip','coco')

数据集加载

dataset.load('coco')
table = dataset.table

读取基础属性

print(table.info)
print(table.features)

推理sdk

引入基础包

from cubestudio.request.model_client import Client,init
from cubestudio.train.task import InferenceService,Project

初始化客户端

HOST = "http://xx.xx.xx.xx/"
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)

添加一个inferenceservice

inferenceservice = Client(InferenceService).add_or_update(
    service_type=f'serving',
    project=Client(Project).one(name='public'),
    label='sdk创建inference service',
    model_name='nginx',
    model_version='v202300801',
    images='nginx',
    resource_memory='2G',
    resource_cpu='2',
    resource_gpu='0',
    min_replicas='1',
    max_replicas='1',
    ports='80',
    volume_mount='kubeflow-user-workspace(pvc):/mnt',
    working_dir='',
    command='',
    env='',
)

部署推理服务

print(json.dumps(inferenceservice.to_dict(), indent=4))
if inferenceservice.model_status!='online':
    inferenceservice.deploy()

训练sdk

如果是在notebook中使用需要先source到对应的notebook内核环境

# 执行命令需要在同一行命令中，先source环境名(base,python27,python36,python37,python38,python39,python310,cube-studio)才能pip安装到指定环境，如果不知道有哪些虚拟环境，可以conda info --envs查看
!source activate cube-studio && pip install pandas

引入基础包

from cubestudio.request.model_client import Client,init
from cubestudio.train.task import Job_Template,Project,Pipeline,Task
import json

初始化客户端

HOST = 'http://xx.xx.xx.xx/'
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)

添加一个画布

pipeline = Client(Pipeline).add_or_update(
    name=f'{username}-default',
    describe='sdk画布',
    project=Client(Project).one(name='public')
)

添加一个任务

job_template = Client(Job_Template).one(name="自定义镜像")
task=Client(Task).add_or_update(
    name='sdk-test1',
    label='sdk发起的任务',
    pipeline=pipeline,
    job_template=Client(Job_Template).one(name="自定义镜像"),
    timeout=3600,
    retry=0,
    args=json.dumps(
        {
            "images":"ubuntu:20.04",
            "command":'for i in {1..50}; do date; sleep 1; done',
            "workdir":"/"
        }
    )
)

运行任务

task.run()

跟踪任务日志

task.log(follow=True)

关闭任务

task.stop()

notebook sdk

引入基础包

import datetime
import json
import random
import time

from cubestudio.request.model import Model
from cubestudio.request.model_client import Client,init
from cubestudio.train.task import Notebook,Project

初始化客户端

HOST = "http://xx.xx.xx.xx/"
token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJhZG1pbiJ9.j6-hUMaFYdSIzfc6i6TJ5DaS96Z9I78SrjxAOg-71yE'
username='admin'
init(host=HOST,username=username,token=token)

添加一个notebook

notebook = Client(Notebook).add_or_update(
    name='sdk-test',
    project=Client(Project).one(name='public'),
    describe='sdk创建notebook，并直接提供给其他平台跳转使用',
    images='ccr.ccs.tencentyun.com/cube-studio/notebook-enterprise:jupyter-ubuntu-machinelearning',
    volume_mount='kubeflow-user-workspace(pvc):/mnt',
    image_pull_policy='Always',
    resource_memory='10G',
    resource_cpu='10',
    resource_gpu='0',
    expand=json.dumps({"root":"examples/Bayesian/bayesian_network.ipynb"})
)

启动重置notebook

print(json.dumps(notebook.to_dict(),indent=4))

# if 'Running' not in notebook.status:
notebook.reset()