Skip to content

Commit

Permalink
Merge pull request #14 from onepanelio/dev
Browse files Browse the repository at this point in the history 
chore: dev > master (v0.12.0)
  • Loading branch information
@rushtehrani
rushtehrani authored Aug 27, 2020
2 parents 3317104 + b27b27a commit bb7ddf2
Show file tree
Hide file tree
Showing 8 changed files with 662 additions and 391 deletions.
9 changes: 0 additions & 9 deletions examples/basic_streaming.py
View file

This file was deleted.

251 changes: 251 additions & 0 deletions examples/execute-workflow.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: onepanel-sdk in /opt/conda/lib/python3.6/site-packages (0.12.0)\n",
"Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2.8.1)\n",
"Requirement already satisfied: certifi in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2020.6.20)\n",
"Requirement already satisfied: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.15.0)\n",
"Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.25.10)\n"
]
}
],
"source": [
"!pip install onepanel-sdk"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from time import sleep\n",
"from pprint import pprint\n",
"\n",
"import onepanel.core.api\n",
"from onepanel.core.api.rest import ApiException\n",
"from onepanel.core.api.models import Parameter"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'created_at': '2020-08-26T22:41:51Z',\n",
" 'finished_at': None,\n",
" 'labels': None,\n",
" 'manifest': None,\n",
" 'metadata': {'url': 'http://app.rush-gcs.onepanel.site/rush/workflows/pytorch-training-qbc6z'},\n",
" 'name': 'pytorch-training-qbc6z',\n",
" 'parameters': None,\n",
" 'phase': None,\n",
" 'started_at': None,\n",
" 'uid': 'pytorch-training-qbc6z',\n",
" 'workflow_template': {'created_at': '2020-08-26T05:25:08Z',\n",
" 'cron_stats': None,\n",
" 'is_archived': None,\n",
" 'is_latest': None,\n",
" 'labels': None,\n",
" 'manifest': 'entrypoint: main\\n'\n",
" 'arguments:\\n'\n",
" ' parameters:\\n'\n",
" ' - name: source\\n'\n",
" ' value: '\n",
" 'https://github.com/onepanelio/pytorch-examples.git\\n'\n",
" ' - name: command\\n'\n",
" ' value: \"python mnist/main.py '\n",
" '--epochs=1\"\\n'\n",
" 'volumeClaimTemplates:\\n'\n",
" ' - metadata:\\n'\n",
" ' name: data\\n'\n",
" ' spec:\\n'\n",
" ' accessModes: [ \"ReadWriteOnce\" ]\\n'\n",
" ' resources:\\n'\n",
" ' requests:\\n'\n",
" ' storage: 2Gi\\n'\n",
" ' - metadata:\\n'\n",
" ' name: output\\n'\n",
" ' spec:\\n'\n",
" ' accessModes: [ \"ReadWriteOnce\" ]\\n'\n",
" ' resources:\\n'\n",
" ' requests:\\n'\n",
" ' storage: 2Gi\\n'\n",
" 'templates:\\n'\n",
" ' - name: main\\n'\n",
" ' dag:\\n'\n",
" ' tasks:\\n'\n",
" ' - name: train-model\\n'\n",
" ' template: pytorch\\n'\n",
" '# Uncomment section below to send metrics '\n",
" 'to Slack\\n'\n",
" '# - name: notify-in-slack\\n'\n",
" '# dependencies: [train-model]\\n'\n",
" '# template: slack-notify-success\\n'\n",
" '# arguments:\\n'\n",
" '# parameters:\\n'\n",
" '# - name: status\\n'\n",
" '# value: '\n",
" '\"{{tasks.train-model.status}}\"\\n'\n",
" '# artifacts:\\n'\n",
" '# - name: metrics\\n'\n",
" '# from: '\n",
" '\"{{tasks.train-model.outputs.artifacts.sys-metrics}}\"\\n'\n",
" ' - name: pytorch\\n'\n",
" ' inputs:\\n'\n",
" ' artifacts:\\n'\n",
" ' - name: src\\n'\n",
" ' path: /mnt/src\\n'\n",
" ' git:\\n'\n",
" ' repo: '\n",
" '\"{{workflow.parameters.source}}\"\\n'\n",
" ' outputs:\\n'\n",
" ' artifacts:\\n'\n",
" ' - name: model\\n'\n",
" ' path: /mnt/output\\n'\n",
" ' optional: true\\n'\n",
" ' archive:\\n'\n",
" ' none: {}\\n'\n",
" ' container:\\n'\n",
" ' image: pytorch/pytorch:latest\\n'\n",
" ' command: [sh,-c]\\n'\n",
" ' args: '\n",
" '[\"{{workflow.parameters.command}}\"]\\n'\n",
" ' workingDir: /mnt/src\\n'\n",
" ' volumeMounts:\\n'\n",
" ' - name: data\\n'\n",
" ' mountPath: /mnt/data\\n'\n",
" ' - name: output\\n'\n",
" ' mountPath: /mnt/output\\n'\n",
" ' - name: slack-notify-success\\n'\n",
" ' container:\\n'\n",
" ' image: technosophos/slack-notify\\n'\n",
" ' command: [sh,-c]\\n'\n",
" \" args: ['SLACK_USERNAME=Worker \"\n",
" 'SLACK_TITLE=\"{{workflow.name}} '\n",
" '{{inputs.parameters.status}}\" '\n",
" 'SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd '\n",
" 'SLACK_MESSAGE=$(cat /tmp/metrics.json)} '\n",
" \"./slack-notify']\\n\"\n",
" ' inputs:\\n'\n",
" ' parameters:\\n'\n",
" ' - name: status\\n'\n",
" ' artifacts:\\n'\n",
" ' - name: metrics\\n'\n",
" ' path: /tmp/metrics.json\\n'\n",
" ' optional: true\\n',\n",
" 'modified_at': '2020-08-26T05:25:08Z',\n",
" 'name': 'PyTorch Training',\n",
" 'parameters': [{'display_name': None,\n",
" 'hint': None,\n",
" 'name': 'source',\n",
" 'options': None,\n",
" 'required': None,\n",
" 'type': None,\n",
" 'value': 'https://github.com/onepanelio/pytorch-examples.git',\n",
" 'visibility': 'public'},\n",
" {'display_name': None,\n",
" 'hint': None,\n",
" 'name': 'command',\n",
" 'options': None,\n",
" 'required': None,\n",
" 'type': None,\n",
" 'value': 'python mnist/main.py '\n",
" '--epochs=1',\n",
" 'visibility': 'public'}],\n",
" 'stats': {'completed': None,\n",
" 'failed': None,\n",
" 'last_executed': '0001-01-01T00:00:00Z',\n",
" 'running': None,\n",
" 'terminated': None,\n",
" 'total': None},\n",
" 'uid': 'pytorch-training',\n",
" 'version': '1598419508627499422',\n",
" 'versions': None}}\n"
]
}
],
"source": [
"# If outside of Onepanel\n",
"# token = '<auth-token>'\n",
"# Set environment variables for:\n",
"# ONEPANEL_API_URL, example: https://app.domain.com/api\n",
"# ONEPANEL_RESOURCE_NAMESPACE, example: default\n",
"\n",
"# If inside of Onepanel, get mounted service account token to use as API Key\n",
"with open('/var/run/secrets/kubernetes.io/serviceaccount/token') as f:\n",
" token = f.read()\n",
"\n",
"# Configure API key authorization: Bearer\n",
"configuration = onepanel.core.api.Configuration(\n",
" host=os.getenv('ONEPANEL_API_URL'),\n",
" api_key={\n",
" 'Bearer': token\n",
" }\n",
")\n",
"configuration.api_key_prefix['Bearer'] = 'Bearer'\n",
"\n",
"# Enter a context with an instance of the API client\n",
"with onepanel.core.api.ApiClient(configuration) as api_client:\n",
" # Create an instance of the API class\n",
" api_instance = onepanel.core.api.WorkflowServiceApi(api_client)\n",
"\n",
" namespace = os.getenv('ONEPANEL_RESOURCE_NAMESPACE')\n",
" params = []\n",
" params.append(Parameter(name='source', value='https://github.com/onepanelio/pytorch-examples.git'))\n",
" params.append(Parameter(name='command', value='python mnist/main.py --epochs=1'))\n",
" body = onepanel.core.api.CreateWorkflowExecutionBody(\n",
" parameters=params,\n",
" workflow_template_uid='pytorch-training',\n",
" )\n",
" try:\n",
" workflow = api_instance.create_workflow_execution(namespace, body)\n",
" pprint(workflow)\n",
" except ApiException as e:\n",
" print('Exception when calling WorkflowServiceApi->create_workflow_execution: %s\\n' % e)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.11"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
129 changes: 129 additions & 0 deletions examples/launch-workspace.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: onepanel-sdk in /opt/conda/lib/python3.6/site-packages (0.12.0)\n",
"Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.25.10)\n",
"Requirement already satisfied: certifi in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2020.6.20)\n",
"Requirement already satisfied: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (1.15.0)\n",
"Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.6/site-packages (from onepanel-sdk) (2.8.1)\n"
]
}
],
"source": [
"!pip install onepanel-sdk"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"from time import sleep\n",
"\n",
"import onepanel.core.api\n",
"from onepanel.core.api.rest import ApiException\n",
"from onepanel.core.api.models import Parameter"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Waiting for workspace to launch...\n",
"\n",
"Workspace is running at http://jupyterlab-workspace--rush.rush-gcs.onepanel.site.\n",
"\n"
]
}
],
"source": [
"# If outside of Onepanel\n",
"# token = '<auth-token>'\n",
"# Set environment variables for:\n",
"# ONEPANEL_API_URL, example: https://app.domain.com/api\n",
"# ONEPANEL_RESOURCE_NAMESPACE, example: default\n",
"\n",
"# If inside of Onepanel, get mounted service account token to use as API Key\n",
"with open('/var/run/secrets/kubernetes.io/serviceaccount/token') as f:\n",
" token = f.read()\n",
"\n",
"# Configure API key authorization: Bearer\n",
"configuration = onepanel.core.api.Configuration(\n",
" host=os.getenv('ONEPANEL_API_URL'),\n",
" api_key={\n",
" 'Bearer': token\n",
" }\n",
")\n",
"configuration.api_key_prefix['Bearer'] = 'Bearer'\n",
"\n",
"# Enter a context with an instance of the API client\n",
"with onepanel.core.api.ApiClient(configuration) as api_client:\n",
" # Create an instance of the API class\n",
" api_instance = onepanel.core.api.WorkspaceServiceApi(api_client)\n",
"\n",
" namespace = os.getenv('ONEPANEL_RESOURCE_NAMESPACE')\n",
" params = []\n",
" params.append(Parameter(name='sys-name', value='JupyterLab Workspace'))\n",
" params.append(Parameter(name='sys-node-pool', value='Standard_NC6'))\n",
" body = onepanel.core.api.CreateWorkspaceBody(\n",
" parameters=params,\n",
" workspace_template_uid='jupyterlab',\n",
" )\n",
" try:\n",
" workspace = api_instance.create_workspace(namespace, body)\n",
" uid = workspace.uid\n",
" print('Waiting for workspace to launch...\\n')\n",
" while workspace.status.phase != 'Running':\n",
" workspace = api_instance.get_workspace(namespace, uid)\n",
" sleep(15)\n",
" print('Workspace is running at %s.\\n' % workspace.url)\n",
" except ApiException as e:\n",
" print(\"Exception when calling WorkspaceServiceApi->create_workspace: %s\\n\" % e)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.11"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit bb7ddf2

Please sign in to comment.