Merge branch 'main' into monitoring_documentation

sgl-project · Nov 8, 2024 · d231452 · d231452
2 parents 2b08172 + d32fba2
commit d231452
Show file tree

Hide file tree

Showing 35 changed files with 659 additions and 518 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -30,6 +30,6 @@ repos:
     rev: 24.10.0
     hooks:
       - id: black
-        additional_dependencies: ['.[jupyter]']
-        types: [python, jupyter]
-        types_or: [python, jupyter]
+        types: [python]
+      - id: black-jupyter
+        types: [jupyter]
diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm
@@ -8,7 +8,7 @@ FROM $BASE_IMAGE AS base
 USER root
 
 WORKDIR /sgl-workspace
-
+ARG BUILD_TYPE=all
 ARG SGL_REPO="https://github.com/sgl-project/sglang"
 ENV SGL_DEFAULT="main"
 ARG SGL_BRANCH=${SGL_DEFAULT}
@@ -41,5 +41,7 @@ ENV VLLM_FP8_PADDING=1
 ENV VLLM_FP8_ACT_PADDING=1
 ENV VLLM_FP8_WEIGHT_PADDING=1
 ENV VLLM_FP8_REDUCE_CONV=1
+ENV TORCHINDUCTOR_MAX_AUTOTUNE=1
+ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1
 
 CMD ["/bin/bash"]
diff --git a/docs/backend/native_api.ipynb b/docs/backend/native_api.ipynb
@@ -34,10 +34,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:08.536886Z",
-     "iopub.status.busy": "2024-11-05T05:08:08.536763Z",
-     "iopub.status.idle": "2024-11-05T05:08:34.725831Z",
-     "shell.execute_reply": "2024-11-05T05:08:34.725316Z"
+     "iopub.execute_input": "2024-11-07T18:44:42.063503Z",
+     "iopub.status.busy": "2024-11-07T18:44:42.063379Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.255300Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.254547Z"
     }
    },
    "outputs": [],
@@ -73,10 +73,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:34.727530Z",
-     "iopub.status.busy": "2024-11-05T05:08:34.727333Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.359784Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.359090Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.258292Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.257710Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.611559Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.610842Z"
     }
    },
    "outputs": [],
@@ -101,10 +101,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.362286Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.362140Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.368711Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.368220Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.613911Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.613746Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.620286Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.619779Z"
     }
    },
    "outputs": [],
@@ -132,10 +132,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.371313Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.370877Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.376712Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.376230Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.622407Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.622267Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.628290Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.627793Z"
     }
    },
    "outputs": [],
@@ -164,10 +164,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.378982Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.378597Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.391820Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.391336Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.630585Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.630235Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.643498Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.643007Z"
     }
    },
    "outputs": [],
@@ -183,10 +183,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.393748Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.393606Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.398645Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.398145Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.645336Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.645196Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.650363Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.649837Z"
     }
    },
    "outputs": [],
@@ -211,10 +211,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.400683Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.400419Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.406146Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.405661Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.652212Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.652076Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.658633Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.658119Z"
     }
    },
    "outputs": [],
@@ -241,10 +241,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.408176Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.407884Z",
-     "iopub.status.idle": "2024-11-05T05:08:35.413587Z",
-     "shell.execute_reply": "2024-11-05T05:08:35.413108Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.660468Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.660325Z",
+     "iopub.status.idle": "2024-11-07T18:45:07.666476Z",
+     "shell.execute_reply": "2024-11-07T18:45:07.665984Z"
     }
    },
    "outputs": [],
@@ -271,10 +271,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:35.416090Z",
-     "iopub.status.busy": "2024-11-05T05:08:35.415793Z",
-     "iopub.status.idle": "2024-11-05T05:08:36.552549Z",
-     "shell.execute_reply": "2024-11-05T05:08:36.551870Z"
+     "iopub.execute_input": "2024-11-07T18:45:07.668242Z",
+     "iopub.status.busy": "2024-11-07T18:45:07.668108Z",
+     "iopub.status.idle": "2024-11-07T18:45:08.725709Z",
+     "shell.execute_reply": "2024-11-07T18:45:08.725021Z"
     }
    },
    "outputs": [],
@@ -296,10 +296,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:36.554823Z",
-     "iopub.status.busy": "2024-11-05T05:08:36.554680Z",
-     "iopub.status.idle": "2024-11-05T05:08:38.053945Z",
-     "shell.execute_reply": "2024-11-05T05:08:38.053034Z"
+     "iopub.execute_input": "2024-11-07T18:45:08.727865Z",
+     "iopub.status.busy": "2024-11-07T18:45:08.727721Z",
+     "iopub.status.idle": "2024-11-07T18:45:11.165841Z",
+     "shell.execute_reply": "2024-11-07T18:45:11.165282Z"
     }
    },
    "outputs": [],
@@ -335,10 +335,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:08:38.056783Z",
-     "iopub.status.busy": "2024-11-05T05:08:38.056497Z",
-     "iopub.status.idle": "2024-11-05T05:09:04.436030Z",
-     "shell.execute_reply": "2024-11-05T05:09:04.435311Z"
+     "iopub.execute_input": "2024-11-07T18:45:11.167853Z",
+     "iopub.status.busy": "2024-11-07T18:45:11.167711Z",
+     "iopub.status.idle": "2024-11-07T18:45:39.542988Z",
+     "shell.execute_reply": "2024-11-07T18:45:39.542135Z"
     }
    },
    "outputs": [],
@@ -360,10 +360,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:09:04.438987Z",
-     "iopub.status.busy": "2024-11-05T05:09:04.438568Z",
-     "iopub.status.idle": "2024-11-05T05:09:04.485291Z",
-     "shell.execute_reply": "2024-11-05T05:09:04.484829Z"
+     "iopub.execute_input": "2024-11-07T18:45:39.545416Z",
+     "iopub.status.busy": "2024-11-07T18:45:39.545005Z",
+     "iopub.status.idle": "2024-11-07T18:45:39.588793Z",
+     "shell.execute_reply": "2024-11-07T18:45:39.588054Z"
     }
    },
    "outputs": [],
@@ -392,10 +392,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:09:04.487191Z",
-     "iopub.status.busy": "2024-11-05T05:09:04.486929Z",
-     "iopub.status.idle": "2024-11-05T05:09:25.553481Z",
-     "shell.execute_reply": "2024-11-05T05:09:25.552747Z"
+     "iopub.execute_input": "2024-11-07T18:45:39.590729Z",
+     "iopub.status.busy": "2024-11-07T18:45:39.590446Z",
+     "iopub.status.idle": "2024-11-07T18:45:59.660376Z",
+     "shell.execute_reply": "2024-11-07T18:45:59.659992Z"
     }
    },
    "outputs": [],
@@ -419,10 +419,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:09:25.555813Z",
-     "iopub.status.busy": "2024-11-05T05:09:25.555666Z",
-     "iopub.status.idle": "2024-11-05T05:09:26.354372Z",
-     "shell.execute_reply": "2024-11-05T05:09:26.353693Z"
+     "iopub.execute_input": "2024-11-07T18:45:59.661779Z",
+     "iopub.status.busy": "2024-11-07T18:45:59.661641Z",
+     "iopub.status.idle": "2024-11-07T18:46:00.475726Z",
+     "shell.execute_reply": "2024-11-07T18:46:00.475269Z"
     }
    },
    "outputs": [],
@@ -445,10 +445,7 @@
     "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False)\n",
     "\n",
     "url = \"http://localhost:30030/classify\"\n",
-    "data = {\n",
-    "    \"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \n",
-    "    \"text\": prompts\n",
-    "}\n",
+    "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n",
     "\n",
     "responses = requests.post(url, json=data).json()\n",
     "for response in responses:\n",
@@ -460,10 +457,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:09:26.356532Z",
-     "iopub.status.busy": "2024-11-05T05:09:26.356327Z",
-     "iopub.status.idle": "2024-11-05T05:09:26.396590Z",
-     "shell.execute_reply": "2024-11-05T05:09:26.395914Z"
+     "iopub.execute_input": "2024-11-07T18:46:00.477283Z",
+     "iopub.status.busy": "2024-11-07T18:46:00.477025Z",
+     "iopub.status.idle": "2024-11-07T18:46:00.525758Z",
+     "shell.execute_reply": "2024-11-07T18:46:00.525236Z"
     }
    },
    "outputs": [],

diff --git a/docs/backend/offline_engine_api.ipynb b/docs/backend/offline_engine_api.ipynb
@@ -35,10 +35,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:21:27.503026Z",
-     "iopub.status.busy": "2024-11-05T05:21:27.502741Z",
-     "iopub.status.idle": "2024-11-05T05:21:49.554631Z",
-     "shell.execute_reply": "2024-11-05T05:21:49.553690Z"
+     "iopub.execute_input": "2024-11-07T18:46:04.789536Z",
+     "iopub.status.busy": "2024-11-07T18:46:04.789418Z",
+     "iopub.status.idle": "2024-11-07T18:46:27.038169Z",
+     "shell.execute_reply": "2024-11-07T18:46:27.037540Z"
     }
    },
    "outputs": [],
@@ -64,10 +64,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:21:49.558275Z",
-     "iopub.status.busy": "2024-11-05T05:21:49.558110Z",
-     "iopub.status.idle": "2024-11-05T05:21:52.717287Z",
-     "shell.execute_reply": "2024-11-05T05:21:52.716842Z"
+     "iopub.execute_input": "2024-11-07T18:46:27.040005Z",
+     "iopub.status.busy": "2024-11-07T18:46:27.039872Z",
+     "iopub.status.idle": "2024-11-07T18:46:30.203840Z",
+     "shell.execute_reply": "2024-11-07T18:46:30.203368Z"
     }
    },
    "outputs": [],
@@ -99,10 +99,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:21:52.721738Z",
-     "iopub.status.busy": "2024-11-05T05:21:52.720908Z",
-     "iopub.status.idle": "2024-11-05T05:22:01.770341Z",
-     "shell.execute_reply": "2024-11-05T05:22:01.769510Z"
+     "iopub.execute_input": "2024-11-07T18:46:30.205880Z",
+     "iopub.status.busy": "2024-11-07T18:46:30.205719Z",
+     "iopub.status.idle": "2024-11-07T18:46:39.256561Z",
+     "shell.execute_reply": "2024-11-07T18:46:39.255880Z"
     }
    },
    "outputs": [],
@@ -137,10 +137,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:22:01.772662Z",
-     "iopub.status.busy": "2024-11-05T05:22:01.772377Z",
-     "iopub.status.idle": "2024-11-05T05:22:04.897499Z",
-     "shell.execute_reply": "2024-11-05T05:22:04.896867Z"
+     "iopub.execute_input": "2024-11-07T18:46:39.259464Z",
+     "iopub.status.busy": "2024-11-07T18:46:39.259309Z",
+     "iopub.status.idle": "2024-11-07T18:46:42.384955Z",
+     "shell.execute_reply": "2024-11-07T18:46:42.384378Z"
     }
    },
    "outputs": [],
@@ -179,10 +179,10 @@
    "execution_count": null,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:22:04.899754Z",
-     "iopub.status.busy": "2024-11-05T05:22:04.899478Z",
-     "iopub.status.idle": "2024-11-05T05:22:13.970245Z",
-     "shell.execute_reply": "2024-11-05T05:22:13.969779Z"
+     "iopub.execute_input": "2024-11-07T18:46:42.387431Z",
+     "iopub.status.busy": "2024-11-07T18:46:42.387279Z",
+     "iopub.status.idle": "2024-11-07T18:46:51.448572Z",
+     "shell.execute_reply": "2024-11-07T18:46:51.447781Z"
     }
    },
    "outputs": [],
@@ -216,10 +216,10 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-05T05:22:13.972039Z",
-     "iopub.status.busy": "2024-11-05T05:22:13.971846Z",
-     "iopub.status.idle": "2024-11-05T05:22:14.027421Z",
-     "shell.execute_reply": "2024-11-05T05:22:14.027003Z"
+     "iopub.execute_input": "2024-11-07T18:46:51.451177Z",
+     "iopub.status.busy": "2024-11-07T18:46:51.450952Z",
+     "iopub.status.idle": "2024-11-07T18:46:51.497530Z",
+     "shell.execute_reply": "2024-11-07T18:46:51.496850Z"
     }
    },
    "outputs": [],