Dev ci master

.github/script/Dockerfile

@@ -0,0 +1,9 @@
+# Container image that runs your code
+FROM python:3.6
+
+RUN pip3 install requests -i https://mirrors.aliyun.com/pypi/simple/ && pip3 install art -i https://mirrors.aliyun.com/pypi/simple/
+# Copies your code file from your action repository to the filesystem path `/` of the container
+COPY seekdb.py /seekdb.py
+
+# Code file to execute when the docker container starts up (`entrypoint.sh`)
+ENTRYPOINT ["python3", "-u", "/seekdb.py"]

.github/script/action.yml

@@ -0,0 +1,24 @@
+# action.yml
+name: 'call seekdb Farm2 to run task'
+description: ''
+inputs:
+  pipeline_id:
+    description: 'pipeline_id'
+    required: true
+  project:
+    description: 'project'
+    required: true
+  timeout:
+    description: 'timeout'
+    required: false
+    default: '10800'
+outputs:
+  success:
+    description: 'the status for the task'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.pipeline_id }}
+    - ${{ inputs.project }}
+    - ${{ inputs.timeout }}

.github/script/seekdb.py

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+import copy
+import os
+import sys
+import traceback
+import time
+import json
+import requests
+from enum import Enum
+from http import HTTPStatus
+
+from art import text2art
+
+OUTPUT = {}
+RESULT_FILE_KEY = "farm/seekdb_results/"
+TASK_QUEUE_FILE_KEY = "farm/seekdb_jobs/{}.json"
+
+
+def _range(start, last):
+    def to_str(pos):
+        if pos is None:
+            return ''
+        else:
+            return str(pos)
+
+    return to_str(start) + '-' + to_str(last)
+
+
+def _make_range_string(range):
+    if range is None:
+        return ''
+
+    start = range[0]
+    last = range[1]
+
+    if start is None and last is None:
+        return ''
+
+    return 'bytes=' + _range(start, last)
+
+
+class OssProxy:
+
+    def __init__(self, endpoint=""):
+        self.endpoint = endpoint
+
+    def get_object(self, key, _range=None):
+        url = "{}/{}".format(self.endpoint, key)
+        headers = {}
+        if _range is not None:
+            _range = (_range, None)
+            headers.update({"range": _make_range_string(_range)})
+        res = requests.get(url, headers=headers)
+        if res.status_code < 400:
+            result = res.content.decode()
+            return result
+        return ""
+
+    def get_object_meta(self, key):
+        url = "{}/{}".format(self.endpoint, key)
+        headers = {}
+        res = requests.head(url, headers=headers)
+        return res.headers
+
+    def exists_object(self, key):
+        ...
+
+
+class GithubProxy:
+
+    def __init__(self, host="api.github.com"):
+        self.host = host
+
+    def get_job_by_id(self, project, pipeline_id):
+        url = "https://{}/repos/{}/actions/runs/{}".format(
+            self.host, project, pipeline_id
+        )
+        try:
+            res = requests.get(
+                url, headers={
+                    "Accept": "application/vnd.github+json"
+                }
+            )
+            status_code = res.status_code
+            if status_code == HTTPStatus.NOT_FOUND:
+                return {}
+            return res.json()
+        except:
+            traceback.print_exc()
+            return {}
+
+
+class TaskStatusEnum(Enum):
+    submitting = 0
+    pending = 1
+    running = 2
+    stopping = 3
+    success = 4
+    fail = -1
+    kill = -2
+    timeout = -3
+    submit_task_fail = -4
+
+
+def request(method, url, params=None, payload=None, timeout=10, data=None, without_check_status=False):
+    params = params or {}
+    try:
+        response = requests.request(
+            method,
+            url,
+            params=params,
+            json=payload,
+            data=data,
+            timeout=timeout
+        )
+        if not without_check_status and response.status_code >= 300:
+            try:
+                msg = response.json()["msg"]
+            except:
+                msg = response.text
+            print("[ERROR] MSG:{}".format(msg))
+            exit(1)
+        return response
+    except Exception:
+        import traceback
+        traceback.print_exc()
+        print("Please contact the management personnel for assistance !")
+        if not without_check_status:
+            exit(1)
+
+
+def monitor_tasks(oss_proxy: OssProxy, github_pipeline_id, timeout, result_out_path=None):
+    end_time = time.time() + int(timeout)
+    end_task = False
+    task_data = None
+    while time.time() <= end_time:
+        if end_task is True:
+            pass
+        task_data = get_task_res(oss_proxy, github_pipeline_id)
+        if task_data:
+            end_task = True
+
+        time.sleep(1)
+        if task_data is not None:
+            task_status = int(task_data["status"])
+            output_url = "https://ce-farm.oceanbase-dev.com/farm2/ci/?id={}".format(task_data["task_id"])
+            if task_status <= TaskStatusEnum.fail.value:
+                print(TaskStatusEnum._value2member_map_[task_status])
+                print("there is the output url: {}".format(output_url))
+                if result_out_path:
+                    _write_result_json(result_out_path, task_data, success=False)
+                return False
+            elif task_status >= TaskStatusEnum.success.value:
+                print(TaskStatusEnum._value2member_map_[task_status])
+                print("there is the output url: {}".format(output_url))
+                if result_out_path:
+                    _write_result_json(result_out_path, task_data, success=True)
+                return True
+
+        time.sleep(5)
+    if result_out_path:
+        _write_result_json(result_out_path, task_data or {}, success=False, timeout=True)
+    return False
+
+
+def _write_result_json(path, task_data, success, timeout=False):
+    out = {
+        "success": success,
+        "timeout": timeout,
+        "task_id": task_data.get("task_id") if task_data else None,
+        "status": task_data.get("status") if task_data else None,
+        "output_url": "https://ce-farm.oceanbase-dev.com/farm2/ci/?id={}".format(task_data["task_id"]) if task_data and task_data.get("task_id") else None,
+    }
+    with open(path, "w") as f:
+        json.dump(out, f, indent=2)
+
+
+def get_task_res(oss_proxy: OssProxy, github_pipeline_id):
+    try:
+        result_key = RESULT_FILE_KEY + "{}.json".format(github_pipeline_id)
+        origin_task_data = oss_proxy.get_object(result_key)
+        return json.loads(origin_task_data)
+    except:
+        return
+
+
+def main(pipeline_id, project, timeout, result_out_path=None):
+    print("create a new task")
+    print("working....")
+    logo = text2art('seekdb Farm2')
+    print(logo)
+    oss_proxy = OssProxy("https://obfarm-ce.oss-cn-hongkong.aliyuncs.com")
+    github_proxy = GithubProxy()
+    job_info = github_proxy.get_job_by_id(project, pipeline_id)
+    attempt_number = job_info["run_attempt"]
+    run_pipeline_id = "{}-{}".format(pipeline_id, attempt_number)
+    result = monitor_tasks(oss_proxy, run_pipeline_id, timeout, result_out_path=result_out_path)
+    if not result:
+        exit(1)
+
+
+if __name__ == "__main__":
+    print(sys.argv)
+    if len(sys.argv) < 4:
+        print("Missing relevant parameters !")
+        OUTPUT.update({"success": -1})
+        sys.exit(1)
+    out_path = sys.argv[4] if len(sys.argv) > 4 else None
+    main(sys.argv[1], sys.argv[2], sys.argv[3], result_out_path=out_path)

.github/script/seekdb_native/README.md

@@ -0,0 +1,32 @@
+# SeekDB 执行下沉脚本（Native）
+
+与 `seekdb-native.yml` 配套，在 self-hosted Runner 上直接执行 Prepare → Compile → Mysqltest，不经过 Farm2。
+
+**脚本来源**：已从 farm-jenkins 复制到本目录 `scripts/`，无需再配置 FARM2_SCRIPTS_REPO 或 clone。
+
+| 文件 | 说明 |
+|------|------|
+| `scripts/frame.sh` | 自 farm-jenkins 复制并改造（/etc/hosts、dep_cache 在无权限时跳过） |
+| `scripts/farm_compile.sh` | 自 farm-jenkins 复制 |
+| `scripts/farm_post_compile.sh` | 自 farm-jenkins 复制 |
+| `scripts/mysqltest_for_farm.sh` | 自 farm-jenkins 复制 |
+| `scripts/dep_cache.sh` | 本仓 stub，供 frame 在无 dep_create 时使用 |
+| `prepare.sh` | 仅生成 jobargs.output、run_jobs.output |
+| `compile.sh` | 调用 scripts/farm_compile.sh（frame prepare + build） |
+| `mysqltest_slice.sh` | 调用 scripts/mysqltest_for_farm.sh |
+| `collect_result.sh` | 汇总 fail_cases 写 seekdb_result.json |
+
+## 仓库变量（可选）
+
+| 变量 | 说明 |
+|------|------|
+| `FARM2_WORKER_IMAGE` | 编译/测试用 Docker 镜像；不设则在 Runner 本机执行 |
+| `FORWARDING_HOST` | mirrors.oceanbase.com 解析到该主机时填写 |
+| `RELEASE_MODE` | 非空则 release 编译 |
+
+## 产物位置
+
+- 任务目录：`$GITHUB_WORKSPACE/seekdb_build/$GITHUB_RUN_ID/`
+- 编译产出：observer.zst、obproxy.zst、compile.output
+- 各 slice：mysqltest.output.$i、collected_log_$i.tar.gz 等
+- 小结果由 workflow 上传为 artifact `seekdb-result-native`

.github/script/seekdb_native/collect_result.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Collect result and write seekdb_result.json for native execution.
+# 参考 farm-jenkins: scripts/opensource/scripts/opensource_publish_result.sh 的结论汇总方式。
+set -e
+
+WORKSPACE="${GITHUB_WORKSPACE:-.}"
+TASK_DIR="${SEEKDB_TASK_DIR:-$WORKSPACE/seekdb_build/$GITHUB_RUN_ID}"
+OUT_JSON="${1:-$WORKSPACE/seekdb_result.json}"
+
+# Success if no fail_cases or empty; otherwise fail
+FAILED=""
+if [[ -f "$TASK_DIR/fail_cases.output" ]] && [[ -s "$TASK_DIR/fail_cases.output" ]]; then
+  FAILED=$(cat "$TASK_DIR/fail_cases.output" | tr '\n' ' ' | sed 's/"/\\"/g')
+  SUCCESS="false"
+else
+  SUCCESS="true"
+fi
+
+cat > "$OUT_JSON" << EOF
+{
+  "success": ${SUCCESS},
+  "task_id": "${GITHUB_RUN_ID:-native}",
+  "output_url": "",
+  "native": true,
+  "failed_cases": "${FAILED//\"/\\\"}"
+}
+EOF
+echo "Wrote $OUT_JSON"
+cat "$OUT_JSON"

.github/script/seekdb_native/compile.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Compile step for SeekDB native execution (执行下沉).
+# 使用本仓已复制的 farm 脚本：.github/script/seekdb_native/scripts/
+set -e
+
+WORKSPACE="${GITHUB_WORKSPACE:-.}"
+TASK_DIR="${SEEKDB_TASK_DIR:-$WORKSPACE/seekdb_build/$GITHUB_RUN_ID}"
+COMPILE_RUNDIR="$TASK_DIR/compile_rundir"
+# 脚本来自本仓，不再 clone
+SCRIPTS_DIR="$WORKSPACE/.github/script/seekdb_native/scripts"
+mkdir -p "$COMPILE_RUNDIR"
+
+if [[ ! -f "$TASK_DIR/jobargs.output" ]] || [[ ! -f "$TASK_DIR/run_jobs.output" ]]; then
+  echo "Missing jobargs.output or run_jobs.output in $TASK_DIR. Run prepare first."
+  exit 1
+fi
+
+if [[ ! -f "$SCRIPTS_DIR/farm_compile.sh" ]] || [[ ! -f "$SCRIPTS_DIR/frame.sh" ]]; then
+  echo "Missing farm_compile.sh / frame.sh under $SCRIPTS_DIR. Scripts should be copied from farm-jenkins."
+  exit 1
+fi
+
+# 链到 HOME，供 source $HOME/scripts/frame.sh 与 farm_compile.sh 使用
+ln -sfn "$SCRIPTS_DIR" "$COMPILE_RUNDIR/scripts"
+
+export HOME="$COMPILE_RUNDIR"
+export _CONDOR_JOB_IWD="$COMPILE_RUNDIR"
+export REPO="server"
+export CREATE_AGENTSERVER=0
+export CREATE_LIBOBSERVER_SO=0
+export ENABLE_LIBOBLOG=0
+export BUILD_TARGET=""
+export CODE_URL="${CODE_URL:-https://github.com/${GITHUB_REPOSITORY}.git}"
+export BRANCH="${BRANCH:-$GITHUB_REF_NAME}"
+export COMMIT="${COMMIT:-}"
+if [[ -n "${RELEASE_MODE:-}" ]]; then
+  export PACKAGE_TYPE="release"
+else
+  export PACKAGE_TYPE="debug"
+fi
+if [[ -n "${FORWARDING_HOST:-}" ]]; then
+  echo "$FORWARDING_HOST mirrors.oceanbase.com" >> /etc/hosts 2>/dev/null || true
+fi
+
+# 执行 farm_compile.sh：内部 source frame.sh && main
+cd "$HOME"
+bash "$HOME/scripts/farm_compile.sh"
+COMPILE_EXIT=$?
+[[ -f "$HOME/scripts/farm_post_compile.sh" ]] && bash "$HOME/scripts/farm_post_compile.sh" "$COMPILE_EXIT" || true
+
+# 压缩并拷贝产物到 TASK_DIR
+for f in observer obproxy; do
+  [[ -f "$COMPILE_RUNDIR/$f" ]] && zstd -f "$COMPILE_RUNDIR/$f" 2>/dev/null || true
+done
+for fn in observer.zst obproxy.zst compile.output; do
+  [[ -f "$COMPILE_RUNDIR/$fn" ]] && cp "$COMPILE_RUNDIR/$fn" "$TASK_DIR/"
+done
+[[ -f "$COMPILE_RUNDIR/dep_cache.tar.zst" ]] && cp "$COMPILE_RUNDIR/dep_cache.tar.zst" "$TASK_DIR/" || true
+[[ -f "$COMPILE_RUNDIR/post_compile.output" ]] && cp "$COMPILE_RUNDIR/post_compile.output" "$TASK_DIR/" || true
+
+echo "Compile done. Artifacts in $TASK_DIR"
+ls -la "$TASK_DIR"