feat(router): add cache_aware_length policy with long/short pool split

.github/workflows/build-docker-images.yml

@@ -0,0 +1,211 @@
+name: Build Docker Images
+
+on:
+  push:
+    branches: ["main", "test"]
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_PREFIX: ${{ github.repository_owner }}/1xtoken
+  CLUSTER_GOPROXY: https://proxy.golang.org
+  # 阿里云镜像仓库：国内服务器拉 ghcr.io 太慢，构建时同步推一份到阿里云
+  ALIYUN_REGISTRY: crpi-w0utadukedw6a9ld.cn-hangzhou.personal.cr.aliyuncs.com
+  ALIYUN_NAMESPACE: 1xtoken
+
+jobs:
+  # ------------------------------------------------------------------
+  # 阶段 1: 智能分析 (Configuration Job)
+  # ------------------------------------------------------------------
+  configure:
+    name: Configure Matrix
+    runs-on: astro-web3-arc-runner-set
+    outputs:
+      has-changes: ${{ steps.detect.outputs.has-changes }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect changes to build inputs
+        id: detect
+        run: |
+          # 只在影响镜像构建的文件变更时才触发构建，避免无关改动白白排队。
+          # workflow_dispatch 始终构建。
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "has-changes=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          BEFORE="${{ github.event.before }}"
+          AFTER="${{ github.event.after }}"
+          if [ -z "$BEFORE" ] || [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then
+            echo "has-changes=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BEFORE" "$AFTER" || true)
+          PATTERNS='^(docker/Dockerfile|\.cargo/config\.toml|bindings/python/pyproject\.toml|model_gateway/|crates/|Cargo\.lock)'
+          if printf '%s\n' "$CHANGED" | grep -qE "$PATTERNS"; then
+            echo "has-changes=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has-changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build matrix
+        id: matrix
+        run: |
+          # 单镜像仓库，matrix 只有一行；保留 include 结构便于后续多 app 扩展。
+          MATRIX=$(cat <<'JSON'
+          {"include":[{"app":"smg","dockerfile":"docker/Dockerfile","context":"."}]}
+          JSON
+          )
+          # jq -c 压成单行 JSON 给 fromJSON 用
+          echo "matrix=$(printf '%s' "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
+
+  # ------------------------------------------------------------------
+  # 阶段 2: 构建 (Build Job)
+  # ------------------------------------------------------------------
+  build-and-push:
+    name: Build ${{ matrix.app }}
+    needs: configure
+    if: needs.configure.outputs.has-changes == 'true'
+    runs-on: astro-web3-arc-runner-set
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.configure.outputs.matrix) }}
+    steps:
+      - name: Export pod env to GitHub env
+        run: |
+          echo "ZOT_REGISTRY=${ZOT_REGISTRY}" >> $GITHUB_ENV
+          echo "CLUSTER_GOPROXY=${CLUSTER_GOPROXY}" >> $GITHUB_ENV
+          echo "BUILDKIT_HOST=${BUILDKIT_HOST}" >> $GITHUB_ENV
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: ${{ env.BUILDKIT_HOST }}
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # 阿里云镜像仓库偶发 TLS 握手超时（自建 runner 到杭州个人版实例网络抖动），
+      # docker/login-action 内部只 login 一次无重试，这里改成带退避的重试。
+      - name: Log in to Aliyun Registry
+        env:
+          ALIYUN_REGISTRY: ${{ env.ALIYUN_REGISTRY }}
+          ALIYUN_USERNAME: ${{ secrets.ALIYUN_USERNAME }}
+          ALIYUN_PASSWORD: ${{ secrets.ALIYUN_PASSWORD }}
+        run: |
+          max_attempts=5
+          for attempt in $(seq 1 "$max_attempts"); do
+            if printf '%s' "$ALIYUN_PASSWORD" | docker login "$ALIYUN_REGISTRY" -u "$ALIYUN_USERNAME" --password-stdin; then
+              echo "Aliyun login succeeded on attempt $attempt"
+              exit 0
+            fi
+            echo "Aliyun login attempt $attempt failed, backing off before retry..."
+            sleep $((attempt * 5))
+          done
+          echo "::error::Aliyun registry login failed after $max_attempts attempts"
+          exit 1
+
+      - name: Metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}
+            ${{ env.ZOT_REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}
+            ${{ env.ALIYUN_REGISTRY }}/${{ env.ALIYUN_NAMESPACE }}/1xtoken-${{ matrix.app }}
+          tags: |
+            type=sha,prefix=${{ github.ref_name }}-,suffix=-${{ github.run_number }}
+
+      # 构建与推送拆分：构建是确定性故障（pnpm install、Dockerfile 语法、编译错误），
+      # 重试无意义，失败直接在这一步标红并阻断 job，报错落在真实出错位置；
+      # 推送才可能因阿里云个人版仓库鉴权/网络抖动失败，值得退避重试。
+      - name: Build image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.context || '.' }}
+          file: ${{ matrix.dockerfile }}
+          push: false
+          load: false
+          cache-from: type=registry,ref=${{ env.ZOT_REGISTRY }}/cache/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}:buildcache
+          cache-to: type=registry,ref=${{ env.ZOT_REGISTRY }}/cache/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}:buildcache,mode=min
+
+      # 阿里云鉴权端点偶发 connection reset by peer（push 时拉 oauth token 网络抖动），
+      # docker/build-push-action 内部不重试推送，这里用「首尝试 + 2 次退避重试 + 状态校验」兜底。
+      # 构建产物已在 BuildKit 缓存里，重试几乎只重推 manifest，成本可忽略。
+      - name: Push image
+        id: push-1
+        continue-on-error: true
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.context || '.' }}
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=${{ env.ZOT_REGISTRY }}/cache/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}:buildcache
+
+      - name: Retry push (attempt 2)
+        if: steps.push-1.outcome == 'failure'
+        run: |
+          echo "::warning::首次推送失败（可能阿里云鉴权端点网络抖动），退避 5s 后重试..."
+          sleep 5
+
+      - name: Push image (retry 2)
+        id: push-2
+        continue-on-error: true
+        if: steps.push-1.outcome == 'failure'
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.context || '.' }}
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=${{ env.ZOT_REGISTRY }}/cache/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}:buildcache
+
+      - name: Retry push (attempt 3)
+        if: steps.push-2.outcome == 'failure'
+        run: |
+          echo "::warning::第二次推送失败，退避 10s 后最后一次重试..."
+          sleep 10
+
+      - name: Push image (retry 3)
+        id: push-3
+        continue-on-error: true
+        if: steps.push-2.outcome == 'failure'
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.context || '.' }}
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=${{ env.ZOT_REGISTRY }}/cache/${{ env.IMAGE_PREFIX }}-${{ matrix.app }}:buildcache
+
+      # 三次推送任一成功即通过；全败则标红并阻断 job。
+      - name: Verify push result
+        if: always()
+        run: |
+          if [ "${{ steps.push-1.outcome }}" = "success" ] \
+             || [ "${{ steps.push-2.outcome }}" = "success" ] \
+             || [ "${{ steps.push-3.outcome }}" = "success" ]; then
+            echo "镜像推送成功"
+          else
+            echo "::error::镜像推送在 3 次尝试后均失败（阿里云个人版仓库鉴权/网络问题）"
+            exit 1
+          fi

model_gateway/src/config/types.rs

@@ -644,6 +644,45 @@ pub enum PolicyConfig {
         cache_boundaries: Vec<usize>,
     },
 
+    /// Cache-aware length policy: cache affinity with a long/short pool split
+    /// driven by the `pool` worker label (`pool=long` → long pool, otherwise
+    /// short pool). Step 1-3 mirror `cache_aware` (string tree only); step 4
+    /// routes by uncached prefill tokens. See `policies/cache_aware_length.rs`.
+    #[serde(rename = "cache_aware_length")]
+    CacheAwareLength {
+        /// Minimum matched-prefix share before a request pins to a holder.
+        #[serde(alias = "cache_match_threshold")]
+        #[serde(default = "default_cal_cache_threshold")]
+        cache_threshold: f32,
+        /// Spill gate, absolute part: the global imbalance fires when the
+        /// healthy-fleet load spread exceeds this.
+        #[serde(alias = "spill_abs_threshold")]
+        #[serde(default = "default_cal_balance_abs_threshold")]
+        balance_abs_threshold: usize,
+        /// Spill gate, relative part (multiple of the healthy-fleet min load);
+        /// fires only together with `balance_abs_threshold`.
+        #[serde(alias = "spill_rel_threshold")]
+        #[serde(default = "default_cal_balance_rel_threshold")]
+        balance_rel_threshold: f32,
+        #[serde(default = "default_cal_eviction_interval_secs")]
+        eviction_interval_secs: u64,
+        #[serde(default = "default_cal_max_tree_size")]
+        max_tree_size: usize,
+        /// Divisor for char-level token estimation when `X-Prompt-Tokens` is
+        /// absent (default 4).
+        #[serde(default = "default_cal_chars_per_token")]
+        chars_per_token: usize,
+        /// Uncached-prefill-token boundary between long and short requests.
+        #[serde(default = "default_cal_long_prefill_threshold")]
+        long_prefill_threshold: usize,
+        /// Load ceiling for the long pool (`pool=long` workers).
+        #[serde(default = "default_cal_long_pool_max_load")]
+        long_pool_max_load: usize,
+        /// Load ceiling for the short pool (remaining workers).
+        #[serde(default = "default_cal_short_pool_max_load")]
+        short_pool_max_load: usize,
+    },
+
     /// Power-of-two choices policy: samples two workers and routes to the one
     /// with the lower expected wait, scored like `least_load`
     /// (`(queued_tokens + inflight_tokens) / throughput + kv_pressure_weight * k/(1-k)`).
@@ -777,6 +816,35 @@ fn default_cache_ttl_secs() -> u64 {
     180
 }
 
+// cache_aware_length defaults (kept aligned with CacheAwareLengthConfig::default).
+fn default_cal_cache_threshold() -> f32 {
+    0.3
+}
+fn default_cal_balance_abs_threshold() -> usize {
+    32
+}
+fn default_cal_balance_rel_threshold() -> f32 {
+    1.1
+}
+fn default_cal_eviction_interval_secs() -> u64 {
+    30
+}
+fn default_cal_max_tree_size() -> usize {
+    10000
+}
+fn default_cal_chars_per_token() -> usize {
+    4
+}
+fn default_cal_long_prefill_threshold() -> usize {
+    100_000
+}
+fn default_cal_long_pool_max_load() -> usize {
+    4
+}
+fn default_cal_short_pool_max_load() -> usize {
+    32
+}
+
 fn default_prefix_token_count() -> usize {
     256
 }
@@ -828,6 +896,7 @@ impl PolicyConfig {
             PolicyConfig::RoundRobin => "round_robin",
             PolicyConfig::Passthrough => "passthrough",
             PolicyConfig::CacheAware { .. } => "cache_aware",
+            PolicyConfig::CacheAwareLength { .. } => "cache_aware_length",
             PolicyConfig::PowerOfTwo { .. } => "power_of_two",
             PolicyConfig::LeastLoad { .. } => "least_load",
             PolicyConfig::Bucket { .. } => "bucket",

model_gateway/src/config/validation.rs

@@ -565,6 +565,81 @@ impl ConfigValidator {
                     });
                 }
             }
+            PolicyConfig::CacheAwareLength {
+                cache_threshold,
+                balance_abs_threshold: _,
+                balance_rel_threshold,
+                eviction_interval_secs,
+                max_tree_size,
+                chars_per_token,
+                long_prefill_threshold,
+                long_pool_max_load,
+                short_pool_max_load,
+            } => {
+                if !(0.0..=1.0).contains(cache_threshold) {
+                    return Err(ConfigError::InvalidValue {
+                        field: "cache_threshold".to_string(),
+                        value: cache_threshold.to_string(),
+                        reason: "Must be between 0.0 and 1.0".to_string(),
+                    });
+                }
+
+                if *balance_rel_threshold < 1.0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "balance_rel_threshold".to_string(),
+                        value: balance_rel_threshold.to_string(),
+                        reason: "Must be >= 1.0".to_string(),
+                    });
+                }
+
+                if *eviction_interval_secs == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "eviction_interval_secs".to_string(),
+                        value: eviction_interval_secs.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+
+                if *max_tree_size == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "max_tree_size".to_string(),
+                        value: max_tree_size.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+
+                if *chars_per_token == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "chars_per_token".to_string(),
+                        value: chars_per_token.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+
+                if *long_prefill_threshold == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "long_prefill_threshold".to_string(),
+                        value: long_prefill_threshold.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+
+                if *long_pool_max_load == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "long_pool_max_load".to_string(),
+                        value: long_pool_max_load.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+
+                if *short_pool_max_load == 0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "short_pool_max_load".to_string(),
+                        value: short_pool_max_load.to_string(),
+                        reason: "Must be > 0".to_string(),
+                    });
+                }
+            }
             PolicyConfig::PowerOfTwo {
                 load_check_interval_secs,
             } => {