TML Inkling deployment guideline

[lightseek-bot]

Blog

https://lightseek.org/blog/tokenspeed-inkling.html

Docker

# nvidia
docker pull lightseekorg/tokenspeed:tml
# amd
docker pull lightseekorg/tokenspeed-amd:tml

Launch command

# nvidia
ts serve \
    --model thinkingmachines/Inkling-NVFP4 \
    --attn-tp-size 4 \
    --moe-tp-size 4 \
    --max-model-len 81920 \
    --max-num-seqs 16 \
    --max-prefill-tokens 8192 \
    --chunked-prefill-size 8192 \
    --gpu-memory-utilization 0.95 \
    --disable-cuda-graph-padding \
    --trust-remote-code \
    --attention-backend fa4 \
    --moe-backend flashinfer_trtllm \
    --enable-prefix-caching \
    --disable-kvstore \
    --block-size 128 \
    --enable-cache-report \
    --speculative-algorithm MTP \
    --speculative-num-steps 3 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 4

# amd
ts serve \
    --model lightseekorg/Inkling-MXFP4 \
    --attn-tp-size 4 \
    --moe-tp-size 4 \
    --max-model-len 81920 \
    --max-num-seqs 16 \
    --max-prefill-tokens 8192 \
    --chunked-prefill-size 8192 \
    --gpu-memory-utilization 0.95 \
    --disable-cuda-graph-padding \
    --trust-remote-code \
    --enable-prefix-caching \
    --disable-kvstore \
    --block-size 128 \
    --enable-cache-report \
    --speculative-algorithm MTP \
    --speculative-num-steps 3 \
    --speculative-eagle-topk 1 \
    --speculative-num-draft-tokens 4

PR

TokenSpeed PR #689
SMG PR smg-project/smg#1926

Dependency (for source installation)

# tokenspeed-scheduler
# Turn on Flat KV Cache
pip install tokenspeed-scheduler==0.1.2

# smg
pip install tokenspeed-smg==1.7.0.post20260715
pip install tokenspeed-smg-grpc-proto==0.4.14.post20260715
pip install tokenspeed-smg-grpc-servicer==0.6.0.post20260715