example-qwen2.env

MODEL_PATH="/absolute/path/to/model/qwen2-instruct.gguf"

BOT=""
PREFIX_TEMPLATE="<|im_start|>{agent}
"
EOS="<|im_end|>
"
AGENT_SYSTEM="system"
AGENT_USER="user"
AGENT_ASSISTANT="assistant"
SYSTEM_PROMPT="You are a helpful developer assistant, answer all the questions correctly and concisely."
ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"

REAL_TIME=1
N_CTX=4096
N_GENERATE=1024
TEMPERATURE=0.8
TOP_P=0.9
TOP_K=40
SEED=69
USE_MMAP=1
USE_MLOCK=1
USE_GPU=1