num_tokens_from_messages() needs updating

[OTheDev]

tiktoken-rs/tiktoken-rs/src/api.rs

Lines 96 to 132 in 35ae83e

	pub fn num_tokens_from_messages(
	model: &str,
	messages: &[ChatCompletionRequestMessage],
	) -> Result<usize> {
	let tokenizer =
	get_tokenizer(model).ok_or_else(|| anyhow!("No tokenizer found for model {}", model))?;
	if tokenizer != Tokenizer::Cl100kBase && tokenizer != Tokenizer::O200kBase {
	anyhow::bail!("Chat completion is only supported chat models")
	}
	let bpe = get_bpe_from_tokenizer(tokenizer)?;
	let (tokens_per_message, tokens_per_name) = if model.starts_with("gpt-3.5") {
	(
	4, // every message follows <im_start>{role/name}\n{content}<im_end>\n
	-1, // if there's a name, the role is omitted
	)
	} else {
	(3, 1)
	};
	let mut num_tokens: i32 = 0;
	for message in messages {
	num_tokens += tokens_per_message;
	num_tokens += bpe
	.encode_with_special_tokens(&message.role.to_string())
	.len() as i32;
	num_tokens += bpe
	.encode_with_special_tokens(&message.content.clone().unwrap_or_default())
	.len() as i32;
	if let Some(name) = &message.name {
	num_tokens += bpe.encode_with_special_tokens(name).len() as i32;
	num_tokens += tokens_per_name;
	}
	}
	num_tokens += 3; // every reply is primed with <|start|>assistant<|message|>
	Ok(num_tokens as usize)
	}

Currently, any model with a name prefixed with gpt-3.5 is treated as having (tokens_per_message, tokens_per_name) set to (4, -1). However, this is only true of gpt-3.5-turbo-0301. Future snapshots of gpt-3.5-turbo reverted to the normal (3, 1) configuration.

AFAIK, the (4, -1) configuration shouldn't apply for all gpt-3.5 models, it should only apply for gpt-3.5-turbo-0301.