Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow sampler modifications #28

Merged
merged 3 commits into from
Feb 13, 2025
Merged

Allow sampler modifications #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88f9031
feat: allow sampler changing through instance, ref #23
pminev Jan 3, 2025
4f351c7
fix: deprecation warnings, ref #28
pminev Feb 13, 2025
6c4cdc9
fix: add warning for the sampler reset, ref #28
pminev Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions code/ac/llama/Instance.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ llama_context_params llamaFromInstanceInitParams(const Instance::InitParams& par

Instance::Instance(Model& model, InitParams params)
: m_model(model)
, m_sampler(model, {
, m_sampler(new Sampler(model, {
.grammar = params.grammar,
})
}))
, m_lctx(llama_init_from_model(model.lmodel(), llamaFromInstanceInitParams(params)), llama_free)
{
if (!m_lctx) {
Expand Down
11 changes: 9 additions & 2 deletions code/ac/llama/Instance.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,18 @@ class AC_LLAMA_EXPORT Instance {
void stopSession() noexcept;

const Model& model() const noexcept { return m_model; }
Sampler& sampler() noexcept { return m_sampler; }

Sampler& sampler() noexcept { return *m_sampler; }

// Change sampler settings by resetting it
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add comment: "warning: this will clear any previous sampler state"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fed

// warning: this will clear any previous sampler state
void resetSampler(const Sampler::Params& params) {
m_sampler.reset(new Sampler(m_model, params));
}

private:
Model& m_model;
Sampler m_sampler;
std::unique_ptr<Sampler> m_sampler;
astl::c_unique_ptr<llama_context> m_lctx;
std::optional<Session> m_session;
};
Expand Down
2 changes: 1 addition & 1 deletion code/ac/llama/Model.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ std::shared_ptr<llama_model> ModelRegistry::loadModel(
}

if (!model) {
model = std::shared_ptr<llama_model>(llama_load_model_from_file(gguf.c_str(), llamaFromModelParams(params, pcb)), llama_free_model);
model = std::shared_ptr<llama_model>(llama_model_load_from_file(gguf.c_str(), llamaFromModelParams(params, pcb)), llama_model_free);
m_models.push_back({key, model});
}

Expand Down
54 changes: 42 additions & 12 deletions test/t-integration.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,20 +67,50 @@ TEST_CASE("inference") {
tokens = model.vocab().tokenize("President George W.", true, true);
s.setInitialPrompt(tokens);
{
auto t = s.getToken();
REQUIRE(t != ac::llama::Token_Invalid);
auto text = model.vocab().tokenToString(t);
CHECK(text == " Bush");
auto t = s.getToken();
REQUIRE(t != ac::llama::Token_Invalid);
auto text = model.vocab().tokenToString(t);
CHECK(text == " Bush");
}

// add more very suggestive stuff
tokens = model.vocab().tokenize(" sent troops to Cleveland which was hit by torrential", false, false);
s.pushPrompt(tokens);
{
auto t = s.getToken();
REQUIRE(t != ac::llama::Token_Invalid);
auto text = model.vocab().tokenToString(t);
CHECK(text.starts_with(" rain")); // could be rains
SUBCASE("default sampler") {
// add more very suggestive stuff
tokens = model.vocab().tokenize(" sent troops to Cleveland which was hit by torrential", false, false);
s.pushPrompt(tokens);
{
auto t = s.getToken();
REQUIRE(t != ac::llama::Token_Invalid);
auto text = model.vocab().tokenToString(t);
CHECK(text.starts_with(" rain")); // could be rains
}
}

SUBCASE("custom sampler") {
ac::llama::Sampler::Params samplerParams = {};
samplerParams.rngSeed = 1717;
samplerParams.minP = 0.2f;
samplerParams.topK = 100;
samplerParams.topP = 0.2f;
samplerParams.minKeep = 1000;
samplerParams.temp = 10.0f;
samplerParams.tempExp = 5.0f;
samplerParams.samplerSequence = {
ac::llama::Sampler::SamplingType::Min_P,
ac::llama::Sampler::SamplingType::Temperature,
ac::llama::Sampler::SamplingType::Top_K,
ac::llama::Sampler::SamplingType::Top_P,
};
inst.resetSampler(samplerParams);

// add more very suggestive stuff
tokens = model.vocab().tokenize(" sent troops to Cleveland which was hit by torrential", false, false);
s.pushPrompt(tokens);
{
auto t = s.getToken();
REQUIRE(t != ac::llama::Token_Invalid);
auto text = model.vocab().tokenToString(t);
CHECK(text.starts_with(" down"));
}
}
}
}
Expand Down