Skip to content

Commit 49dc7bd

Browse files
wrmsrwrmsr
committed
Add tinyllm package with CLI, web chat UI, GGUF loader, tokenizer, and transformer model implementation
Includes LICENSE, entry point scripts, an HTML chat interface, and a full transformer architecture supporting MoE, SSM, and various model formats.
1 parent 4afc6c7 commit 49dc7bd

7 files changed

Lines changed: 1958 additions & 0 deletions

File tree

‎x/tinyllm/LICENSE‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
Copyright (c) 2024, the tiny corp
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
4+
documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
5+
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
6+
persons to whom the Software is furnished to do so, subject to the following conditions:
7+
8+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
9+
Software.
10+
11+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
12+
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
13+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
14+
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎x/tinyllm/__init__.py‎

Whitespace-only changes.

‎x/tinyllm/__main__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from .cli import main
2+
3+
4+
if __name__ == '__main__':
5+
main()

‎x/tinyllm/chat.html‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
<!DOCTYPE html>
2+
<html>
3+
4+
<head>
5+
<title>tinygrad chat</title>
6+
<style>
7+
* { margin: 0 }
8+
body { background: #212121; color: #e3e3e3; font-family: system-ui;
9+
height: 100vh; display: flex; flex-direction: column }
10+
#chat { flex: 1; overflow-y: auto; padding: 20px }
11+
.msg { padding: 10px 16px; margin: 8px 0; white-space: pre-wrap; border-radius: 18px }
12+
.user { background: #2f2f2f; margin-left: auto; width: fit-content; max-width: 70% }
13+
#input { max-width: 768px; width: 100%; margin: 20px auto; padding: 14px 20px;
14+
background: #2f2f2f; color: inherit; font: inherit;
15+
border: none; outline: none; resize: none; border-radius: 24px; field-sizing: content }
16+
</style>
17+
</head>
18+
19+
<body>
20+
21+
<div id="chat"></div>
22+
23+
<textarea id="input" rows="1" placeholder="Ask anything" autofocus></textarea>
24+
25+
<script>
26+
input.onkeydown = (e) => {
27+
if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) {
28+
e.preventDefault();
29+
send()
30+
}
31+
}
32+
const msgs = [];
33+
34+
async function send() {
35+
if (!input.value.trim()) {
36+
return;
37+
}
38+
39+
msgs.push({
40+
role: 'user',
41+
content: input.value.trim()
42+
});
43+
44+
chat.innerHTML += '<div class="msg user">' + input.value.trim().replace(/</g, '&lt;') + '</div>';
45+
input.value = '';
46+
47+
const d = document.createElement('div');
48+
d.className = 'msg';
49+
chat.appendChild(d);
50+
51+
const r = await fetch(
52+
'/v1/chat/completions',
53+
{
54+
method: 'POST',
55+
headers: {
56+
'Content-Type': 'application/json'
57+
},
58+
body: JSON.stringify({model: 'llama', messages: msgs, stream: true, temperature: 0.7})
59+
}
60+
);
61+
62+
let buf = '';
63+
for (const rd = r.body.getReader(), dec = new TextDecoder(); ;) {
64+
const {done, value} = await rd.read();
65+
if (done) {
66+
break;
67+
}
68+
69+
buf += dec.decode(value, {stream: true});
70+
const lines = buf.split('\n');
71+
buf = lines.pop();
72+
73+
for (const ln of lines) {
74+
if (ln.startsWith('data: ') && !ln.includes('[DONE]')) {
75+
try {
76+
d.textContent += JSON.parse(ln.slice(6)).choices[0]?.delta?.content || ''
77+
} catch {
78+
}
79+
}
80+
}
81+
82+
chat.scrollTop = chat.scrollHeight;
83+
}
84+
85+
msgs.push({role: 'assistant', content: d.textContent});
86+
}
87+
</script>
88+
89+
</body>
90+
</html>

0 commit comments

Comments
 (0)