Skip to content

Commit af19eb0

Browse files
author
TangibleResearch
committed
docs and ci setup
1 parent 284d52a commit af19eb0

4 files changed

Lines changed: 324 additions & 124 deletions

File tree

‎docs/pointrun.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,12 @@ v0.1 behavior:
1212
6. If the engine is unsupported, print a clear stub message.
1313

1414
OpenAI keys are read only from `OPENAI_API_KEY`.
15+
16+
## Parsing Boundary
17+
18+
The Rust compiler owns AInfra source lexing and parsing. InfraVM should execute
19+
compiled AIF and avoid re-parsing the language.
20+
21+
The C VM does include a small JSON tokenizer/parser in `vm_json.h` for provider
22+
connector responses. This prevents fragile substring searches when model output
23+
contains conversational text that happens to mention JSON-looking keys.

‎infravm/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ LDLIBS ?= -lcurl
66

77
all: infravm
88

9-
infravm: infravm.c infravm.h vm_openai.h vm_ollama.h ../optimizer/c/optimizer.c ../optimizer/c/optimizer.h
9+
infravm: infravm.c infravm.h vm_openai.h vm_json.h vm_ollama.h ../optimizer/c/optimizer.c ../optimizer/c/optimizer.h
1010
$(CC) $(CFLAGS) -I../optimizer/c -o $@ infravm.c ../optimizer/c/optimizer.c $(LDLIBS)
1111

1212
clean:

‎infravm/vm_json.h‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
#ifndef VM_JSON_H
2+
#define VM_JSON_H
3+
4+
#ifdef __cplusplus
5+
extern "C" {
6+
#endif
7+
8+
/*
9+
* Small header-only JSON tokenizer/parser for the VM.
10+
*
11+
* This is intentionally not a full DOM parser. It walks valid JSON tokens and
12+
* extracts string values by key without substring searching inside model text.
13+
*/
14+
15+
char *vm_json_find_string_value(const char *json, const char *key);
16+
17+
#ifdef __cplusplus
18+
}
19+
#endif
20+
21+
#endif
22+
23+
#if defined(VM_JSON_IMPLEMENTATION) && !defined(VM_JSON_IMPLEMENTED)
24+
#define VM_JSON_IMPLEMENTED
25+
26+
#include <ctype.h>
27+
#include <stdlib.h>
28+
#include <string.h>
29+
30+
typedef enum {
31+
VM_JSON_EOF = 0,
32+
VM_JSON_LBRACE,
33+
VM_JSON_RBRACE,
34+
VM_JSON_LBRACKET,
35+
VM_JSON_RBRACKET,
36+
VM_JSON_COLON,
37+
VM_JSON_COMMA,
38+
VM_JSON_STRING,
39+
VM_JSON_NUMBER,
40+
VM_JSON_TRUE,
41+
VM_JSON_FALSE,
42+
VM_JSON_NULL,
43+
VM_JSON_ERROR
44+
} VMJsonTokenKind;
45+
46+
typedef struct {
47+
VMJsonTokenKind kind;
48+
char *text;
49+
} VMJsonToken;
50+
51+
typedef struct {
52+
const char *src;
53+
size_t pos;
54+
} VMJsonParser;
55+
56+
static char *vm_json_strdup_range(const char *start, size_t len) {
57+
char *out = (char *)malloc(len + 1);
58+
if (!out) return NULL;
59+
memcpy(out, start, len);
60+
out[len] = '\0';
61+
return out;
62+
}
63+
64+
static int vm_json_append_char(char **out, size_t *len, size_t *cap, char ch) {
65+
char *next;
66+
if (*len + 2 > *cap) {
67+
size_t next_cap = *cap ? *cap * 2 : 32;
68+
next = (char *)realloc(*out, next_cap);
69+
if (!next) return 0;
70+
*out = next;
71+
*cap = next_cap;
72+
}
73+
(*out)[(*len)++] = ch;
74+
(*out)[*len] = '\0';
75+
return 1;
76+
}
77+
78+
static void vm_json_skip_ws(VMJsonParser *parser) {
79+
while (parser->src[parser->pos] && isspace((unsigned char)parser->src[parser->pos])) {
80+
parser->pos++;
81+
}
82+
}
83+
84+
static char *vm_json_read_string(VMJsonParser *parser) {
85+
char *out = NULL;
86+
size_t len = 0;
87+
size_t cap = 0;
88+
char ch;
89+
90+
if (parser->src[parser->pos] != '"') return NULL;
91+
parser->pos++;
92+
while ((ch = parser->src[parser->pos++]) != '\0') {
93+
if (ch == '"') {
94+
return out ? out : vm_json_strdup_range("", 0);
95+
}
96+
if (ch == '\\') {
97+
ch = parser->src[parser->pos++];
98+
switch (ch) {
99+
case '"':
100+
case '\\':
101+
case '/':
102+
if (!vm_json_append_char(&out, &len, &cap, ch)) goto oom;
103+
break;
104+
case 'b':
105+
if (!vm_json_append_char(&out, &len, &cap, '\b')) goto oom;
106+
break;
107+
case 'f':
108+
if (!vm_json_append_char(&out, &len, &cap, '\f')) goto oom;
109+
break;
110+
case 'n':
111+
if (!vm_json_append_char(&out, &len, &cap, '\n')) goto oom;
112+
break;
113+
case 'r':
114+
if (!vm_json_append_char(&out, &len, &cap, '\r')) goto oom;
115+
break;
116+
case 't':
117+
if (!vm_json_append_char(&out, &len, &cap, '\t')) goto oom;
118+
break;
119+
case 'u':
120+
/* Keep unicode escapes stable without pretending to decode UTF-16. */
121+
if (!vm_json_append_char(&out, &len, &cap, '\\')) goto oom;
122+
if (!vm_json_append_char(&out, &len, &cap, 'u')) goto oom;
123+
for (int i = 0; i < 4; i++) {
124+
ch = parser->src[parser->pos++];
125+
if (!isxdigit((unsigned char)ch)) goto oom;
126+
if (!vm_json_append_char(&out, &len, &cap, ch)) goto oom;
127+
}
128+
break;
129+
default:
130+
goto oom;
131+
}
132+
} else {
133+
if ((unsigned char)ch < 0x20) goto oom;
134+
if (!vm_json_append_char(&out, &len, &cap, ch)) goto oom;
135+
}
136+
}
137+
138+
oom:
139+
free(out);
140+
return NULL;
141+
}
142+
143+
static int vm_json_match_literal(VMJsonParser *parser, const char *literal) {
144+
size_t len = strlen(literal);
145+
if (strncmp(parser->src + parser->pos, literal, len) != 0) return 0;
146+
parser->pos += len;
147+
return 1;
148+
}
149+
150+
static VMJsonToken vm_json_next(VMJsonParser *parser) {
151+
VMJsonToken token;
152+
const char *start;
153+
char ch;
154+
memset(&token, 0, sizeof(token));
155+
vm_json_skip_ws(parser);
156+
ch = parser->src[parser->pos];
157+
if (!ch) {
158+
token.kind = VM_JSON_EOF;
159+
return token;
160+
}
161+
parser->pos++;
162+
switch (ch) {
163+
case '{': token.kind = VM_JSON_LBRACE; return token;
164+
case '}': token.kind = VM_JSON_RBRACE; return token;
165+
case '[': token.kind = VM_JSON_LBRACKET; return token;
166+
case ']': token.kind = VM_JSON_RBRACKET; return token;
167+
case ':': token.kind = VM_JSON_COLON; return token;
168+
case ',': token.kind = VM_JSON_COMMA; return token;
169+
case '"':
170+
parser->pos--;
171+
token.kind = VM_JSON_STRING;
172+
token.text = vm_json_read_string(parser);
173+
if (!token.text) token.kind = VM_JSON_ERROR;
174+
return token;
175+
case 't':
176+
parser->pos--;
177+
token.kind = vm_json_match_literal(parser, "true") ? VM_JSON_TRUE : VM_JSON_ERROR;
178+
return token;
179+
case 'f':
180+
parser->pos--;
181+
token.kind = vm_json_match_literal(parser, "false") ? VM_JSON_FALSE : VM_JSON_ERROR;
182+
return token;
183+
case 'n':
184+
parser->pos--;
185+
token.kind = vm_json_match_literal(parser, "null") ? VM_JSON_NULL : VM_JSON_ERROR;
186+
return token;
187+
default:
188+
if (ch == '-' || isdigit((unsigned char)ch)) {
189+
parser->pos--;
190+
start = parser->src + parser->pos;
191+
if (parser->src[parser->pos] == '-') parser->pos++;
192+
while (isdigit((unsigned char)parser->src[parser->pos])) parser->pos++;
193+
if (parser->src[parser->pos] == '.') {
194+
parser->pos++;
195+
while (isdigit((unsigned char)parser->src[parser->pos])) parser->pos++;
196+
}
197+
if (parser->src[parser->pos] == 'e' || parser->src[parser->pos] == 'E') {
198+
parser->pos++;
199+
if (parser->src[parser->pos] == '+' || parser->src[parser->pos] == '-') parser->pos++;
200+
while (isdigit((unsigned char)parser->src[parser->pos])) parser->pos++;
201+
}
202+
token.kind = VM_JSON_NUMBER;
203+
token.text = vm_json_strdup_range(start, (size_t)(parser->src + parser->pos - start));
204+
if (!token.text) token.kind = VM_JSON_ERROR;
205+
return token;
206+
}
207+
token.kind = VM_JSON_ERROR;
208+
return token;
209+
}
210+
}
211+
212+
static void vm_json_free_token(VMJsonToken *token) {
213+
if (!token) return;
214+
free(token->text);
215+
token->text = NULL;
216+
}
217+
218+
static int vm_json_search_from_token(VMJsonParser *parser, VMJsonToken token, const char *key, char **out);
219+
220+
static int vm_json_search_object(VMJsonParser *parser, const char *key, char **out) {
221+
for (;;) {
222+
VMJsonToken name = vm_json_next(parser);
223+
VMJsonToken colon;
224+
VMJsonToken value;
225+
int matched;
226+
227+
if (name.kind == VM_JSON_RBRACE) return 0;
228+
if (name.kind != VM_JSON_STRING) {
229+
vm_json_free_token(&name);
230+
return 0;
231+
}
232+
colon = vm_json_next(parser);
233+
if (colon.kind != VM_JSON_COLON) {
234+
vm_json_free_token(&name);
235+
return 0;
236+
}
237+
value = vm_json_next(parser);
238+
matched = strcmp(name.text, key) == 0;
239+
vm_json_free_token(&name);
240+
if (matched && value.kind == VM_JSON_STRING) {
241+
*out = value.text;
242+
value.text = NULL;
243+
return 1;
244+
}
245+
if (vm_json_search_from_token(parser, value, key, out)) return 1;
246+
value = vm_json_next(parser);
247+
if (value.kind == VM_JSON_RBRACE) return 0;
248+
if (value.kind != VM_JSON_COMMA) {
249+
vm_json_free_token(&value);
250+
return 0;
251+
}
252+
}
253+
}
254+
255+
static int vm_json_search_array(VMJsonParser *parser, const char *key, char **out) {
256+
for (;;) {
257+
VMJsonToken value = vm_json_next(parser);
258+
if (value.kind == VM_JSON_RBRACKET) return 0;
259+
if (vm_json_search_from_token(parser, value, key, out)) return 1;
260+
value = vm_json_next(parser);
261+
if (value.kind == VM_JSON_RBRACKET) return 0;
262+
if (value.kind != VM_JSON_COMMA) {
263+
vm_json_free_token(&value);
264+
return 0;
265+
}
266+
}
267+
}
268+
269+
static int vm_json_search_from_token(VMJsonParser *parser, VMJsonToken token, const char *key, char **out) {
270+
switch (token.kind) {
271+
case VM_JSON_LBRACE:
272+
return vm_json_search_object(parser, key, out);
273+
case VM_JSON_LBRACKET:
274+
return vm_json_search_array(parser, key, out);
275+
case VM_JSON_STRING:
276+
case VM_JSON_NUMBER:
277+
vm_json_free_token(&token);
278+
return 0;
279+
default:
280+
return 0;
281+
}
282+
}
283+
284+
char *vm_json_find_string_value(const char *json, const char *key) {
285+
VMJsonParser parser;
286+
VMJsonToken token;
287+
char *out = NULL;
288+
289+
if (!json || !key || !*key) return NULL;
290+
parser.src = json;
291+
parser.pos = 0;
292+
token = vm_json_next(&parser);
293+
if (vm_json_search_from_token(&parser, token, key, &out)) return out;
294+
return NULL;
295+
}
296+
297+
#endif

0 commit comments

Comments
 (0)