diff --git a/BUILD.bazel b/BUILD.bazel index 3569ed78a2..44aa66744b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -212,3 +212,14 @@ alias( name = "urbit", actual = "//pkg/vere:urbit", ) + +load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile_commands") +refresh_compile_commands( + name = "refresh_compile_commands", + + # Specify the targets of interest. + # For example, specify a dict of targets and any flags required to build. + targets = { + "//:urbit": "", + }, +) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 39ddfe03e9..0f9579e5c5 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -419,3 +419,22 @@ container_pull( registry = "docker.io", repository = "alpine", ) + + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + +# Hedron's Compile Commands Extractor for Bazel +# https://github.com/hedronvision/bazel-compile-commands-extractor +http_archive( + name = "hedron_compile_commands", + + # Replace the commit hash (daae6f40adfa5fdb7c89684cbe4d88b691c63b2d) in both places (below) with the latest (https://github.com/hedronvision/bazel-compile-commands-extractor/commits/main), rather than using the stale one here. + # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README). + url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/eac41eefb5c19d9a2d2bcdd60d6989660288333d.tar.gz", + strip_prefix = "bazel-compile-commands-extractor-eac41eefb5c19d9a2d2bcdd60d6989660288333d", + # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." +) +load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") +hedron_compile_commands_setup() + diff --git a/pkg/noun/BUILD.bazel b/pkg/noun/BUILD.bazel index a6b8de6d7a..fd6963cb82 100644 --- a/pkg/noun/BUILD.bazel +++ b/pkg/noun/BUILD.bazel @@ -40,6 +40,7 @@ vere_library( "@sigsegv", "@softfloat", "@urcrypt", + "@zlib", ] + select({ "@platforms//os:macos": ["//pkg/noun/platform/darwin"], "@platforms//os:linux": ["//pkg/noun/platform/linux"], diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c new file mode 100644 index 0000000000..7368c9af76 --- /dev/null +++ b/pkg/noun/jets/e/git.c @@ -0,0 +1,352 @@ +#include + +#include "jets/q.h" +#include "jets/w.h" +#include "c3.h" + +#include "allocate.h" +#include "noun.h" + +static u3_atom _q_octs; + +static inline c3_y _cord_cmp(u3_atom cord, const char* s) +{ + u3a_atom* cord_p = u3a_to_ptr(cord); + const c3_y* c = (c3_y*)cord_p->buf_w; + c3_w len_w = u3r_met(3, cord); + + if (len_w == 0 || *s == '\0' ) { + return (len_w == 0 && *s == '\0') ? c3y : c3n; + } + + do { + if (*(c++) != *(s++)) { + return c3n; + } + + } while (--len_w && *s != '\0'); + + return (len_w == 0 && *s == '\0') ? c3y : c3n; +} + +static c3_y* _unpack_octs(u3_atom p_octs, u3_atom* q_octs_p, c3_w* len_wp) { + + c3_y* buf_y; + c3_w len_w; + + u3_atom q_octs = *q_octs_p; + + u3_assert(c3y == u3a_is_cat(p_octs)); + + if (c3y == u3a_is_cat(q_octs)) { + + buf_y = (c3_y*)q_octs_p; + len_w = sizeof(q_octs); + } + else { + u3a_atom* a = u3a_to_ptr(q_octs); + buf_y = (c3_y*)a->buf_w; + len_w = u3r_met(3, q_octs); + } + + u3_assert(len_w <= p_octs); + + *len_wp = len_w; + + return buf_y; +} + +static inline int _read_octs_byte(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { + + c3_w pos_w = *pos_wp; + + u3_assert(buf_len_w <= len_w); + u3_assert(pos_w < len_w); + + c3_y bat_y = 0; + + if (pos_w < buf_len_w) { + bat_y = *(buf_y + pos_w); + } + + (*pos_wp)++; + + return bat_y; +} + +static c3_w _read_size(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { + + c3_y bat_y = 0; + c3_w bits = 0; + c3_w size = 0; + + c3_w pos_w = *pos_wp; + + // fprintf(stderr, "_read_size: ***\r\n"); + while (pos_w < len_w) { + bat_y = _read_octs_byte(buf_y, &pos_w, buf_len_w, len_w); + + // fprintf(stderr, "_read_size [%d/%d]: 0x%hhx\r\n", pos_w, len_w, bat_y); + + size += (bat_y & 0x7f) << bits; + bits += 7; + + if (!(bat_y & 0x80)) break; + } + + *pos_wp = pos_w; + + return size; +} + +u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, + u3_noun delta) { + + // +$ raw-object [type=object-type size=@ud data=octs] + + /* +$ raw-object [type=object-type size=@ud data=octs] + +$ pack-object $% raw-object + [%ofs-delta pos=@ud base-offset=@ud =octs] + [%ref-delta pos=@ud =hash =octs] + == + +$ pack-delta-object $>(?(%ofs-delta %ref-delta) pack-object) + */ + + // base=raw-object + // + u3_atom base_type; + u3_atom base_size; + u3_noun base_data; + + u3_atom base_data_p_octs; + u3_atom base_data_q_octs; + + u3x_trel(base, &base_type, &base_size, &base_data); + u3x_cell(base_data, &base_data_p_octs, &base_data_q_octs); + + // delta=pack-object + // + u3_atom delta_type; + u3_noun delta_obj; + + u3_atom delta_pos; + u3_atom delta_base_offset; + u3_noun delta_octs; + + u3_atom delta_p_octs; + u3_atom delta_q_octs; + + u3x_cell(delta, &delta_type, &delta_obj); + + if (c3y == _cord_cmp(delta_type, "ref-delta")) { + return u3_none; + } + if (c3y == _cord_cmp(delta_type, "ofs-delta")) { + u3x_trel(delta_obj, &delta_pos, &delta_base_offset, &delta_octs); + } + + u3x_cell(delta_octs, &delta_p_octs, &delta_q_octs); + + c3_y* sea_y; + c3_y* sea_begin_y; + c3_w sea_pos_w; + + c3_w sea_buf_len_w; + c3_w sea_len_w; + + sea_y = _unpack_octs(delta_p_octs, &delta_q_octs, &sea_buf_len_w); + sea_begin_y = sea_y; + sea_len_w = delta_p_octs; + sea_pos_w = 0; + + c3_y* bas_y; + c3_y* bas_begin_y; + + c3_w bas_buf_len_w; + c3_w bas_len_w; + + bas_y = _unpack_octs(base_data_p_octs, &base_data_q_octs, &bas_buf_len_w); + u3_assert(base_size == base_data_p_octs); + bas_begin_y = bas_y; + bas_len_w = base_data_p_octs; + + /* Base object size (biz) and target + * object size (siz) + */ + // XX should exit cleanly + c3_w biz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); + c3_w siz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); + + // fprintf(stderr, "u3qe__pack_expand_delta_object: biz_w = %d, siz_w = %d\r\n", biz_w, siz_w); + + // Base size mismatch + // + if (biz_w != base_data_p_octs) { + fprintf(stderr, "bas_buf_len_w = %d, bas_len_w = %d\r\n", bas_buf_len_w, bas_len_w); + fprintf(stderr, "sea_pos = %d, sea_buf_len = %d, sea_len = %d\r\n", sea_pos_w, sea_buf_len_w, sea_len_w); + // u3_assert(false); + // _free(); + return u3_none; + } + + // Target buffer + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, siz_w); + + c3_y* tar_y = sab_u.buf_y; + c3_w tar_len_w = siz_w; + + c3_y bat_y; + + while (sea_pos_w < sea_len_w) { + + bat_y = _read_octs_byte(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); + + // XX ?> (lth pos.sea p.octs.sea) + if ( 0x0 == bat_y ) { + fprintf(stderr, "u3qe__pack_expand_delta_object: hit reserved instruction 0x0\r\n"); + + u3m_bail(c3__fail); + } + else { + /* ADD instruction + */ + if (!(bat_y & 0x80)) { + + c3_w siz_w = bat_y & 0x7f; + + if (sea_len_w - sea_pos_w < siz_w) { + fprintf(stderr, "u3qe__pack_expand_delta_object: invalid add instruction\r\n"); + + u3_assert(false); + return u3_none; + } + + // fprintf(stderr, "u3qe__pack_expand_delta: ADD[siz_w = %d]\r\n", siz_w); + + if (tar_len_w < siz_w) { + fprintf(stderr, "u3qe__pack_expand_delta: ADD overflowed\r\n"); + + u3_assert(false); + return u3_none; + } + + // Some part to be copied falls inside + // the atom buffer + // + if (sea_buf_len_w > sea_pos_w) { + + c3_w cin_w = (sea_buf_len_w - sea_pos_w); + + if (siz_w < cin_w) { + memcpy(tar_y, sea_y + sea_pos_w, siz_w); + } + else { + memcpy(tar_y, sea_y + sea_pos_w, cin_w); + memset(tar_y+cin_w, 0, siz_w - cin_w); + } + } + else { + memset(tar_y, 0, siz_w); + } + + sea_pos_w += siz_w; + + tar_y += siz_w; + tar_len_w -= siz_w; + } + /* COPY instruction + */ + else { + + /* Retrieve offset and size + */ + c3_w off_w = 0; + c3_w siz_w = 0; + +#define _parse_cp_param(bit, var, shift) \ + { \ + if (bat_y & (bit)) { \ + if (!(sea_pos_w < sea_len_w)) { \ + u3_assert(false); \ + return u3_none; \ + } \ + var |= _read_octs_byte(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w) << shift; \ + } \ + } \ + + /* Parse offset + */ + _parse_cp_param(0x1, off_w, 0); + _parse_cp_param(0x2, off_w, 8); + _parse_cp_param(0x4, off_w, 16); + _parse_cp_param(0x8, off_w, 24); + + /* Parse size + */ + _parse_cp_param(0x10, siz_w, 0); + _parse_cp_param(0x20, siz_w, 8); + _parse_cp_param(0x40, siz_w, 16); + + if (siz_w == 0) { + siz_w = 0x10000; + } + + if (tar_len_w < siz_w || (bas_len_w - off_w) < siz_w) { + fprintf(stderr, "u3qe__pack_expand_delta: copy out of range\r\n"); + u3_assert(false); + return u3_none; + } + + // fprintf(stderr, "u3qe__pack_expand_delta: COPY[siz_w = %d, off_w = %d]\r\n", siz_w, off_w); + + // Region to be copied overlaps with the atom buffer + // + if (bas_buf_len_w > off_w) { + + c3_w cin_w = (bas_buf_len_w - off_w); + + // Region to be copied is wholly inside + // + if (siz_w < cin_w) { + memcpy(tar_y, bas_y + off_w, siz_w); + } + // Region to be copied is partially inside + // + else { + memcpy(tar_y, bas_y + off_w, cin_w); + memset(tar_y+cin_w, 0, siz_w - cin_w); + } + } + else { + memset(tar_y, 0, siz_w); + } + + tar_y += siz_w; + tar_len_w -= siz_w; + } + } + } + + if (tar_len_w) { + fprintf(stderr, "u3qe__pack_expand_delta: target object underfilled (%d bytes left)\r\n", tar_len_w); + u3_assert(false); + return u3_none; + } + + u3_noun data = u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u)); + u3_noun rob = u3nt(u3k(base_type), u3i_chub(siz_w), data); + + return rob; +} + +u3_noun u3we_git_pack_expand_delta_object(u3_noun cor) { + + u3_noun base; + u3_noun delta; + + u3x_mean(cor, u3x_sam_2, &base, + u3x_sam_3, &delta, 0); + + return u3qe_git_pack_expand_delta_object(base, delta); +} diff --git a/pkg/noun/jets/e/git_http.c b/pkg/noun/jets/e/git_http.c new file mode 100644 index 0000000000..1280437ef9 --- /dev/null +++ b/pkg/noun/jets/e/git_http.c @@ -0,0 +1,287 @@ + +#include "c3.h" + +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + +#define FLUSH_PKT 0 +#define DELIM_PKT 1 +#define END_PKT 2 + +#define PKT_LINE_MAX ((1 << 16) - 1 - 4) + +static c3_w _hex_dit(c3_y byt) +{ + // XX parse in the main function + // + u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); + + c3_w dit_w; + + if (byt > '9') { + dit_w = (byt - 'a') + 10; + } + else { + dit_w = (byt - '0'); + } + + return dit_w; +} +static c3_y _to_hex_dit(c3_w byt) +{ + // XX parse in the main function + // + u3_assert(byt < 0xff); + + if (byt < 10) { + return byt + '0'; + } + else { + return (byt - 10) + 'a'; + } +} + +u3_noun u3qe_git_http_read_pkt_lines_on_band( + u3_atom band, + u3_noun sea) { + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + c3_w len_w = PKT_LINE_MAX; + c3_w total_w = 0; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, len_w); + + c3_y* buf_y = sab_u.buf_y; + + c3_w pkt_len_w; + c3_y pkt_band_y; + + while (pos_w < p_octs_w) { + + /* + * Parse pkt-line length + */ + if (pos_w + 4 > p_octs_w) { + c3_free(buf_y); + fprintf(stderr, "u3qe_git__read_pkt_lines_on_band: parsing failure\r\n"); + return u3_none; + } + + pkt_len_w = (_hex_dit(sea_y[0]) << 12) + + (_hex_dit(sea_y[1]) << 8) + + (_hex_dit(sea_y[2]) << 4) + + _hex_dit(sea_y[3]); + // fprintf(stderr, "u3qe_git__read_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); + + if (FLUSH_PKT == pkt_len_w) { + break; + } + + pos_w += 4; + sea_y += 4; + pkt_len_w -= 4; + + u3_assert(pkt_len_w <= PKT_LINE_MAX); + + /* + * Make space for pkt_len_w bytes in the buffer + */ + if (pos_w + pkt_len_w > p_octs_w) { + u3i_slab_free(&sab_u); + u3m_bail(c3__fail); + } + + /* + * Parse band + */ + pkt_band_y = *sea_y; + + sea_y += 1; + pos_w += 1; + pkt_len_w -= 1; + + + if (pkt_band_y == band_w) { + if (total_w + pkt_len_w > len_w) { + + len_w += 4*PKT_LINE_MAX; + u3i_slab_grow(&sab_u, 3, len_w); + buf_y = sab_u.buf_y; + } + + memcpy(buf_y + total_w, sea_y, pkt_len_w); + total_w += pkt_len_w; + } + else { + // buf_y[total_w + pkt_len_w] = 0; + // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); + } + + sea_y += pkt_len_w; + pos_w += pkt_len_w; + } + fprintf(stderr, "Assembled %d bytes\r\n", total_w); + u3_noun octs_red = u3nc(u3i_word(total_w), u3i_slab_mint(&sab_u)); + return u3nc(octs_red, u3k(sea)); +} +u3_noun u3we_git_http_read_pkt_lines_on_band(u3_noun cor) { + + u3_noun sea; + u3_atom band; + + u3x_mean(cor, u3x_sam_2, &sea, + u3x_sam_3, &band, 0); + + return u3qe_git_http_read_pkt_lines_on_band(sea, band); +} + +#define CHUNK_SZ ((1 << 13) - 4 - 1) + +/* XX handle octs with p < met q and p > met q */ +u3_noun u3qe_git_http_write_pkt_lines_on_band( + u3_noun sea, + u3_atom band){ + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + // XX This is broken if p does not match + // the length of q + // + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + // XX handle length properly + const c3_w len_w = (p_octs_w - pos_w); + const c3_w num_lines_w = (len_w / CHUNK_SZ) + ((len_w % CHUNK_SZ) ? 1 : 0); + // 4 bytes for packet length, 1 byte for band + // + const c3_w out_len_w = len_w + num_lines_w * 5; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, out_len_w); + + c3_y* buf_y = sab_u.buf_y; + + c3_w rem_w = len_w; + + while (rem_w) { + const c3_w copy_len_w = (rem_w < CHUNK_SZ) ? rem_w : CHUNK_SZ; + const c3_w line_len_w = copy_len_w + 5; + + // Write line legth + // + buf_y[0] = _to_hex_dit((line_len_w >> 12) & 0xf); + buf_y[1] = _to_hex_dit((line_len_w >> 8) & 0xf); + buf_y[2] = _to_hex_dit((line_len_w >> 4) & 0xf); + buf_y[3] = _to_hex_dit(line_len_w & 0xf); + + // Write band + // + buf_y[4] = 1; + + buf_y += 5; + + memcpy(buf_y, sea_y, copy_len_w); + + buf_y += copy_len_w; + sea_y += copy_len_w; + rem_w -= copy_len_w; + } + + u3_assert(buf_y == sab_u.buf_y + out_len_w); + + return u3nc(u3i_chub(out_len_w), u3i_slab_mint(&sab_u)); +} +u3_noun u3we_git_http_write_pkt_lines_on_band(u3_noun cor) { + + u3_noun sea; + u3_atom band; + + u3x_mean(cor, u3x_sam_2, &band, + u3x_sam_3, &sea, 0); + + return u3qe_git_http_write_pkt_lines_on_band(band, sea); +} diff --git a/pkg/noun/jets/e/stream.c b/pkg/noun/jets/e/stream.c new file mode 100644 index 0000000000..83dcc65c65 --- /dev/null +++ b/pkg/noun/jets/e/stream.c @@ -0,0 +1,242 @@ +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + + +u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea) +{ + u3_atom pos_red; + u3_atom p_octs_red, q_octs_red; + + u3_atom pos_sea; + u3_atom p_octs_sea, q_octs_sea; + + u3x_mean(red, 2, &pos_red, + 6, &p_octs_red, + 7, &q_octs_red, 0); + + u3x_mean(sea, 2, &pos_sea, + 6, &p_octs_sea, + 7, &q_octs_sea, 0); + + if (c3y == u3a_is_cat(n) && n == 0) { + return u3nc(u3k(red), u3k(sea)); + } + c3_w pos_red_w, pos_sea_w; + + if (c3y != u3r_safe_word(pos_red, &pos_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(pos_sea, &pos_sea_w)) { + return u3_none; + } + + c3_w n_w; + + if (c3y != u3r_safe_word(n, &n_w)) { + return u3_none; + } + + c3_y *red_buf_y, *sea_buf_y; + + if (c3y == u3a_is_cat(q_octs_red)) { + red_buf_y = (c3_y*)&q_octs_red; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_red); + red_buf_y = (c3_y*)a->buf_w; + } + + if (c3y == u3a_is_cat(q_octs_sea)) { + sea_buf_y = (c3_y*)&q_octs_sea; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_sea); + sea_buf_y = (c3_y*)a->buf_w; + } + + c3_w p_octs_red_w, p_octs_sea_w; + + if (c3y != u3r_safe_word(p_octs_red, &p_octs_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(p_octs_sea, &p_octs_sea_w)) { + return u3_none; + } + // + // 1. Verify sea has n bytes from its current position + // 2. Get current addresses of red and sea + // 3. Resize red to accomodate n bytes more + // 4. Copy data from red to sea + // 5. Return and update size of red + // + if (pos_sea_w + n_w > p_octs_sea_w) { + fprintf(stderr, "u3qe_stream_append_get_bytes: sea exhausted\r\n"); + return u3_none; + } + + // XX is there a way to enlarge an existing storage? + + // Make space for expanded red + // + c3_i red_len_i = u3r_met(3, q_octs_red); + c3_i new_red_len_i = p_octs_red_w + n_w; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, new_red_len_i); + + c3_y* buf_y = sab_u.buf_y; + + // Copy over existing content in red + // + memcpy(buf_y, red_buf_y, red_len_i); + + if (p_octs_red_w > red_len_i) { + memset(buf_y + red_len_i, 0, p_octs_red_w - red_len_i); + } + + // Copy n_w bytes from sea + // + memcpy(buf_y+p_octs_red_w, sea_buf_y + pos_sea_w, n_w); + + u3_atom new_red = u3nt(u3k(pos_red), + u3i_chub(new_red_len_i), + u3i_slab_mint(&sab_u)); + + return u3nc(new_red, u3k(sea)); +} + +u3_noun u3we_stream_append_get_bytes(u3_noun cor){ + + u3_atom n; + u3_noun red, sea; + + u3x_mean(cor, u3x_sam_2, &n, + u3x_sam_6, &red, + u3x_sam_7, &sea, 0); + + return u3qe_stream_append_get_bytes(n, red, sea); +} + +u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea) +{ + u3_atom pos_red; + u3_atom p_octs_red, q_octs_red; + + u3_atom pos_sea; + u3_noun octs_sea; + u3_atom p_octs_sea, q_octs_sea; + + u3x_mean(red, 2, &pos_red, + 6, &p_octs_red, + 7, &q_octs_red, 0); + + u3x_mean(sea, 2, &pos_sea, 3, &octs_sea, 0); + u3x_mean(octs_sea, 2, &p_octs_sea, 3, &q_octs_sea, 0); + + if (c3y == u3a_is_cat(n) && n == 0) { + return u3nc(u3k(red), u3k(sea)); + } + c3_w pos_red_w, pos_sea_w; + + if (c3y != u3r_safe_word(pos_red, &pos_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(pos_sea, &pos_sea_w)) { + return u3_none; + } + + c3_w n_w; + + if (c3y != u3r_safe_word(n, &n_w)) { + return u3_none; + } + + c3_y *red_buf_y, *sea_buf_y; + + if (c3y == u3a_is_cat(q_octs_red)) { + red_buf_y = (c3_y*)&q_octs_red; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_red); + red_buf_y = (c3_y*)a->buf_w; + } + + if (c3y == u3a_is_cat(q_octs_sea)) { + sea_buf_y = (c3_y*)&q_octs_sea; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_sea); + sea_buf_y = (c3_y*)a->buf_w; + } + + c3_w p_octs_red_w, p_octs_sea_w; + + if (c3y != u3r_safe_word(p_octs_red, &p_octs_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(p_octs_sea, &p_octs_sea_w)) { + return u3_none; + } + // + // 1. Verify sea has n bytes from its current position + // 2. Get current addresses of red and sea + // 3. Resize red to accomodate n bytes more + // 4. Copy data from red to sea + // 5. Return and update size of red + // + if (pos_sea_w + n_w > p_octs_sea_w) { + fprintf(stderr, "u3qe_stream_append_read_bytes: sea exhausted\r\n"); + return u3_none; + } + + // XX is there a way to enlarge an existing storage? + + // Make space for expanded red + // + c3_i red_len_i = u3r_met(3, q_octs_red); + c3_i new_red_len_i = p_octs_red_w + n_w; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, new_red_len_i); + sab_u.buf_w[sab_u.len_w - 1] = 0; + + c3_y* buf_y = sab_u.buf_y; + + // Copy over existing content in red + // + memcpy(buf_y, red_buf_y, red_len_i); + + if (p_octs_red_w > red_len_i) { + memset(buf_y + red_len_i, 0, p_octs_red_w - red_len_i); + } + + // Copy n_w bytes from sea + // + memcpy(buf_y+p_octs_red_w, sea_buf_y + pos_sea_w, n_w); + + u3_atom new_red = u3nt(u3k(pos_red), + u3i_chub(new_red_len_i), + u3i_slab_mint(&sab_u)); + + return u3nc(new_red, u3nc(u3i_word(pos_sea_w + n_w), u3k(octs_sea))); +} + +u3_noun u3we_stream_append_read_bytes(u3_noun cor){ + + u3_atom n; + u3_noun red, sea; + + u3x_mean(cor, u3x_sam_2, &n, + u3x_sam_6, &red, + u3x_sam_7, &sea, 0); + + return u3qe_stream_append_read_bytes(n, red, sea); +} + diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c new file mode 100644 index 0000000000..c82bc7540e --- /dev/null +++ b/pkg/noun/jets/e/zlib.c @@ -0,0 +1,135 @@ +/* +* DEFLATE decompression +*/ + +#include "jets/q.h" +#include "jets/w.h" +#include "manage.h" +#include "retrieve.h" + +#include "noun.h" + +#include +#include + +#define OUTBUF_SZ 4096 + +u3_noun u3qe_zlib_expand(u3_atom pos, u3_atom oft, u3_noun octs) { + + c3_d pos_d; + c3_d oft_d; + c3_d wid_d; + + u3_atom wid; + u3_atom dat; + + u3x_cell(octs, &wid, &dat); + + size_t sad_i = u3r_met(3, dat); + + if (c3n == (u3r_safe_chub(pos, &pos_d))) { + return u3_none; + } + if (c3n == (u3r_safe_chub(oft, &oft_d))) { + return u3_none; + } + if (c3n == (u3r_safe_chub(wid, &wid_d))) { + return u3_none; + } + + c3_y* byt_y; + + if ( c3y == u3a_is_cat(dat)) { + byt_y = (c3_y*)&dat; + } + else { + u3a_atom* vat_u = u3a_to_ptr(dat); + byt_y = (c3_y*)vat_u->buf_w; + } + + z_stream zea; + c3_w zas_w; + + zea.next_in = byt_y + oft_d + pos_d; + zea.avail_in = (wid_d - oft_d - pos_d); + + zea.zalloc = Z_NULL; + zea.zfree = Z_NULL; + zea.opaque = Z_NULL; + + + /* Allocate output buffer + */ + + // Size of the output buffer + // + size_t sob_i = OUTBUF_SZ; + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, sob_i); + c3_y* cuf_y = sab_u.buf_y; + + u3_assert(cuf_y != NULL); + + zea.next_out = cuf_y; + zea.avail_out = sob_i; + + zas_w = inflateInit2(&zea, 47); + + if( Z_OK != zas_w) { + fprintf(stderr, "u3qe_zlib_expand: error while initializing Zlib, zas_w = %d\r\n", zas_w); + + u3i_slab_free(&sab_u); + return u3_none; + } + + while (Z_OK == (zas_w = inflate(&zea, Z_FINISH)) || zas_w == Z_BUF_ERROR) { + + if (zea.avail_in == 0){ + fprintf(stderr, "Exhausted input\r\n"); + break; + } + + if (zea.avail_out == 0) { + u3i_slab_grow(&sab_u, 3, sob_i + OUTBUF_SZ); + cuf_y = sab_u.buf_y + sob_i; + + zea.avail_out = OUTBUF_SZ; + zea.next_out = cuf_y; + u3_assert(sob_i == zea.total_out); + sob_i += OUTBUF_SZ; + } + } + + + size_t len_i = sob_i - zea.avail_out; + pos_d += zea.total_in; + + zas_w = inflateEnd(&zea); + + if (zas_w != Z_OK) { + fprintf(stderr, "u3qe_zlib_expand: zlib stream inconsistent upon finish, zas_w = %d\r\n", + zas_w); + + u3i_slab_free(&sab_u); + return u3_none; + } + + u3_atom len_a = u3i_chub(len_i); + u3_atom buf_a = u3i_slab_mint(&sab_u); + + return u3nc(u3nc(len_a, buf_a), + u3nt(u3i_chub(pos_d), u3k(oft), u3k(octs))); +} + +u3_noun u3we_zlib_expand(u3_noun cor) { + + u3_atom pos; + u3_atom oft; + u3_noun octs; + + u3x_mean(cor, u3x_sam_2, &pos, + u3x_sam_6, &oft, + u3x_sam_7, &octs, 0); + + return u3qe_zlib_expand(pos, oft, octs); +} diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index fd7c5981fb..c0bef1a27b 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -159,6 +159,16 @@ u3_noun u3qe_json_de(u3_atom); u3_atom u3qe_json_en(u3_noun); + u3_noun u3qe_zlib_expand(u3_atom, u3_atom, u3_noun); + + u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); + u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); + + u3_noun u3qe_git_http_read_pkt_lines_on_band(u3_atom band, u3_noun sea); + u3_noun u3qe_git_http_write_pkt_lines_on_band(u3_atom band, u3_noun sea); + + u3_noun u3qe_git_pack_expand_delta_object(u3_noun, u3_noun); + u3_noun u3qeo_raw(u3_atom, u3_atom); u3_noun u3qef_drg(u3_noun, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index e40fe03846..3b148b25ca 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2340,6 +2340,52 @@ static u3j_core _138_hex_blake_d[] = { "blake3-impl", 7, 0, _138_hex_blake3_impl_d, no_hashes }, {} }; +/* + * Zlib + */ +static u3j_harm _138_hex_zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; +static u3j_core _138_hex_zlib_d[] = + { + { "expand", 7, _138_hex_zlib_expand_a, 0, no_hashes }, + {} + }; + +/* + * Stream library + */ +static u3j_harm _138_hex_stream_append_get_bytes_a[] = + {{".2", u3we_stream_append_get_bytes, c3y},{}}; + +static u3j_harm _138_hex_stream_append_read_bytes_a[] = + {{".2", u3we_stream_append_read_bytes, c3y},{}}; + +static u3j_core _138_hex_stream_d[] = + { + { "append-get-bytes", 7, _138_hex_stream_append_get_bytes_a, 0, no_hashes }, + { "append-read-bytes", 7, _138_hex_stream_append_read_bytes_a, 0, no_hashes }, + {} + }; + +/* + * Git HTTP protocol + */ +static u3j_harm _138_hex_git_http_read_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_read_pkt_lines_on_band, c3y}, {}}; +static u3j_harm _138_hex_git_http_write_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_write_pkt_lines_on_band, c3y}, {}}; + +static u3j_core _138_hex_git_http_d[] = + { + { "read-pkt-lines-on-band", 7, _138_hex_git_http_read_pkt_lines_on_band_a, 0, no_hashes}, + { "write-pkt-lines-on-band", 7, _138_hex_git_http_write_pkt_lines_on_band_a, 0, no_hashes}, + {} + }; + +static u3j_harm _138_hex_git_pack_expand_delta_object_a[] = + {{".2", u3we_git_pack_expand_delta_object, c3y}, {}}; + +static u3j_core _138_hex_git_pack_d[] = + {{"expand-delta-object", 7, _138_hex_git_pack_expand_delta_object_a, c3y}, {}}; static u3j_core _138_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, @@ -2359,6 +2405,11 @@ static u3j_core _138_hex_d[] = { "secp", 6, 0, _140_hex_secp_d, no_hashes }, { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, + /* Git jets */ + { "zlib", 31, 0, _138_hex_zlib_d, no_hashes}, + { "stream", 31, 0, _138_hex_stream_d, no_hashes}, + { "git-pack", 31, 0, _138_hex_git_pack_d, no_hashes}, + { "git-http", 31, 0, _138_hex_git_http_d, no_hashes}, {} }; diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 34716971bd..ae58b224e3 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -226,6 +226,16 @@ u3_noun u3we_json_de(u3_noun); u3_atom u3we_json_en(u3_noun); + u3_noun u3we_zlib_expand(u3_noun); + + u3_noun u3we_stream_append_get_bytes(u3_noun); + u3_noun u3we_stream_append_read_bytes(u3_noun); + + u3_noun u3we_git_http_read_pkt_lines_on_band(u3_noun); + u3_noun u3we_git_http_write_pkt_lines_on_band(u3_noun); + + u3_noun u3we_git_pack_expand_delta_object(u3_noun); + u3_noun u3we_bend_fun(u3_noun); u3_noun u3we_cold_fun(u3_noun); u3_noun u3we_cook_fun(u3_noun);