From ab36df6ef5af059bfcd1c320990db42eb6319794 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Mon, 8 Mar 2021 11:49:23 +0100 Subject: [PATCH 01/15] registry: Create an abstraction for registries. --- Makefile | 2 +- deps/wiki-registry/github-registry.c | 131 ++++++++++++ deps/wiki-registry/github-registry.h | 8 + deps/wiki-registry/gitlab-registry.c | 12 ++ deps/wiki-registry/gitlab-registry.h | 8 + deps/wiki-registry/wiki-registry-internal.h | 17 ++ deps/wiki-registry/wiki-registry.c | 219 ++++++++------------ deps/wiki-registry/wiki-registry.h | 51 +++-- src/clib-search.c | 68 +++--- 9 files changed, 329 insertions(+), 187 deletions(-) create mode 100644 deps/wiki-registry/github-registry.c create mode 100644 deps/wiki-registry/github-registry.h create mode 100644 deps/wiki-registry/gitlab-registry.c create mode 100644 deps/wiki-registry/gitlab-registry.h create mode 100644 deps/wiki-registry/wiki-registry-internal.h diff --git a/Makefile b/Makefile index 35d5528c..1d888661 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ MAKEFILES = $(wildcard deps/*/Makefile) export CC -CFLAGS += -std=c99 -Ideps -Wall -Wno-unused-function -U__STRICT_ANSI__ +CFLAGS += -std=c99 -Ideps -Wall -Werror=return--type -Wno-unused-function -U__STRICT_ANSI__ ifdef STATIC CFLAGS += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags) diff --git a/deps/wiki-registry/github-registry.c b/deps/wiki-registry/github-registry.c new file mode 100644 index 00000000..9d9286b4 --- /dev/null +++ b/deps/wiki-registry/github-registry.c @@ -0,0 +1,131 @@ +// +// github-registry.c +// +// Copyright (c) 2020 Elbert van de Put +// Based on work by Stephen Mathieson +// MIT licensed +// +#include +#include "github-registry.h" +#include "gumbo-text-content/gumbo-text-content.h" +#include "gumbo-get-element-by-id/get-element-by-id.h" +#include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" +#include "http-get/http-get.h" +#include +#include "substr/substr.h" +#include "strdup/strdup.h" +#include "case/case.h" +#include "trim/trim.h" +#include "wiki-registry-internal.h" + +/** + * Add `href` to the given `package`. + */ +static void add_package_href(wiki_package_ptr_t self) { + size_t len = strlen(self->repo) + 20; // https://github.com/ \0 + self->href = malloc(len); + if (self->href) + sprintf(self->href, "https://github.com/%s", self->repo); +} + +/** + * Parse the given wiki `li` into a package. + */ +static wiki_package_ptr_t parse_li(GumboNode *li) { + wiki_package_ptr_t self = wiki_package_new(); + char *text = NULL; + + if (!self) goto cleanup; + + text = gumbo_text_content(li); + if (!text) goto cleanup; + + // TODO support unicode dashes + char *tok = strstr(text, " - "); + if (!tok) goto cleanup; + + int pos = tok - text; + self->repo = substr(text, 0, pos); + self->description = substr(text, pos + 3, -1); + if (!self->repo || !self->description) goto cleanup; + trim(self->description); + trim(self->repo); + + add_package_href(self); + + cleanup: + free(text); + return self; +} + +/** + * Parse a list of packages from the given `html` + */ +list_t *wiki_registry_parse(const char *html) { + GumboOutput *output = gumbo_parse(html); + list_t *pkgs = list_new(); + + GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); + if (body) { + // grab all category `

`s + list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); + list_node_t *heading_node; + list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); + while ((heading_node = list_iterator_next(heading_iterator))) { + GumboNode *heading = (GumboNode *) heading_node->val; + char *category = gumbo_text_content(heading); + // die if we failed to parse a category, as it's + // almost certinaly a malloc error + if (!category) break; + trim(case_lower(category)); + GumboVector *siblings = &heading->parent->v.element.children; + size_t pos = heading->index_within_parent; + + // skip elements until the UL + // TODO: don't hardcode position here + // 2: + // 1 - whitespace + // 2 - actual node + GumboNode *ul = siblings->data[pos + 2]; + if (GUMBO_TAG_UL != ul->v.element.tag) { + free(category); + continue; + } + + list_t *lis = gumbo_get_elements_by_tag_name("li", ul); + list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); + list_node_t *li_node; + while ((li_node = list_iterator_next(li_iterator))) { + wiki_package_ptr_t package = parse_li(li_node->val); + if (package && package->description) { + package->category = strdup(category); + list_rpush(pkgs, list_node_new(package)); + } else { + // failed to parse package + if (package) wiki_package_free(package); + } + } + list_iterator_destroy(li_iterator); + list_destroy(lis); + free(category); + } + list_iterator_destroy(heading_iterator); + list_destroy(h2s); + } + + gumbo_destroy_output(&kGumboDefaultOptions, output); + return pkgs; +} + +/** + * Get a list of packages from the given GitHub wiki `url`. + */ +list_t *github_registry_fetch(const char *url) { + http_get_response_t *res = http_get(url); + if (!res->ok) return NULL; + + list_t *list = wiki_registry_parse(res->data); + http_get_free(res); + return list; +} + diff --git a/deps/wiki-registry/github-registry.h b/deps/wiki-registry/github-registry.h new file mode 100644 index 00000000..82d9ddce --- /dev/null +++ b/deps/wiki-registry/github-registry.h @@ -0,0 +1,8 @@ +#ifndef CLIB_GITHUB_REGISTRY_H +#define CLIB_GITHUB_REGISTRY_H + +#include "list/list.h" + +list_t* github_registry_fetch(const char *url); + +#endif //CLIB_GITHUB_REGISTRY_H diff --git a/deps/wiki-registry/gitlab-registry.c b/deps/wiki-registry/gitlab-registry.c new file mode 100644 index 00000000..1a6b10bd --- /dev/null +++ b/deps/wiki-registry/gitlab-registry.c @@ -0,0 +1,12 @@ +// +// gitlab-registry.c +// +// Copyright (c) 2020 Elbert van de Put +// MIT licensed +// +#include "gitlab-registry.h" + +list_t* gitlab_registry_fetch(const char *url) { + return NULL; +} + diff --git a/deps/wiki-registry/gitlab-registry.h b/deps/wiki-registry/gitlab-registry.h new file mode 100644 index 00000000..4a3198ab --- /dev/null +++ b/deps/wiki-registry/gitlab-registry.h @@ -0,0 +1,8 @@ +#ifndef CLIB_GITLAB_REGISTRY_H +#define CLIB_GITLAB_REGISTRY_H + +#include "list/list.h" + +list_t* gitlab_registry_fetch(const char *url); + +#endif //CLIB_GITLAB_REGISTRY_H diff --git a/deps/wiki-registry/wiki-registry-internal.h b/deps/wiki-registry/wiki-registry-internal.h new file mode 100644 index 00000000..62f94f10 --- /dev/null +++ b/deps/wiki-registry/wiki-registry-internal.h @@ -0,0 +1,17 @@ +#ifndef WIKI_REGISTRY_HELPER_H +#define WIKI_REGISTRY_HELPER_H +// DO NOT INCLUDE. THIS HEADER IS INTERNAL ONLY +#include "wiki-registry.h" + +struct wiki_package_t { + char *repo; + char *href; + char *description; + char *category; +}; + +wiki_package_ptr_t wiki_package_new(); + +void wiki_package_free(wiki_package_ptr_t pkg); + +#endif //WIKI_REGISTRY_HELPER_H diff --git a/deps/wiki-registry/wiki-registry.c b/deps/wiki-registry/wiki-registry.c index e06e8c5a..b2d35dba 100644 --- a/deps/wiki-registry/wiki-registry.c +++ b/deps/wiki-registry/wiki-registry.c @@ -6,169 +6,120 @@ // MIT licensed // -#include #include #include #include "gumbo-parser/gumbo.h" -#include "gumbo-text-content/gumbo-text-content.h" -#include "gumbo-get-element-by-id/get-element-by-id.h" -#include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" -#include "http-get/http-get.h" #include "list/list.h" -#include "substr/substr.h" -#include "strdup/strdup.h" -#include "case/case.h" -#include "trim/trim.h" -#include "wiki-registry.h" +#include "wiki-registry-internal.h" +#include "github-registry.h" +#include "gitlab-registry.h" -// -// TODO find dox on gumbo so the node iteration isn't so ugly -// +enum wiki_registry_type_t { + REGISTRY_TYPE_GITHUB, + REGISTRY_TYPE_GITLAB, +}; + +struct wiki_registry_t { + enum wiki_registry_type_t type; + char *url; + list_t *packages; +}; /** * Create a new wiki package. */ - -static wiki_package_t * -wiki_package_new() { - wiki_package_t *pkg = malloc(sizeof(wiki_package_t)); - if (pkg) { - pkg->repo = NULL; - pkg->href = NULL; - pkg->description = NULL; - pkg->category = NULL; - } - return pkg; +wiki_package_ptr_t wiki_package_new() { + wiki_package_ptr_t pkg = malloc(sizeof(struct wiki_package_t)); + if (pkg) { + pkg->repo = NULL; + pkg->href = NULL; + pkg->description = NULL; + pkg->category = NULL; + } + return pkg; } /** - * Add `href` to the given `package`. + * Free a wiki_package_t. */ - -static void -add_package_href(wiki_package_t *self) { - size_t len = strlen(self->repo) + 20; // https://github.com/ \0 - self->href = malloc(len); - if (self->href) - sprintf(self->href, "https://github.com/%s", self->repo); +void wiki_package_free(wiki_package_ptr_t pkg) { + free(pkg->repo); + free(pkg->href); + free(pkg->description); + free(pkg->category); + free(pkg); } -/** - * Parse the given wiki `li` into a package. - */ - -static wiki_package_t * -parse_li(GumboNode *li) { - wiki_package_t *self = wiki_package_new(); - char *text = NULL; +wiki_registry_ptr_t wiki_registry_create(const char *url) { + wiki_registry_ptr_t registry = malloc(sizeof(struct wiki_registry_t)); + registry->url = malloc(strlen(url)); + strcpy(registry->url, url); + + if (strstr(url, "github.com") != NULL) { + registry->type = REGISTRY_TYPE_GITHUB; + } else if (strstr(url, "gitlab") != NULL) { + registry->type = REGISTRY_TYPE_GITLAB; + } else { + return NULL; + } - if (!self) goto cleanup; + return registry; +} - text = gumbo_text_content(li); - if (!text) goto cleanup; +void wiki_registry_free(wiki_registry_ptr_t registry) { + free(registry->url); + if (registry->packages != NULL) { + list_destroy(registry->packages); + } + free(registry); +} - // TODO support unicode dashes - char *tok = strstr(text, " - "); - if (!tok) goto cleanup; +// TODO, return false if the request fails. +bool wiki_registry_fetch(wiki_registry_ptr_t registry) { + switch (registry->type) { + case REGISTRY_TYPE_GITLAB: + registry->packages = gitlab_registry_fetch(registry->url); + break; + case REGISTRY_TYPE_GITHUB: + registry->packages = github_registry_fetch(registry->url); + break; + default: + return false; + } - int pos = tok - text; - self->repo = substr(text, 0, pos); - self->description = substr(text, pos + 3, -1); - if (!self->repo || !self->description) goto cleanup; - trim(self->description); - trim(self->repo); + return false; +} - add_package_href(self); -cleanup: - free(text); - return self; +wiki_registry_iterator_t wiki_registry_iterator_new(wiki_registry_ptr_t registry) { + return list_iterator_new(registry->packages, LIST_HEAD); } -/** - * Parse a list of packages from the given `html` - */ - -list_t * -wiki_registry_parse(const char *html) { - GumboOutput *output = gumbo_parse(html); - list_t *pkgs = list_new(); - - GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); - if (body) { - // grab all category `

`s - list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); - list_node_t *heading_node; - list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); - while ((heading_node = list_iterator_next(heading_iterator))) { - GumboNode *heading = (GumboNode *) heading_node->val; - char *category = gumbo_text_content(heading); - // die if we failed to parse a category, as it's - // almost certinaly a malloc error - if (!category) break; - trim(case_lower(category)); - GumboVector *siblings = &heading->parent->v.element.children; - size_t pos = heading->index_within_parent; - - // skip elements until the UL - // TODO: don't hardcode position here - // 2: - // 1 - whitespace - // 2 - actual node - GumboNode *ul = siblings->data[pos + 2]; - if (GUMBO_TAG_UL != ul->v.element.tag) { - free(category); - continue; - } - - list_t *lis = gumbo_get_elements_by_tag_name("li", ul); - list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); - list_node_t *li_node; - while ((li_node = list_iterator_next(li_iterator))) { - wiki_package_t *package = parse_li(li_node->val); - if (package && package->description) { - package->category = strdup(category); - list_rpush(pkgs, list_node_new(package)); - } else { - // failed to parse package - if (package) wiki_package_free(package); - } - } - list_iterator_destroy(li_iterator); - list_destroy(lis); - free(category); +wiki_package_ptr_t wiki_registry_iterator_next(wiki_registry_iterator_t iterator) { + list_node_t *node = list_iterator_next(iterator); + if (node == NULL) { + return NULL; } - list_iterator_destroy(heading_iterator); - list_destroy(h2s); - } - gumbo_destroy_output(&kGumboDefaultOptions, output); - return pkgs; + return (wiki_package_ptr_t )node->val; } -/** - * Get a list of packages from the given GitHub wiki `url`. - */ - -list_t * -wiki_registry(const char *url) { - http_get_response_t *res = http_get(url); - if (!res->ok) return NULL; +void wiki_registry_iterator_destroy(wiki_registry_iterator_t iterator) { + list_iterator_destroy(iterator); +} - list_t *list = wiki_registry_parse(res->data); - http_get_free(res); - return list; +char* wiki_package_get_repo(wiki_package_ptr_t package) { + return package->repo; } -/** - * Free a wiki_package_t. - */ +char* wiki_package_get_href(wiki_package_ptr_t package) { + return package->href; +} -void -wiki_package_free(wiki_package_t *pkg) { - free(pkg->repo); - free(pkg->href); - free(pkg->description); - free(pkg->category); - free(pkg); +char* wiki_package_get_description(wiki_package_ptr_t package) { + return package->description; } + +char* wiki_package_get_category(wiki_package_ptr_t package) { + return package->category; +} \ No newline at end of file diff --git a/deps/wiki-registry/wiki-registry.h b/deps/wiki-registry/wiki-registry.h index 21c219cd..677ead26 100644 --- a/deps/wiki-registry/wiki-registry.h +++ b/deps/wiki-registry/wiki-registry.h @@ -10,22 +10,39 @@ #ifndef WIKI_REGISTRY_H #define WIKI_REGISTRY_H 1 -#include "list/list.h" - -typedef struct { - char *repo; - char *href; - char *description; - char *category; -} wiki_package_t; - -list_t * -wiki_registry(const char *); - -list_t * -wiki_registry_parse(const char *); - -void -wiki_package_free(wiki_package_t *); +typedef struct wiki_package_t* wiki_package_ptr_t; +typedef struct wiki_registry_t* wiki_registry_ptr_t; +typedef list_iterator_t* wiki_registry_iterator_t; + +/** + * Create a new wiki_registry for the given url. + * @param url + * @return + */ +wiki_registry_ptr_t wiki_registry_create(const char* url); + +/** + * Free the memory held by the registry. + * @param registry + */ +void wiki_registry_free(wiki_registry_ptr_t registry); + +/** + * Fetch the list of packages from the registry. + * @param registry + */ +bool wiki_registry_fetch(wiki_registry_ptr_t registry); + +/** + * An iterator through the packages in the registry. + */ +wiki_registry_iterator_t wiki_registry_iterator_new(wiki_registry_ptr_t registry); +wiki_package_ptr_t wiki_registry_iterator_next(wiki_registry_iterator_t iterator); +void wiki_registry_iterator_destroy(wiki_registry_iterator_t iterator); + +char* wiki_package_get_repo(wiki_package_ptr_t package); +char* wiki_package_get_href(wiki_package_ptr_t package); +char* wiki_package_get_description(wiki_package_ptr_t package); +char* wiki_package_get_category(wiki_package_ptr_t package); #endif diff --git a/src/clib-search.c b/src/clib-search.c index 6e0edf30..7e243f93 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -61,7 +61,7 @@ static void setopt_json(command_t *self) { opt_json = 1; } } \ } -static int matches(int count, char *args[], wiki_package_t *pkg) { +static int matches(int count, char *args[], wiki_package_ptr_t pkg) { // Display all packages if there's no query if (0 == count) return 1; @@ -72,16 +72,16 @@ static int matches(int count, char *args[], wiki_package_t *pkg) { char *href = NULL; int rc = 0; - name = clib_package_parse_name(pkg->repo); + name = clib_package_parse_name(wiki_package_get_repo(pkg)); COMPARE(name); - description = strdup(pkg->description); + description = strdup(wiki_package_get_description(pkg)); COMPARE(description); - repo = strdup(pkg->repo); + repo = strdup(wiki_package_get_repo(pkg)); COMPARE(repo); - href = strdup(pkg->href); + href = strdup(wiki_package_get_href(pkg)); COMPARE(href); cleanup: @@ -100,12 +100,8 @@ static char *wiki_html_cache() { if (data) { return data; } - - goto set_cache; } -set_cache: - debug(&debugger, "setting cache from %s", CLIB_WIKI_URL); http_get_response_t *res = http_get(CLIB_WIKI_URL); if (!res->ok) @@ -119,30 +115,30 @@ static char *wiki_html_cache() { if (NULL == html) return html; clib_cache_save_search(html); - debug(&debugger, "wrote cach"); + debug(&debugger, "wrote cache"); return html; } -static void display_package(const wiki_package_t *pkg, +static void display_package(const wiki_package_ptr_t pkg, cc_color_t fg_color_highlight, cc_color_t fg_color_text) { - cc_fprintf(fg_color_highlight, stdout, " %s\n", pkg->repo); + cc_fprintf(fg_color_highlight, stdout, " %s\n", wiki_package_get_repo(pkg)); printf(" url: "); - cc_fprintf(fg_color_text, stdout, "%s\n", pkg->href); + cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_href(pkg)); printf(" desc: "); - cc_fprintf(fg_color_text, stdout, "%s\n", pkg->description); + cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_description(pkg)); printf("\n"); } -static void add_package_to_json(const wiki_package_t *pkg, +static void add_package_to_json(const wiki_package_ptr_t pkg, JSON_Array *json_list) { JSON_Value *json_pkg_root = json_value_init_object(); JSON_Object *json_pkg = json_value_get_object(json_pkg_root); - json_object_set_string(json_pkg, "repo", pkg->repo); - json_object_set_string(json_pkg, "href", pkg->href); - json_object_set_string(json_pkg, "description", pkg->description); - json_object_set_string(json_pkg, "category", pkg->category); + json_object_set_string(json_pkg, "repo", wiki_package_get_repo(pkg)); + json_object_set_string(json_pkg, "href", wiki_package_get_href(pkg)); + json_object_set_string(json_pkg, "description", wiki_package_get_description(pkg)); + json_object_set_string(json_pkg, "category", wiki_package_get_category(pkg)); json_array_append_value(json_list, json_pkg_root); } @@ -177,20 +173,25 @@ int main(int argc, char *argv[]) { cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; - char *html = wiki_html_cache(); - if (NULL == html) { - command_free(&program); - logger_error("error", "failed to fetch wiki HTML"); - return 1; - } + wiki_registry_ptr_t registry = wiki_registry_create(CLIB_WIKI_URL); + wiki_registry_fetch(registry); + + // TODO, implement caching for the new registries. + /* + char *html = wiki_html_cache(); + if (NULL == html) { + command_free(&program); + logger_error("error", "failed to fetch wiki HTML"); + return 1; + } list_t *pkgs = wiki_registry_parse(html); free(html); - debug(&debugger, "found %zu packages", pkgs->len); + */ - list_node_t *node; - list_iterator_t *it = list_iterator_new(pkgs, LIST_HEAD); + wiki_package_ptr_t pkg; + wiki_registry_iterator_t it = wiki_registry_iterator_new(registry); JSON_Array *json_list = NULL; JSON_Value *json_list_root = NULL; @@ -202,8 +203,7 @@ int main(int argc, char *argv[]) { printf("\n"); - while ((node = list_iterator_next(it))) { - wiki_package_t *pkg = (wiki_package_t *)node->val; + while ((pkg = wiki_registry_iterator_next(it))) { if (matches(program.argc, program.argv, pkg)) { if (opt_json) { add_package_to_json(pkg, json_list); @@ -211,10 +211,8 @@ int main(int argc, char *argv[]) { display_package(pkg, fg_color_highlight, fg_color_text); } } else { - debug(&debugger, "skipped package %s", pkg->repo); + debug(&debugger, "skipped package %s", wiki_package_get_repo(pkg)); } - - wiki_package_free(pkg); } if (opt_json) { @@ -225,8 +223,8 @@ int main(int argc, char *argv[]) { json_value_free(json_list_root); } - list_iterator_destroy(it); - list_destroy(pkgs); + wiki_registry_iterator_destroy(it); + wiki_registry_free(registry); command_free(&program); return 0; } From 205676f35b5c2c72ba5b691203400ee71034b6fd Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Mon, 8 Mar 2021 15:53:12 +0100 Subject: [PATCH 02/15] registry: Start with a gitlab registry. --- Makefile | 4 +- clib.json | 11 +- deps/wiki-registry/gitlab-registry.c | 12 -- deps/wiki-registry/package.json | 24 ---- src/clib-search.c | 2 +- .../registry}/github-registry.c | 0 .../registry}/github-registry.h | 0 src/registry/gitlab-registry.c | 131 ++++++++++++++++++ .../registry}/gitlab-registry.h | 2 +- .../registry}/wiki-registry-internal.h | 0 .../registry}/wiki-registry.c | 21 ++- .../registry}/wiki-registry.h | 0 12 files changed, 161 insertions(+), 46 deletions(-) delete mode 100644 deps/wiki-registry/gitlab-registry.c delete mode 100644 deps/wiki-registry/package.json rename {deps/wiki-registry => src/registry}/github-registry.c (100%) rename {deps/wiki-registry => src/registry}/github-registry.h (100%) create mode 100644 src/registry/gitlab-registry.c rename {deps/wiki-registry => src/registry}/gitlab-registry.h (63%) rename {deps/wiki-registry => src/registry}/wiki-registry-internal.h (100%) rename {deps/wiki-registry => src/registry}/wiki-registry.c (84%) rename {deps/wiki-registry => src/registry}/wiki-registry.h (100%) diff --git a/Makefile b/Makefile index 1d888661..62c1e1b8 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ RM = rm -f MKDIR = mkdir -p SRC = $(wildcard src/*.c) -COMMON_SRC = $(wildcard src/common/*.c) -ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h test/package/*.c test/cache/*.c) +COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c) +ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h test/package/*.c test/cache/*.c) SDEPS = $(wildcard deps/*/*.c) ODEPS = $(SDEPS:.c=.o) DEPS = $(filter-out $(ODEPS), $(SDEPS)) diff --git a/clib.json b/clib.json index f25c6722..8c6954c6 100644 --- a/clib.json +++ b/clib.json @@ -37,9 +37,16 @@ "stephenmathieson/debug.c": "0.0.0", "stephenmathieson/tempdir.c": "0.0.2", "isty001/copy": "0.0.0", - "stephenmathieson/rimraf.c": "0.1.0" + "stephenmathieson/rimraf.c": "0.1.0", + "thlorenz/gumbo-parser.c": "*", + "stephenmathieson/http-get.c": "*", + "stephenmathieson/gumbo-text-content.c": "*", + "stephenmathieson/gumbo-get-element-by-id.c": "*", + "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", + "clibs/list": "*", + "jwerle/url.h": "0.0.*" }, "development": { "stephenmathieson/describe.h": "2.0.1" } -} +} \ No newline at end of file diff --git a/deps/wiki-registry/gitlab-registry.c b/deps/wiki-registry/gitlab-registry.c deleted file mode 100644 index 1a6b10bd..00000000 --- a/deps/wiki-registry/gitlab-registry.c +++ /dev/null @@ -1,12 +0,0 @@ -// -// gitlab-registry.c -// -// Copyright (c) 2020 Elbert van de Put -// MIT licensed -// -#include "gitlab-registry.h" - -list_t* gitlab_registry_fetch(const char *url) { - return NULL; -} - diff --git a/deps/wiki-registry/package.json b/deps/wiki-registry/package.json deleted file mode 100644 index 6ab5362e..00000000 --- a/deps/wiki-registry/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "wiki-registry", - "version": "0.0.4", - "repo": "stephenmathieson/wiki-registry.c", - "description": "Turn a GitHub Wiki page into a package registry", - "keywords": [ "registry", "github", "wiki" ], - "license": "MIT", - "src": [ - "src/wiki-registry.c", - "src/wiki-registry.h" - ], - "dependencies": { - "thlorenz/gumbo-parser.c": "*", - "stephenmathieson/substr.c": "*", - "strdup": "0.0.0", - "stephenmathieson/http-get.c": "*", - "stephenmathieson/case.c": "*", - "stephenmathieson/trim.c": "*", - "stephenmathieson/gumbo-text-content.c": "*", - "stephenmathieson/gumbo-get-element-by-id.c": "*", - "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", - "clibs/list": "*" - } -} diff --git a/src/clib-search.c b/src/clib-search.c index 7e243f93..926e065a 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -20,7 +20,7 @@ #include "strdup/strdup.h" #include "tempdir/tempdir.h" #include "version.h" -#include "wiki-registry/wiki-registry.h" +#include "registry/wiki-registry.h" #include #include #include diff --git a/deps/wiki-registry/github-registry.c b/src/registry/github-registry.c similarity index 100% rename from deps/wiki-registry/github-registry.c rename to src/registry/github-registry.c diff --git a/deps/wiki-registry/github-registry.h b/src/registry/github-registry.h similarity index 100% rename from deps/wiki-registry/github-registry.h rename to src/registry/github-registry.h diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c new file mode 100644 index 00000000..a3c0b82c --- /dev/null +++ b/src/registry/gitlab-registry.c @@ -0,0 +1,131 @@ +// +// gitlab-registry.c +// +// Copyright (c) 2020 Elbert van de Put +// MIT licensed +// +#include "gitlab-registry.h" +#include +#include "gumbo-text-content/gumbo-text-content.h" +#include "gumbo-get-element-by-id/get-element-by-id.h" +#include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" +#include "http-get/http-get.h" +#include +#include "substr/substr.h" +#include "strdup/strdup.h" +#include "case/case.h" +#include "trim/trim.h" +#include "wiki-registry-internal.h" + +/** + * Add `href` to the given `package`. + * We assume that all packages listed by a registry live on the same platform as the registry. + */ +static void add_package_href(wiki_package_ptr_t self, const char* hostname) { + size_t len = strlen(self->repo) + strlen(hostname); + self->href = malloc(len); + if (self->href) + sprintf(self->href, "https://%s/%s", hostname, self->repo); +} + +/** + * Parse the given wiki `li` into a package. + */ +static wiki_package_ptr_t parse_li(GumboNode *li, const char* hostname) { + wiki_package_ptr_t self = wiki_package_new(); + char *text = NULL; + + if (!self) goto cleanup; + + text = gumbo_text_content(li); + if (!text) goto cleanup; + + // TODO support unicode dashes + char *tok = strstr(text, " - "); + if (!tok) goto cleanup; + + int pos = tok - text; + self->repo = substr(text, 0, pos); + self->description = substr(text, pos + 3, -1); + if (!self->repo || !self->description) goto cleanup; + trim(self->description); + trim(self->repo); + + add_package_href(self, hostname); + + cleanup: + free(text); + return self; +} + +/** + * Parse a list of packages from the given `html` + */ +static list_t *wiki_registry_parse(const char* hostname, const char *html) { + GumboOutput *output = gumbo_parse(html); + list_t *pkgs = list_new(); + + GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); + if (body) { + // grab all category `

`s + list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); + list_node_t *heading_node; + list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); + while ((heading_node = list_iterator_next(heading_iterator))) { + GumboNode *heading = (GumboNode *) heading_node->val; + char *category = gumbo_text_content(heading); + // die if we failed to parse a category, as it's + // almost certinaly a malloc error + if (!category) break; + trim(case_lower(category)); + GumboVector *siblings = &heading->parent->v.element.children; + size_t pos = heading->index_within_parent; + + // skip elements until the UL + // TODO: don't hardcode position here + // 2: + // 1 - whitespace + // 2 - actual node + GumboNode *ul = siblings->data[pos + 2]; + if (GUMBO_TAG_UL != ul->v.element.tag) { + free(category); + continue; + } + + list_t *lis = gumbo_get_elements_by_tag_name("li", ul); + list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); + list_node_t *li_node; + while ((li_node = list_iterator_next(li_iterator))) { + wiki_package_ptr_t package = parse_li(li_node->val, hostname); + if (package && package->description) { + package->category = strdup(category); + list_rpush(pkgs, list_node_new(package)); + } else { + // failed to parse package + if (package) wiki_package_free(package); + } + } + list_iterator_destroy(li_iterator); + list_destroy(lis); + free(category); + } + list_iterator_destroy(heading_iterator); + list_destroy(h2s); + } + + gumbo_destroy_output(&kGumboDefaultOptions, output); + return pkgs; +} + +/** + * Get a list of packages from the given gitlab wiki `url`. + */ +list_t *gitlab_registry_fetch(const char *url, const char* hostname) { + http_get_response_t *res = http_get(url); + if (!res->ok) return NULL; + + list_t *list = wiki_registry_parse(url, res->data); + http_get_free(res); + return list; +} + diff --git a/deps/wiki-registry/gitlab-registry.h b/src/registry/gitlab-registry.h similarity index 63% rename from deps/wiki-registry/gitlab-registry.h rename to src/registry/gitlab-registry.h index 4a3198ab..38d7de00 100644 --- a/deps/wiki-registry/gitlab-registry.h +++ b/src/registry/gitlab-registry.h @@ -3,6 +3,6 @@ #include "list/list.h" -list_t* gitlab_registry_fetch(const char *url); +list_t* gitlab_registry_fetch(const char* url, const char* hostname); #endif //CLIB_GITLAB_REGISTRY_H diff --git a/deps/wiki-registry/wiki-registry-internal.h b/src/registry/wiki-registry-internal.h similarity index 100% rename from deps/wiki-registry/wiki-registry-internal.h rename to src/registry/wiki-registry-internal.h diff --git a/deps/wiki-registry/wiki-registry.c b/src/registry/wiki-registry.c similarity index 84% rename from deps/wiki-registry/wiki-registry.c rename to src/registry/wiki-registry.c index b2d35dba..a1eedb0e 100644 --- a/deps/wiki-registry/wiki-registry.c +++ b/src/registry/wiki-registry.c @@ -8,11 +8,13 @@ #include #include +#include "strdup/strdup.h" #include "gumbo-parser/gumbo.h" #include "list/list.h" #include "wiki-registry-internal.h" #include "github-registry.h" #include "gitlab-registry.h" +#include "url/url.h" enum wiki_registry_type_t { REGISTRY_TYPE_GITHUB, @@ -21,7 +23,8 @@ enum wiki_registry_type_t { struct wiki_registry_t { enum wiki_registry_type_t type; - char *url; + char* url; + char* hostname; list_t *packages; }; @@ -52,8 +55,7 @@ void wiki_package_free(wiki_package_ptr_t pkg) { wiki_registry_ptr_t wiki_registry_create(const char *url) { wiki_registry_ptr_t registry = malloc(sizeof(struct wiki_registry_t)); - registry->url = malloc(strlen(url)); - strcpy(registry->url, url); + registry->url = strdup(url); if (strstr(url, "github.com") != NULL) { registry->type = REGISTRY_TYPE_GITHUB; @@ -63,12 +65,23 @@ wiki_registry_ptr_t wiki_registry_create(const char *url) { return NULL; } + url_data_t *parsed = url_parse(url); + registry->hostname = strdup(parsed->hostname); + url_free(parsed); + return registry; } void wiki_registry_free(wiki_registry_ptr_t registry) { free(registry->url); + free(registry->hostname); if (registry->packages != NULL) { + list_iterator_t* it = list_iterator_new(registry->packages, LIST_HEAD); + list_node_t* node; + while ((node = list_iterator_next(it))) { + wiki_package_free(node->val); + } + list_iterator_destroy(it); list_destroy(registry->packages); } free(registry); @@ -78,7 +91,7 @@ void wiki_registry_free(wiki_registry_ptr_t registry) { bool wiki_registry_fetch(wiki_registry_ptr_t registry) { switch (registry->type) { case REGISTRY_TYPE_GITLAB: - registry->packages = gitlab_registry_fetch(registry->url); + registry->packages = gitlab_registry_fetch(registry->url, registry->hostname); break; case REGISTRY_TYPE_GITHUB: registry->packages = github_registry_fetch(registry->url); diff --git a/deps/wiki-registry/wiki-registry.h b/src/registry/wiki-registry.h similarity index 100% rename from deps/wiki-registry/wiki-registry.h rename to src/registry/wiki-registry.h From 91430a46aab02b249b119bd4f5be96611ce33371 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Wed, 10 Mar 2021 23:42:36 +0100 Subject: [PATCH 03/15] registry: Store extra registries in clib.json and read these registries in clib-search --- Makefile | 5 +- deps/http-get/http-get.c | 15 +- deps/http-get/http-get.h | 4 +- deps/url/package.json | 9 + deps/url/url.h | 594 +++++++++++++++++++++++++++++++++ src/clib-search.c | 279 +++++++++------- src/common/clib-package.c | 20 +- src/common/clib-package.h | 1 + src/common/clib-release-info.c | 2 +- src/registry/github-registry.c | 2 +- src/registry/gitlab-registry.c | 44 ++- src/registry/wiki-registry.c | 11 +- src/registry/wiki-registry.h | 7 + 13 files changed, 849 insertions(+), 144 deletions(-) create mode 100644 deps/url/package.json create mode 100644 deps/url/url.h diff --git a/Makefile b/Makefile index 62c1e1b8..7150695a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -CC ?= cc +CC ?= gcc PREFIX ?= /usr/local BINS = clib clib-install clib-search clib-init clib-configure clib-build clib-update clib-upgrade clib-uninstall @@ -18,7 +18,6 @@ SDEPS = $(wildcard deps/*/*.c) ODEPS = $(SDEPS:.c=.o) DEPS = $(filter-out $(ODEPS), $(SDEPS)) OBJS = $(DEPS:.c=.o) -MAKEFILES = $(wildcard deps/*/Makefile) export CC @@ -48,7 +47,7 @@ all: $(BINS) build: $(BINS) -$(BINS): $(SRC) $(MAKEFILES) $(OBJS) +$(BINS): $(SRC) $(COMMON_SRC) $(OBJS) $(CC) $(CFLAGS) -o $@ $(COMMON_SRC) src/$(@:.exe=).c $(OBJS) $(LDFLAGS) $(MAKEFILES): diff --git a/deps/http-get/http-get.c b/deps/http-get/http-get.c index 0aeaaffa..d11c32ad 100644 --- a/deps/http-get/http-get.c +++ b/deps/http-get/http-get.c @@ -40,7 +40,7 @@ static size_t http_get_cb(void *contents, size_t size, size_t nmemb, void *userp return realsize; } -http_get_response_t *http_get_shared(const char *url, CURLSH *share) { +http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char** headers, int header_count) { CURL *req = curl_easy_init(); http_get_response_t *res = malloc(sizeof(http_get_response_t)); @@ -50,6 +50,15 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) { curl_easy_setopt(req, CURLOPT_SHARE, share); } + if (header_count > 0) { + struct curl_slist *chunk = NULL; + for (int i = 0; i < header_count; i++) { + chunk = curl_slist_append(chunk, headers[i]); + } + /* set our custom set of headers */ + curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk); + } + curl_easy_setopt(req, CURLOPT_URL, url); curl_easy_setopt(req, CURLOPT_HTTPGET, 1); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1); @@ -70,8 +79,8 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) { * Perform an HTTP(S) GET on `url` */ -http_get_response_t *http_get(const char *url) { - return http_get_shared(url, NULL); +http_get_response_t *http_get(const char *url, const char** headers, int header_count) { + return http_get_shared(url, NULL, headers, header_count); } /** diff --git a/deps/http-get/http-get.h b/deps/http-get/http-get.h index d584086d..31324f03 100644 --- a/deps/http-get/http-get.h +++ b/deps/http-get/http-get.h @@ -21,8 +21,8 @@ typedef struct { int ok; } http_get_response_t; -http_get_response_t *http_get(const char *); -http_get_response_t *http_get_shared(const char *, void *); +http_get_response_t *http_get(const char *url, const char** headers, int header_count); +http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count); int http_get_file(const char *, const char *); int http_get_file_shared(const char *, const char *, void *); diff --git a/deps/url/package.json b/deps/url/package.json new file mode 100644 index 00000000..c622b7ae --- /dev/null +++ b/deps/url/package.json @@ -0,0 +1,9 @@ +{ + "name": "url", + "version": "0.0.2", + "repo": "jwerle/url.h", + "description": "Parse URLs much like Node's url module", + "keywords": ["url", "parse"], + "license": "MIT", + "src": ["url.h"] +} diff --git a/deps/url/url.h b/deps/url/url.h new file mode 100644 index 00000000..cea26de4 --- /dev/null +++ b/deps/url/url.h @@ -0,0 +1,594 @@ +#ifndef URL_H +#define URL_H 1 + +/** + * Dependencies + */ + +#include +#include +#include +#include + + +/** + * url.h version + */ + +#define URL_VERSION 0.0.2 + + +/** + * Max length of a url protocol scheme + */ + +#define URL_PROTOCOL_MAX_LENGTH 16 + + +/** + * Max length of a url host part + */ + +#define URL_HOSTNAME_MAX_LENGTH 128 + + +/** + * Max length of a url tld part + */ + +#define URL_TLD_MAX_LENGTH 16 + + +/** + * Max length of a url auth part + */ + +#define URL_AUTH_MAX_LENGTH 32 + + +/** + * URI Schemes + * http://en.wikipedia.org/wiki/URI_scheme + */ + +char *URL_SCHEMES[] = { + // official IANA registered schemes + "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw", + "beshare", "bitcoin", "bolo", "callto", "cap", "chrome", "crome-extension", "com-evenbrite-attendee", + "cid", "coap", "coaps","content", "crid", "cvs", "data", "dav", "dict", "lna-playsingle", "dln-playcontainer", + "dns", "dtn", "dvb", "ed2k", "facetime", "fax", "feed", "file", "finger", "fish","ftp", "geo", "gg","git", + "gizmoproject", "go", "gopher", "gtalk", "h323", "hcp", "http", "https", "iax", "icap", "icon","im", + "imap", "info", "ipn", "ipp", "irc", "irc6", "ircs", "iris", "iris.beep", "iris.xpc", "iris.xpcs","iris.lws", + "itms", "jabber", "jar", "jms", "keyparc", "lastfm", "ldap", "ldaps", "magnet", "mailserver","mailto", + "maps", "market", "message", "mid", "mms", "modem", "ms-help", "mssettings-power", "msnim", "msrp", + "msrps", "mtqp", "mumble", "mupdate", "mvn", "news", "nfs", "ni", "nih", "nntp", "notes","oid", + "paquelocktoken", "pack", "palm", "paparazzi", "pkcs11", "platform", "pop", "pres", "prospero", "proxy", + "psyc","query", "reload", "res", "resource", "rmi", "rsync", "rtmp","rtsp", "secondlife", "service","session", + "sftp", "sgn", "shttp", "sieve", "sip", "sips", "skype", "smb", "sms", "snews", "snmp", "soap.beep","soap.beeps", + "soldat", "spotify", "ssh", "steam", "svn", "tag", "teamspeak", "tel", "telnet", "tftp", "things","thismessage", + "tn3270", "tip", "tv", "udp", "unreal", "urn", "ut2004", "vemmi","ventrilo", "videotex", "view-source", "wais","webcal", + "ws", "wss", "wtai", "wyciwyg", "xcon", "xcon-userid", "xfire","xmlrpc.beep", "xmlrpc.beeps", "xmpp", "xri","ymsgr", + + // unofficial schemes + "javascript", "jdbc", "doi" +}; + + +/** + * `url_data` struct that defines parts + * of a parsed URL such as host and protocol + */ + +typedef struct url_data { + char *href; + char *protocol; + char *host; + char *auth; + char *hostname; + char *pathname; + char *search; + char *path; + char *hash; + char *query; + char *port; +} url_data_t; + + +// prototype + +/** + * Parses a url into parts and returns + * a `url_data_t *` pointer + */ + +url_data_t * +url_parse (char *url); + +char * +url_get_protocol (char *url); + +char * +url_get_auth (char *url); + +char * +url_get_hostname (char *url); + +char * +url_get_host (char *url); + +char * +url_get_pathname (char *url); + +char * +url_get_path (char *url); + +char * +url_get_search (char *url); + +char * +url_get_query (char *url); + +char * +url_get_hash (char *url); + +char * +url_get_port (char *url); + +void +url_free (url_data_t *data); + +bool +url_is_protocol (char *str); + +bool +url_is_ssh (char *str); + +void +url_inspect (char *url); + +void +url_data_inspect (url_data_t *data); + + +// implementation + + +// non C99 standard functions +#if _POSIX_C_SOURCE < 200809L +char * +strdup (const char *str) { + int n = strlen(str) + 1; + char *dup = malloc(n); + if (dup) strcpy(dup, str); + return dup; +} +#endif + + +static char * +strff (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + *ptr++; + } + + return strdup(ptr); +} + +static char * +strrwd (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + *ptr--; + } + + return strdup(ptr); +} + +static char * +get_part (char *url, const char *format, int l) { + bool has = false; + char *tmp = strdup(url); + char *tmp_url = strdup(url); + char *fmt_url = strdup(url); + char *ret; + + if (!tmp || !tmp_url || !fmt_url) + return NULL; + + strcpy(tmp, ""); + strcpy(fmt_url, ""); + + // move pointer exactly the amount + // of characters in the `prototcol` char + // plus 3 characters that represent the `://` + // part of the url + char* fmt_url_new = strff(fmt_url, l); + free(fmt_url); + fmt_url = fmt_url_new; + + sscanf(fmt_url, format, tmp); + + if (0 != strcmp(tmp, tmp_url)) { + has = true; + ret = strdup(tmp); + } + + free(tmp); + free(tmp_url); + free(fmt_url); + + return has? ret : NULL; +} + +url_data_t * +url_parse (char *url) { + url_data_t *data = malloc(sizeof(url_data_t)); + if (!data) return NULL; + + data->href = url; + char *tmp_url = strdup(url); + bool is_ssh = false; + + char *protocol = url_get_protocol(tmp_url); + if (!protocol) { + free(tmp_url); + return NULL; + } + // length of protocol plus :// + int protocol_len = (int) strlen(protocol) + 3; + data->protocol = protocol; + + is_ssh = url_is_ssh(protocol); + + char *auth = malloc(sizeof(char)); + int auth_len = 0; + if (strstr(tmp_url, "@")) { + auth = get_part(tmp_url, "%[^@]", protocol_len); + auth_len = strlen(auth); + if (auth) auth_len++; + } + + data->auth = auth; + + char *hostname; + + hostname = (is_ssh) + ? get_part(tmp_url, "%[^:]", protocol_len + auth_len) + : get_part(tmp_url, "%[^/]", protocol_len + auth_len); + + if (!hostname) { + free(tmp_url); + url_free(data); + return NULL; + } + int hostname_len = (int) strlen(hostname); + char *tmp_hostname = strdup(hostname); + data->hostname = hostname; + + char *host = malloc((strlen(tmp_hostname)+1) * sizeof(char)); + sscanf(tmp_hostname, "%[^:]", host); + free(tmp_hostname); + if (!host) { + free(tmp_url); + url_free(data); + return NULL; + } + data->host = host; + + int host_len = (int)strlen(host); + if (hostname_len > host_len) { + data->port = strff(hostname, host_len + 1); // +1 for ':' char; + } else { + data->port = NULL; + } + + char *tmp_path; + + tmp_path = (is_ssh) + ? get_part(tmp_url, ":%s", protocol_len + auth_len + hostname_len) + : get_part(tmp_url, "/%s", protocol_len + auth_len + hostname_len); + + char *path = malloc((strlen(tmp_path) + 2) * sizeof(char)); + if (!path) { + free(tmp_url); + url_free(data); + return NULL; + } + char *fmt = (is_ssh)? "%s" : "/%s"; + sprintf(path, fmt, tmp_path); + data->path = path; + + char *pathname = malloc((strlen(tmp_path) + 2) * sizeof(char)); + free(tmp_path); + if (!pathname) { + free(tmp_url); + url_free(data); + return NULL; + } + strcat(pathname, ""); + tmp_path = strdup(path); + sscanf(tmp_path, "%[^? | ^#]", pathname); + int pathname_len = (int)strlen(pathname); + data->pathname = pathname; + + char *search = malloc(sizeof(search)); + if (!search) { + free(tmp_url); + url_free(data); + return NULL; + } + char* tmp_path_new = strff(tmp_path, pathname_len); + free(tmp_path); + tmp_path = tmp_path_new; + strcpy(search, ""); + sscanf(tmp_path, "%[^#]", search); + data->search = search; + int search_len = (int)strlen(search); + free(tmp_path); + + char *query = malloc(sizeof(char)); + if (!query) { + free(tmp_url); + url_free(data); + return NULL; + } + sscanf(search, "?%s", query); + data->query = query; + + char *hash = malloc(sizeof(char)); + if (!hash) { + free(tmp_url); + url_free(data); + return NULL; + } + tmp_path = strff(path, pathname_len + search_len); + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + data->hash = hash; + free(tmp_path); + free(tmp_url); + + return data; +} + +bool +url_is_protocol (char *str) { + int count = sizeof(URL_SCHEMES) / sizeof(URL_SCHEMES[0]); + + for (int i = 0; i < count; ++i) { + if (0 == strcmp(URL_SCHEMES[i], str)) { + return true; + } + } + + return false; +} + +bool +url_is_ssh (char *str) { + str = strdup(str); + if (0 == strcmp(str, "ssh") || 0 == strcmp(str, "git")) { + free(str); + return true; + + } + free(str); + + return false; +} + +char * +url_get_protocol (char *url) { + char *protocol = malloc(URL_PROTOCOL_MAX_LENGTH * sizeof(char)); + if (!protocol) return NULL; + sscanf(url, "%[^://]", protocol); + if (url_is_protocol(protocol)) return protocol; + return NULL; +} + + +char * +url_get_auth (char *url) { + char *protocol = url_get_protocol(url); + if (!protocol) return NULL; + int l = (int) strlen(protocol) + 3; + return get_part(url, "%[^@]", l); +} + +char * +url_get_hostname (char *url) { + int l = 3; + char *protocol = url_get_protocol(url); + char *tmp_protocol = strdup(protocol); + char *auth = url_get_auth(url); + + if (!protocol) return NULL; + if (auth) l += strlen(auth) + 1; // add one @ symbol + if (auth) free(auth); + + l += (int) strlen(protocol); + + free(protocol); + + char * hostname = url_is_ssh(tmp_protocol) + ? get_part(url, "%[^:]", l) + : get_part(url, "%[^/]", l); + free(tmp_protocol); + return hostname; +} + +char * +url_get_host (char *url) { + char *host = malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + + if (!host || !hostname) return NULL; + + sscanf(hostname, "%[^:]", host); + + free(hostname); + + return host; +} + +char * +url_get_pathname (char *url) { + char *path = url_get_path(url); + char *pathname = malloc(sizeof(char)); + + if (!path || !pathname) return NULL; + + strcat(pathname, ""); + sscanf(path, "%[^?]", pathname); + + free(path); + + return pathname; +} + +char * +url_get_path (char *url) { + int l = 3; + char *tmp_path; + char *protocol = url_get_protocol(url); + char *auth = url_get_auth(url); + char *hostname = url_get_hostname(url); + + + if (!protocol || !hostname) + return NULL; + + bool is_ssh = url_is_ssh(protocol); + + l += (int) strlen(protocol) + (int) strlen(hostname); + + if (auth) l+= (int) strlen(auth) +1; // @ symbol + + tmp_path = (is_ssh) + ? get_part(url, ":%s", l) + : get_part(url, "/%s", l); + + char *fmt = (is_ssh)? "%s" : "/%s"; + char *path = malloc(strlen(tmp_path) * sizeof(char)); + sprintf(path, fmt, tmp_path); + + if (auth) free(auth); + free(protocol); + free(hostname); + free(tmp_path); + + return path; + +} + +char * +url_get_search (char *url) { + char *path = url_get_path(url); + char *pathname = url_get_pathname(url); + char *search = malloc(sizeof(char)); + + if (!path || !search) return NULL; + + char *tmp_path = strff(path, (int)strlen(pathname)); + strcat(search, ""); + sscanf(tmp_path, "%[^#]", search); + + tmp_path = strrwd(tmp_path, (int)strlen(pathname)); + + free(path); + free(pathname); + + return search; +} + +char * +url_get_query (char *url) { + char *search = url_get_search(url); + char *query = malloc(sizeof(char)); + if (!search) return NULL; + sscanf(search, "?%s", query); + free(search); + return query; +} + +char * +url_get_hash (char *url) { + char *hash = malloc(sizeof(char)); + if (!hash) return NULL; + + char *path = url_get_path(url); + if (!path) return NULL; + + char *pathname = url_get_pathname(url); + if (!pathname) return NULL; + char *search = url_get_search(url); + + int pathname_len = (int) strlen(pathname); + int search_len = (int) strlen(search); + char *tmp_path = strff(path, pathname_len + search_len); + + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + tmp_path = strrwd(tmp_path, pathname_len + search_len); + free(tmp_path); + free(pathname); + free(path); + if (search) free(search); + + return hash; +} + +char * +url_get_port (char *url) { + char *port = malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + char *host = url_get_host(url); + if (!port || !hostname) return NULL; + + char *tmp_hostname = strff(hostname, strlen(host) +1); + sscanf(tmp_hostname, "%s", port); + + free(hostname); + free(tmp_hostname); + return port; +} + +void +url_inspect (char *url) { + url_data_inspect(url_parse(url)); +} + + +void +url_data_inspect (url_data_t *data) { + printf("#url =>\n"); + printf(" .href: \"%s\"\n", data->href); + printf(" .protocol: \"%s\"\n", data->protocol); + printf(" .host: \"%s\"\n", data->host); + printf(" .auth: \"%s\"\n", data->auth); + printf(" .hostname: \"%s\"\n", data->hostname); + printf(" .pathname: \"%s\"\n", data->pathname); + printf(" .search: \"%s\"\n", data->search); + printf(" .path: \"%s\"\n", data->path); + printf(" .hash: \"%s\"\n", data->hash); + printf(" .query: \"%s\"\n", data->query); + printf(" .port: \"%s\"\n", data->port); +} + +void +url_free (url_data_t *data) { + if (!data) return; + if (data->auth) free(data->auth); + if (data->protocol) free(data->protocol); + if (data->hostname) free(data->hostname); + if (data->host) free(data->host); + if (data->pathname) free(data->pathname); + if (data->path) free(data->path); + if (data->hash) free(data->hash); + if (data->port) free(data->port); + if (data->search) free(data->search); + if (data->query) free(data->query); + free(data); +} + + +#endif diff --git a/src/clib-search.c b/src/clib-search.c index 926e065a..f273ffec 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -62,169 +62,192 @@ static void setopt_json(command_t *self) { opt_json = 1; } } static int matches(int count, char *args[], wiki_package_ptr_t pkg) { - // Display all packages if there's no query - if (0 == count) - return 1; - - char *description = NULL; - char *name = NULL; - char *repo = NULL; - char *href = NULL; - int rc = 0; - - name = clib_package_parse_name(wiki_package_get_repo(pkg)); - COMPARE(name); - - description = strdup(wiki_package_get_description(pkg)); - COMPARE(description); - - repo = strdup(wiki_package_get_repo(pkg)); - COMPARE(repo); - - href = strdup(wiki_package_get_href(pkg)); - COMPARE(href); - -cleanup: - free(description); - free(name); - free(repo); - free(href); - return rc; + // Display all packages if there's no query + if (0 == count) + return 1; + + char *description = NULL; + char *name = NULL; + char *repo = NULL; + char *href = NULL; + int rc = 0; + + name = clib_package_parse_name(wiki_package_get_repo(pkg)); + COMPARE(name); + + description = strdup(wiki_package_get_description(pkg)); + COMPARE(description); + + repo = strdup(wiki_package_get_repo(pkg)); + COMPARE(repo); + + href = strdup(wiki_package_get_href(pkg)); + COMPARE(href); + + cleanup: + free(description); + free(name); + free(repo); + free(href); + return rc; } +/* static char *wiki_html_cache() { + if (clib_cache_has_search() && opt_cache) { + char *data = clib_cache_read_search(); - if (clib_cache_has_search() && opt_cache) { - char *data = clib_cache_read_search(); - - if (data) { - return data; + if (data) { + return data; + } } - } - debug(&debugger, "setting cache from %s", CLIB_WIKI_URL); - http_get_response_t *res = http_get(CLIB_WIKI_URL); - if (!res->ok) - return NULL; + debug(&debugger, "setting cache from %s", CLIB_WIKI_URL); + http_get_response_t *res = http_get(CLIB_WIKI_URL); + if (!res->ok) + return NULL; - char *html = strdup(res->data); - if (NULL == html) - return NULL; - http_get_free(res); + char *html = strdup(res->data); + if (NULL == html) + return NULL; + http_get_free(res); - if (NULL == html) + if (NULL == html) + return html; + clib_cache_save_search(html); + debug(&debugger, "wrote cache"); return html; - clib_cache_save_search(html); - debug(&debugger, "wrote cache"); - return html; } +*/ static void display_package(const wiki_package_ptr_t pkg, cc_color_t fg_color_highlight, cc_color_t fg_color_text) { - cc_fprintf(fg_color_highlight, stdout, " %s\n", wiki_package_get_repo(pkg)); - printf(" url: "); - cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_href(pkg)); - printf(" desc: "); - cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_description(pkg)); - printf("\n"); + cc_fprintf(fg_color_highlight, stdout, " %s\n", wiki_package_get_repo(pkg)); + printf(" url: "); + cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_href(pkg)); + printf(" desc: "); + cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_description(pkg)); + printf("\n"); } static void add_package_to_json(const wiki_package_ptr_t pkg, JSON_Array *json_list) { - JSON_Value *json_pkg_root = json_value_init_object(); - JSON_Object *json_pkg = json_value_get_object(json_pkg_root); + JSON_Value *json_pkg_root = json_value_init_object(); + JSON_Object *json_pkg = json_value_get_object(json_pkg_root); - json_object_set_string(json_pkg, "repo", wiki_package_get_repo(pkg)); - json_object_set_string(json_pkg, "href", wiki_package_get_href(pkg)); - json_object_set_string(json_pkg, "description", wiki_package_get_description(pkg)); - json_object_set_string(json_pkg, "category", wiki_package_get_category(pkg)); + json_object_set_string(json_pkg, "repo", wiki_package_get_repo(pkg)); + json_object_set_string(json_pkg, "href", wiki_package_get_href(pkg)); + json_object_set_string(json_pkg, "description", wiki_package_get_description(pkg)); + json_object_set_string(json_pkg, "category", wiki_package_get_category(pkg)); - json_array_append_value(json_list, json_pkg_root); + json_array_append_value(json_list, json_pkg_root); } int main(int argc, char *argv[]) { - opt_color = 1; - opt_cache = 1; + opt_color = 1; + opt_cache = 1; - debug_init(&debugger, "clib-search"); + debug_init(&debugger, "clib-search"); - clib_cache_init(CLIB_SEARCH_CACHE_TIME); + clib_cache_init(CLIB_SEARCH_CACHE_TIME); - command_t program; - command_init(&program, "clib-search", CLIB_VERSION); - program.usage = "[options] [query ...]"; + command_t program; + command_init(&program, "clib-search", CLIB_VERSION); + program.usage = "[options] [query ...]"; - command_option(&program, "-n", "--no-color", "don't colorize output", - setopt_nocolor); + command_option(&program, "-n", "--no-color", "don't colorize output", + setopt_nocolor); - command_option(&program, "-c", "--skip-cache", "skip the search cache", - setopt_nocache); + command_option(&program, "-c", "--skip-cache", "skip the search cache", + setopt_nocache); - command_option(&program, "-j", "--json", "generate a serialized JSON output", - setopt_json); + command_option(&program, "-j", "--json", "generate a serialized JSON output", + setopt_json); - command_parse(&program, argc, argv); + command_parse(&program, argc, argv); - for (int i = 0; i < program.argc; i++) - case_lower(program.argv[i]); + for (int i = 0; i < program.argc; i++) + case_lower(program.argv[i]); - // set color theme - cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; - cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; + // set color theme + cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; + cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; + // We search the local manifest for extra registries. + // It is important to give the extra registries preference over the default registry. + clib_package_t *package = clib_package_load_local_manifest(0); + wiki_registry_ptr_t* registries = malloc((package->registries->len + 1) * sizeof(wiki_registry_ptr_t)); + int registry_count = 0; - wiki_registry_ptr_t registry = wiki_registry_create(CLIB_WIKI_URL); - wiki_registry_fetch(registry); - - // TODO, implement caching for the new registries. - /* - char *html = wiki_html_cache(); - if (NULL == html) { - command_free(&program); - logger_error("error", "failed to fetch wiki HTML"); - return 1; + list_iterator_t *registry_iterator = list_iterator_new(package->registries, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(registry_iterator))) { + registries[registry_count] = wiki_registry_create(node->val); + if (!wiki_registry_fetch(registries[registry_count])) { + printf("SEARCH, could not list packages from. %s\n", wiki_registry_get_url(registries[registry_count])); + } else { + registry_count++; + } } - list_t *pkgs = wiki_registry_parse(html); - free(html); - debug(&debugger, "found %zu packages", pkgs->len); - */ - - wiki_package_ptr_t pkg; - wiki_registry_iterator_t it = wiki_registry_iterator_new(registry); - - JSON_Array *json_list = NULL; - JSON_Value *json_list_root = NULL; - - if (opt_json) { - json_list_root = json_value_init_array(); - json_list = json_value_get_array(json_list_root); - } - - printf("\n"); - - while ((pkg = wiki_registry_iterator_next(it))) { - if (matches(program.argc, program.argv, pkg)) { - if (opt_json) { - add_package_to_json(pkg, json_list); - } else { - display_package(pkg, fg_color_highlight, fg_color_text); + list_iterator_destroy(registry_iterator); + + // And add the default registry. + //registries[registry_count] = wiki_registry_create(CLIB_WIKI_URL); + //wiki_registry_fetch(registries[registry_count]); + //registry_count++; + + // TODO, implement caching for the new registries. + /* + char *html = wiki_html_cache(); + if (NULL == html) { + command_free(&program); + logger_error("error", "failed to fetch wiki HTML"); + return 1; } - } else { - debug(&debugger, "skipped package %s", wiki_package_get_repo(pkg)); + list_t *pkgs = wiki_registry_parse(html); + free(html); + debug(&debugger, "found %zu packages", pkgs->len); + */ + + for (int i = 0; i < registry_count; i++) { + printf("SEARCH: packages from %s\n", wiki_registry_get_url(registries[i])); + wiki_package_ptr_t pkg; + wiki_registry_iterator_t it = wiki_registry_iterator_new(registries[i]); + + JSON_Array *json_list = NULL; + JSON_Value *json_list_root = NULL; + + if (opt_json) { + json_list_root = json_value_init_array(); + json_list = json_value_get_array(json_list_root); + } + + printf("\n"); + + while ((pkg = wiki_registry_iterator_next(it))) { + if (matches(program.argc, program.argv, pkg)) { + if (opt_json) { + add_package_to_json(pkg, json_list); + } else { + display_package(pkg, fg_color_highlight, fg_color_text); + } + } else { + debug(&debugger, "skipped package %s", wiki_package_get_repo(pkg)); + } + } + + if (opt_json) { + char *serialized = json_serialize_to_string_pretty(json_list_root); + puts(serialized); + + json_free_serialized_string(serialized); + json_value_free(json_list_root); + } + + wiki_registry_iterator_destroy(it); + wiki_registry_free(registries[i]); } - } - - if (opt_json) { - char *serialized = json_serialize_to_string_pretty(json_list_root); - puts(serialized); - - json_free_serialized_string(serialized); - json_value_free(json_list_root); - } - - wiki_registry_iterator_destroy(it); - wiki_registry_free(registry); - command_free(&program); - return 0; + command_free(&program); + return 0; } diff --git a/src/common/clib-package.c b/src/common/clib-package.c index c01639ca..7e08a0ce 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -563,6 +563,24 @@ clib_package_t *clib_package_new(const char *json, int verbose) { pkg->src = NULL; } + if (!(pkg->registries = list_new())) { + goto cleanup; + } + JSON_Array* registries = json_object_get_array(json_object, "registries"); + if (registries) { + pkg->registries->free = free; + for (unsigned int i = 0; i < json_array_get_count(registries); i++) { + char *file = json_array_get_string_safe(registries, i); + _debug("file: %s", file); + if (!file) + goto cleanup; + if (!(list_rpush(pkg->registries, list_node_new(file)))) + goto cleanup; + } + } else { + _debug("no extra registries listed in clib.json or package.json file"); + } + if ((deps = json_object_get_object(json_object, "dependencies"))) { if (!(pkg->dependencies = parse_package_deps(deps))) { goto cleanup; @@ -661,7 +679,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, _debug("GET %s", json_url); // clean up when retrying http_get_free(res); - res = http_get_shared(json_url, clib_package_curl_share); + res = http_get_shared(json_url, clib_package_curl_share, NULL, 0); #else res = http_get(json_url); #endif diff --git a/src/common/clib-package.h b/src/common/clib-package.h index 3007ec83..dab17fcb 100644 --- a/src/common/clib-package.h +++ b/src/common/clib-package.h @@ -37,6 +37,7 @@ typedef struct { list_t *dependencies; list_t *development; list_t *src; + list_t *registries; void *data; // user data unsigned int refs; } clib_package_t; diff --git a/src/common/clib-release-info.c b/src/common/clib-release-info.c index c64a2dbf..3c75005c 100644 --- a/src/common/clib-release-info.c +++ b/src/common/clib-release-info.c @@ -18,7 +18,7 @@ static debug_t debugger; const char *clib_release_get_latest_tag(void) { debug_init(&debugger, "clib-release-info"); - http_get_response_t *res = http_get(LATEST_RELEASE_ENDPOINT); + http_get_response_t *res = http_get(LATEST_RELEASE_ENDPOINT, NULL, 0); JSON_Value *root_json = NULL; JSON_Object *json_object = NULL; diff --git a/src/registry/github-registry.c b/src/registry/github-registry.c index 9d9286b4..037a570d 100644 --- a/src/registry/github-registry.c +++ b/src/registry/github-registry.c @@ -121,7 +121,7 @@ list_t *wiki_registry_parse(const char *html) { * Get a list of packages from the given GitHub wiki `url`. */ list_t *github_registry_fetch(const char *url) { - http_get_response_t *res = http_get(url); + http_get_response_t *res = http_get(url, NULL, 0); if (!res->ok) return NULL; list_t *list = wiki_registry_parse(res->data); diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c index a3c0b82c..446fd9e6 100644 --- a/src/registry/gitlab-registry.c +++ b/src/registry/gitlab-registry.c @@ -62,10 +62,44 @@ static wiki_package_ptr_t parse_li(GumboNode *li, const char* hostname) { * Parse a list of packages from the given `html` */ static list_t *wiki_registry_parse(const char* hostname, const char *html) { - GumboOutput *output = gumbo_parse(html); list_t *pkgs = list_new(); - GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); + // Try to parse the markdown file. + char* input = strdup(html); + char* line; + char* category = NULL; + while ((line = strsep(&input, "\n"))) { + char* dash_position = strstr(line, "-"); + // The line starts with a dash, so we expect a package. + if (dash_position != NULL && dash_position - line < 4) { + + char* link_name_start = strstr(line, "[")+1; + char* link_name_end = strstr(link_name_start, "]"); + char* link_value_start = strstr(link_name_end, "(")+1; + char* link_value_end = strstr(link_value_start, ")"); + char* description_position = strstr(link_value_end, "-")+1; + + wiki_package_ptr_t package = wiki_package_new(); + package->href = strndup(link_value_start, link_value_end-link_value_start); + package->repo = strndup(link_name_start, link_name_end-link_name_start); + package->description = strdup(description_position); + package->category = strdup(category != NULL ? category: "unknown"); + list_rpush(pkgs, list_node_new(package)); + } + + char* header_position = strstr(line, "##"); + // The category starts with a ##. + if (header_position != NULL && header_position - line < 4) { + category = header_position+2; + } + } + + free(input); + + return pkgs; + + GumboOutput *output = gumbo_parse(html); + GumboNode *body = gumbo_get_element_by_id("content-body", output->root); if (body) { // grab all category `

`s list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); @@ -119,12 +153,14 @@ static list_t *wiki_registry_parse(const char* hostname, const char *html) { /** * Get a list of packages from the given gitlab wiki `url`. + * TODO, get the secret from a secrets file. */ list_t *gitlab_registry_fetch(const char *url, const char* hostname) { - http_get_response_t *res = http_get(url); + char* headers[1] = {"PRIVATE-TOKEN: SECRET"}; + http_get_response_t *res = http_get(url, headers, 1); if (!res->ok) return NULL; - list_t *list = wiki_registry_parse(url, res->data); + list_t *list = wiki_registry_parse(hostname, res->data); http_get_free(res); return list; } diff --git a/src/registry/wiki-registry.c b/src/registry/wiki-registry.c index a1eedb0e..2c842132 100644 --- a/src/registry/wiki-registry.c +++ b/src/registry/wiki-registry.c @@ -87,14 +87,23 @@ void wiki_registry_free(wiki_registry_ptr_t registry) { free(registry); } -// TODO, return false if the request fails. +const char* wiki_registry_get_url(wiki_registry_ptr_t registry) { + return registry->url; +} + bool wiki_registry_fetch(wiki_registry_ptr_t registry) { switch (registry->type) { case REGISTRY_TYPE_GITLAB: registry->packages = gitlab_registry_fetch(registry->url, registry->hostname); + if (registry->packages != NULL) { + return true; + } break; case REGISTRY_TYPE_GITHUB: registry->packages = github_registry_fetch(registry->url); + if (registry->packages != NULL) { + return true; + } break; default: return false; diff --git a/src/registry/wiki-registry.h b/src/registry/wiki-registry.h index 677ead26..695e216e 100644 --- a/src/registry/wiki-registry.h +++ b/src/registry/wiki-registry.h @@ -33,6 +33,13 @@ void wiki_registry_free(wiki_registry_ptr_t registry); */ bool wiki_registry_fetch(wiki_registry_ptr_t registry); +/** + * Get the url for the registry + * @param registry + * @return + */ +const char* wiki_registry_get_url(wiki_registry_ptr_t registry); + /** * An iterator through the packages in the registry. */ From 5ad29779037a1cec565d75c06ae290782d7f8295 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Thu, 25 Mar 2021 00:36:44 +0100 Subject: [PATCH 04/15] Add support for custom registries and repositories, read secrets from clib_secrets.json misc: Cleanup, add headers. --- Makefile | 6 +- clib.json | 2 +- deps/http-get/http-get.c | 13 +- deps/http-get/http-get.h | 2 +- deps/url/url.h | 9 +- src/clib-build.c | 6 +- src/clib-configure.c | 6 +- src/clib-install.c | 48 +-- src/clib-search.c | 73 ++--- src/clib-update.c | 2 +- src/clib-upgrade.c | 2 +- src/common/clib-package.c | 404 ++------------------------ src/common/clib-package.h | 2 +- src/common/clib-secrets.c | 81 ++++++ src/common/clib-secrets.h | 10 + src/common/url.c | 5 + src/registry/github-registry.c | 35 +-- src/registry/gitlab-registry.c | 199 ++++--------- src/registry/gitlab-registry.h | 2 +- src/registry/registry-internal.h | 23 ++ src/registry/registry-manager.c | 78 +++++ src/registry/registry-manager.h | 44 +++ src/registry/registry.c | 160 ++++++++++ src/registry/registry.h | 64 ++++ src/registry/wiki-registry-internal.h | 17 -- src/registry/wiki-registry.c | 147 ---------- src/registry/wiki-registry.h | 55 ---- src/repository/github-repository.c | 37 +++ src/repository/github-repository.h | 12 + src/repository/gitlab-repository.c | 26 ++ src/repository/gitlab-repository.h | 12 + src/repository/repository.c | 264 +++++++++++++++++ src/repository/repository.h | 50 ++++ 33 files changed, 1062 insertions(+), 834 deletions(-) create mode 100644 src/common/clib-secrets.c create mode 100644 src/common/clib-secrets.h create mode 100644 src/common/url.c create mode 100644 src/registry/registry-internal.h create mode 100644 src/registry/registry-manager.c create mode 100644 src/registry/registry-manager.h create mode 100644 src/registry/registry.c create mode 100644 src/registry/registry.h delete mode 100644 src/registry/wiki-registry-internal.h delete mode 100644 src/registry/wiki-registry.c delete mode 100644 src/registry/wiki-registry.h create mode 100644 src/repository/github-repository.c create mode 100644 src/repository/github-repository.h create mode 100644 src/repository/gitlab-repository.c create mode 100644 src/repository/gitlab-repository.h create mode 100644 src/repository/repository.c create mode 100644 src/repository/repository.h diff --git a/Makefile b/Makefile index 7150695a..849ea295 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ RM = rm -f MKDIR = mkdir -p SRC = $(wildcard src/*.c) -COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c) -ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h test/package/*.c test/cache/*.c) +COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*) +ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c) SDEPS = $(wildcard deps/*/*.c) ODEPS = $(SDEPS:.c=.o) DEPS = $(filter-out $(ODEPS), $(SDEPS)) @@ -21,7 +21,7 @@ OBJS = $(DEPS:.c=.o) export CC -CFLAGS += -std=c99 -Ideps -Wall -Werror=return--type -Wno-unused-function -U__STRICT_ANSI__ +CFLAGS += -std=c99 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -Wall -Werror=return--type -Wno-unused-function -U__STRICT_ANSI__ ifdef STATIC CFLAGS += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags) diff --git a/clib.json b/clib.json index 8c6954c6..ac853563 100644 --- a/clib.json +++ b/clib.json @@ -44,7 +44,7 @@ "stephenmathieson/gumbo-get-element-by-id.c": "*", "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", "clibs/list": "*", - "jwerle/url.h": "0.0.*" + "jwerle/url.h": "0.1.*" }, "development": { "stephenmathieson/describe.h": "2.0.1" diff --git a/deps/http-get/http-get.c b/deps/http-get/http-get.c index d11c32ad..e90a664c 100644 --- a/deps/http-get/http-get.c +++ b/deps/http-get/http-get.c @@ -97,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea * Request `url` and save to `file` */ -int http_get_file_shared(const char *url, const char *file, CURLSH *share) { +int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** headers, int header_count) { CURL *req = curl_easy_init(); if (!req) return -1; @@ -108,6 +108,15 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) { curl_easy_setopt(req, CURLOPT_SHARE, share); } + if (header_count > 0) { + struct curl_slist *chunk = NULL; + for (int i = 0; i < header_count; i++) { + chunk = curl_slist_append(chunk, headers[i]); + } + /* set our custom set of headers */ + curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk); + } + curl_easy_setopt(req, CURLOPT_URL, url); curl_easy_setopt(req, CURLOPT_HTTPGET, 1); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1); @@ -125,7 +134,7 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) { } int http_get_file(const char *url, const char *file) { - return http_get_file_shared(url, file, NULL); + return http_get_file_shared(url, file, NULL, NULL, 0); } /** diff --git a/deps/http-get/http-get.h b/deps/http-get/http-get.h index 31324f03..af713512 100644 --- a/deps/http-get/http-get.h +++ b/deps/http-get/http-get.h @@ -25,7 +25,7 @@ http_get_response_t *http_get(const char *url, const char** headers, int header_ http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count); int http_get_file(const char *, const char *); -int http_get_file_shared(const char *, const char *, void *); +int http_get_file_shared(const char *, const char *, void *, const char** headers, int header_count); void http_get_free(http_get_response_t *); diff --git a/deps/url/url.h b/deps/url/url.h index cea26de4..a9ea7597 100644 --- a/deps/url/url.h +++ b/deps/url/url.h @@ -51,6 +51,7 @@ * http://en.wikipedia.org/wiki/URI_scheme */ +#ifdef URL_H_IMPLEMENTATION char *URL_SCHEMES[] = { // official IANA registered schemes "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw", @@ -72,6 +73,7 @@ char *URL_SCHEMES[] = { // unofficial schemes "javascript", "jdbc", "doi" }; +#endif /** @@ -102,7 +104,7 @@ typedef struct url_data { */ url_data_t * -url_parse (char *url); +url_parse (const char *url); char * url_get_protocol (char *url); @@ -151,7 +153,7 @@ url_data_inspect (url_data_t *data); // implementation - +#ifdef URL_H_IMPLEMENTATION // non C99 standard functions #if _POSIX_C_SOURCE < 200809L @@ -220,7 +222,7 @@ get_part (char *url, const char *format, int l) { } url_data_t * -url_parse (char *url) { +url_parse (const char *url) { url_data_t *data = malloc(sizeof(url_data_t)); if (!data) return NULL; @@ -590,5 +592,6 @@ url_free (url_data_t *data) { free(data); } +#endif #endif diff --git a/src/clib-build.c b/src/clib-build.c index 8f3bceda..a84125c0 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -199,7 +199,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { #ifdef DEBUG package = clib_package_new_from_slug(dir, 1); #else - package = clib_package_new_from_slug(dir, 0); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif } @@ -330,7 +330,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { char *dep_dir = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); if (opts.dir && dependency && dependency->name) { dep_dir = path_join(opts.dir, dependency->name); } @@ -401,7 +401,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); diff --git a/src/clib-configure.c b/src/clib-configure.c index b2dffcad..da3fcdc5 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -192,7 +192,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { #ifdef DEBUG package = clib_package_new_from_slug(dir, 1); #else - package = clib_package_new_from_slug(dir, 0); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif } @@ -288,7 +288,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); @@ -359,7 +359,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); diff --git a/src/clib-install.c b/src/clib-install.c index 3e1538aa..07dd3486 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -17,9 +17,12 @@ #include "parson/parson.h" #include "str-replace/str-replace.h" #include "version.h" +#include #include #include #include +#include +#include #include #include #include @@ -251,21 +254,6 @@ static int install_package(const char *slug) { long path_max = 4096; #endif - if (!root_package) { - const char *name = NULL; - char *json = NULL; - unsigned int i = 0; - - do { - name = manifest_names[i]; - json = fs_read(name); - } while (NULL != manifest_names[++i] && !json); - - if (json) { - root_package = clib_package_new(json, opts.verbose); - } - } - if ('.' == slug[0]) { if (1 == strlen(slug) || ('/' == slug[1] && 2 == strlen(slug))) { char dir[path_max]; @@ -291,10 +279,21 @@ static int install_package(const char *slug) { } } - if (!pkg) { - pkg = clib_package_new_from_slug(slug, opts.verbose); + // Read local config files. + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + repository_init(secrets); // The repository requires the secrets for authentication. + clib_package_t *package = clib_package_load_local_manifest(0); + + registries_t registries = registry_manager_init_registries(package->registries, secrets); + registry_manager_fetch_registries(registries); + registry_package_ptr_t package_info = registry_manger_find_package(registries, slug); + if (!package_info) { + debug(&debugger, "Package %s not found in any registry.", slug); + return -1; } + + pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), opts.verbose); if (NULL == pkg) return -1; @@ -437,6 +436,21 @@ int main(int argc, char *argv[]) { clib_package_set_opts(package_opts); + if (!root_package) { + const char *name = NULL; + char *json = NULL; + unsigned int i = 0; + + do { + name = manifest_names[i]; + json = fs_read(name); + } while (NULL != manifest_names[++i] && !json); + + if (json) { + root_package = clib_package_new(json, opts.verbose); + } + } + int code = 0 == program.argc ? install_local_packages() : install_packages(program.argc, program.argv); diff --git a/src/clib-search.c b/src/clib-search.c index f273ffec..e7a7ec95 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -17,14 +17,17 @@ #include "http-get/http-get.h" #include "logger/logger.h" #include "parson/parson.h" +#include "registry-manager.h" +#include "registry/registry.h" #include "strdup/strdup.h" #include "tempdir/tempdir.h" #include "version.h" -#include "registry/wiki-registry.h" +#include #include #include #include #include +#include #define CLIB_WIKI_URL "https://github.com/clibs/clib/wiki/Packages" @@ -61,7 +64,7 @@ static void setopt_json(command_t *self) { opt_json = 1; } } \ } -static int matches(int count, char *args[], wiki_package_ptr_t pkg) { +static int matches(int count, char *args[], registry_package_ptr_t pkg) { // Display all packages if there's no query if (0 == count) return 1; @@ -72,16 +75,16 @@ static int matches(int count, char *args[], wiki_package_ptr_t pkg) { char *href = NULL; int rc = 0; - name = clib_package_parse_name(wiki_package_get_repo(pkg)); + name = clib_package_parse_name(registry_package_get_id(pkg)); COMPARE(name); - description = strdup(wiki_package_get_description(pkg)); + description = strdup(registry_package_get_description(pkg)); COMPARE(description); - repo = strdup(wiki_package_get_repo(pkg)); + repo = strdup(registry_package_get_id(pkg)); COMPARE(repo); - href = strdup(wiki_package_get_href(pkg)); + href = strdup(registry_package_get_href(pkg)); COMPARE(href); cleanup: @@ -120,26 +123,26 @@ static char *wiki_html_cache() { } */ -static void display_package(const wiki_package_ptr_t pkg, +static void display_package(const registry_package_ptr_t pkg, cc_color_t fg_color_highlight, cc_color_t fg_color_text) { - cc_fprintf(fg_color_highlight, stdout, " %s\n", wiki_package_get_repo(pkg)); + cc_fprintf(fg_color_highlight, stdout, " %s\n", registry_package_get_id(pkg)); printf(" url: "); - cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_href(pkg)); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_href(pkg)); printf(" desc: "); - cc_fprintf(fg_color_text, stdout, "%s\n", wiki_package_get_description(pkg)); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_description(pkg)); printf("\n"); } -static void add_package_to_json(const wiki_package_ptr_t pkg, +static void add_package_to_json(const registry_package_ptr_t pkg, JSON_Array *json_list) { JSON_Value *json_pkg_root = json_value_init_object(); JSON_Object *json_pkg = json_value_get_object(json_pkg_root); - json_object_set_string(json_pkg, "repo", wiki_package_get_repo(pkg)); - json_object_set_string(json_pkg, "href", wiki_package_get_href(pkg)); - json_object_set_string(json_pkg, "description", wiki_package_get_description(pkg)); - json_object_set_string(json_pkg, "category", wiki_package_get_category(pkg)); + json_object_set_string(json_pkg, "repo", registry_package_get_id(pkg)); + json_object_set_string(json_pkg, "href", registry_package_get_href(pkg)); + json_object_set_string(json_pkg, "description", registry_package_get_description(pkg)); + json_object_set_string(json_pkg, "category", registry_package_get_category(pkg)); json_array_append_value(json_list, json_pkg_root); } @@ -176,26 +179,11 @@ int main(int argc, char *argv[]) { // We search the local manifest for extra registries. // It is important to give the extra registries preference over the default registry. - clib_package_t *package = clib_package_load_local_manifest(0); - wiki_registry_ptr_t* registries = malloc((package->registries->len + 1) * sizeof(wiki_registry_ptr_t)); - int registry_count = 0; - - list_iterator_t *registry_iterator = list_iterator_new(package->registries, LIST_HEAD); - list_node_t *node; - while ((node = list_iterator_next(registry_iterator))) { - registries[registry_count] = wiki_registry_create(node->val); - if (!wiki_registry_fetch(registries[registry_count])) { - printf("SEARCH, could not list packages from. %s\n", wiki_registry_get_url(registries[registry_count])); - } else { - registry_count++; - } - } - list_iterator_destroy(registry_iterator); + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); - // And add the default registry. - //registries[registry_count] = wiki_registry_create(CLIB_WIKI_URL); - //wiki_registry_fetch(registries[registry_count]); - //registry_count++; + clib_package_t *package = clib_package_load_local_manifest(0); + registries_t registries = registry_manager_init_registries(package->registries, secrets); + registry_manager_fetch_registries(registries); // TODO, implement caching for the new registries. /* @@ -210,10 +198,12 @@ int main(int argc, char *argv[]) { debug(&debugger, "found %zu packages", pkgs->len); */ - for (int i = 0; i < registry_count; i++) { - printf("SEARCH: packages from %s\n", wiki_registry_get_url(registries[i])); - wiki_package_ptr_t pkg; - wiki_registry_iterator_t it = wiki_registry_iterator_new(registries[i]); + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t registry = NULL; + while ((registry = registry_iterator_next(it))) { + printf("SEARCH: packages from %s\n", registry_get_url(registry)); + registry_package_ptr_t pkg; + registry_package_iterator_t it = registry_package_iterator_new(registry); JSON_Array *json_list = NULL; JSON_Value *json_list_root = NULL; @@ -225,7 +215,7 @@ int main(int argc, char *argv[]) { printf("\n"); - while ((pkg = wiki_registry_iterator_next(it))) { + while ((pkg = registry_package_iterator_next(it))) { if (matches(program.argc, program.argv, pkg)) { if (opt_json) { add_package_to_json(pkg, json_list); @@ -233,7 +223,7 @@ int main(int argc, char *argv[]) { display_package(pkg, fg_color_highlight, fg_color_text); } } else { - debug(&debugger, "skipped package %s", wiki_package_get_repo(pkg)); + debug(&debugger, "skipped package %s", registry_package_get_id(pkg)); } } @@ -245,8 +235,7 @@ int main(int argc, char *argv[]) { json_value_free(json_list_root); } - wiki_registry_iterator_destroy(it); - wiki_registry_free(registries[i]); + registry_package_iterator_destroy(it); } command_free(&program); return 0; diff --git a/src/clib-update.c b/src/clib-update.c index 3c5fe4a4..5eebeccb 100755 --- a/src/clib-update.c +++ b/src/clib-update.c @@ -230,7 +230,7 @@ static int install_package(const char *slug) { } if (!pkg) { - pkg = clib_package_new_from_slug(slug, opts.verbose); + pkg = clib_package_new_from_slug_and_url(slug, "FIXME", opts.verbose); } if (NULL == pkg) diff --git a/src/clib-upgrade.c b/src/clib-upgrade.c index ecf8bb57..4046cf72 100755 --- a/src/clib-upgrade.c +++ b/src/clib-upgrade.c @@ -136,7 +136,7 @@ static int install_package(const char *slug) { logger_info("info", "Upgrading to %s", extended_slug); - pkg = clib_package_new_from_slug(extended_slug, opts.verbose); + pkg = clib_package_new_from_slug_and_url(extended_slug, "FIXME", opts.verbose); if (NULL == pkg) { logger_error("error", diff --git a/src/common/clib-package.c b/src/common/clib-package.c index 7e08a0ce..614980de 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -35,6 +35,7 @@ #ifdef HAVE_PTHREADS #include +#include #endif #ifndef DEFAULT_REPO_VERSION @@ -45,9 +46,6 @@ #define DEFAULT_REPO_OWNER "clibs" #endif -#define GITHUB_CONTENT_URL "https://raw.githubusercontent.com/" -#define GITHUB_CONTENT_URL_WITH_TOKEN "https://%s@raw.githubusercontent.com/" - #if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ defined(__MINGW64__) #define setenv(k, v, _) _putenv_s(k, v) @@ -57,17 +55,6 @@ static hash_t *visited_packages = 0; #ifdef HAVE_PTHREADS -typedef struct fetch_package_file_thread_data fetch_package_file_thread_data_t; -struct fetch_package_file_thread_data { - clib_package_t *pkg; - const char *dir; - char *file; - int verbose; - pthread_t thread; - pthread_attr_t attr; - void *data; -}; - typedef struct clib_package_lock clib_package_lock_t; struct clib_package_lock { pthread_mutex_t mutex; @@ -366,7 +353,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { if (NULL == slug) goto loop_cleanup; - pkg = clib_package_new_from_slug(slug, verbose); + pkg = clib_package_new_from_slug_and_url(slug, "FIXME", verbose); if (NULL == pkg) goto loop_cleanup; @@ -406,34 +393,6 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { return rc; } -#ifdef HAVE_PTHREADS -static void curl_lock_callback(CURL *handle, curl_lock_data data, - curl_lock_access access, void *userptr) { - pthread_mutex_lock(&lock.mutex); -} - -static void curl_unlock_callback(CURL *handle, curl_lock_data data, - curl_lock_access access, void *userptr) { - pthread_mutex_unlock(&lock.mutex); -} - -static void init_curl_share() { - if (0 == clib_package_curl_share) { - pthread_mutex_lock(&lock.mutex); - clib_package_curl_share = curl_share_init(); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_SHARE, - CURL_LOCK_DATA_CONNECT); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_LOCKFUNC, - curl_lock_callback); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_UNLOCKFUNC, - curl_unlock_callback); - curl_share_setopt(clib_package_curl_share, CURLOPT_NETRC, - CURL_NETRC_OPTIONAL); - pthread_mutex_unlock(&lock.mutex); - } -} -#endif - /** * Create a new clib package from the given `json` */ @@ -613,12 +572,10 @@ clib_package_t *clib_package_new(const char *json, int verbose) { } static clib_package_t * -clib_package_new_from_slug_with_package_name(const char *slug, int verbose, - const char *file) { +clib_package_new_from_slug_with_package_name(const char *slug, const char* url, int verbose, const char *file) { char *author = NULL; char *name = NULL; char *version = NULL; - char *url = NULL; char *json_url = NULL; char *repo = NULL; char *json = NULL; @@ -637,10 +594,6 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, goto error; if (!(version = parse_repo_version(slug, DEFAULT_REPO_VERSION))) goto error; - if (!(url = clib_package_url(author, name, version))) - goto error; - if (!(json_url = clib_package_file_url(url, file))) - goto error; _debug("author: %s", author); _debug("name: %s", name); @@ -674,15 +627,10 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, if (retries-- <= 0) { goto error; } else { -#ifdef HAVE_PTHREADS - init_curl_share(); - _debug("GET %s", json_url); + _debug("Fetching package manifest for %s", slug); // clean up when retrying - http_get_free(res); - res = http_get_shared(json_url, clib_package_curl_share, NULL, 0); -#else - res = http_get(json_url); -#endif + res = repository_fetch_package_manifest(url, slug, version); + json = res->data; _debug("status: %d", res->status); if (!res || !res->ok) { @@ -706,6 +654,9 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, pkg = clib_package_new(json, verbose); } + // Set the url so that we can download the other files. + pkg->url = url; + if (!pkg) goto error; @@ -740,20 +691,6 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, goto error; } - if (pkg->repo) { - if (0 != strcmp(repo, pkg->repo)) { - free(url); - if (!(url = clib_package_url_from_repo(pkg->repo, pkg->version))) - goto error; - } - free(repo); - repo = NULL; - } else { - pkg->repo = repo; - } - - pkg->url = url; - #ifdef HAVE_PTHREADS pthread_mutex_lock(&lock.mutex); #endif @@ -792,7 +729,6 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, free(author); free(name); free(version); - free(url); free(json_url); free(repo); if (!res && json) @@ -807,15 +743,14 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, /** * Create a package from the given repo `slug` */ - -clib_package_t *clib_package_new_from_slug(const char *slug, int verbose) { +clib_package_t *clib_package_new_from_slug_and_url(const char *slug, const char* url, int verbose) { clib_package_t *package = NULL; const char *name = NULL; unsigned int i = 0; do { name = manifest_names[i]; - package = clib_package_new_from_slug_with_package_name(slug, verbose, name); + package = clib_package_new_from_slug_with_package_name(slug, url, verbose, name); if (NULL != package) { package->filename = (char *)name; } @@ -824,63 +759,6 @@ clib_package_t *clib_package_new_from_slug(const char *slug, int verbose) { return package; } -/** - * Get a slug for the package `author/name@version` - */ - -char *clib_package_url(const char *author, const char *name, - const char *version) { - if (!author || !name || !version) - return NULL; - int size = strlen(GITHUB_CONTENT_URL) + strlen(author) + 1 // / - + strlen(name) + 1 // / - + strlen(version) + 1 // \0 - ; - - if (0 != opts.token) { - size += strlen(opts.token); - size += 1; // @ - } - - char *slug = malloc(size); - if (slug) { - memset(slug, '\0', size); - if (0 != opts.token) { - sprintf(slug, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s/%s", opts.token, - author, name, version); - } else { - sprintf(slug, GITHUB_CONTENT_URL "%s/%s/%s", author, name, version); - } - } - - return slug; -} - -char *clib_package_url_from_repo(const char *repo, const char *version) { - if (!repo || !version) - return NULL; - int size = strlen(GITHUB_CONTENT_URL) + strlen(repo) + 1 // / - + strlen(version) + 1 // \0 - ; - - if (0 != opts.token) { - size += strlen(opts.token); - size += 1; // @ - } - - char *slug = malloc(size); - if (slug) { - memset(slug, '\0', size); - if (0 != opts.token) { - sprintf(slug, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s", opts.token, repo, - version); - } else { - sprintf(slug, GITHUB_CONTENT_URL "%s/%s", repo, version); - } - } - return slug; -} - /** * Parse the package author from the given `slug` */ @@ -928,162 +806,6 @@ clib_package_dependency_t *clib_package_dependency_new(const char *repo, return dep; } -static int fetch_package_file_work(clib_package_t *pkg, const char *dir, - char *file, int verbose) { - char *url = NULL; - char *path = NULL; - int saved = 0; - int rc = 0; - - _debug("fetch file: %s/%s", pkg->repo, file); - - if (NULL == pkg) { - return 1; - } - - if (NULL == pkg->url) { - return 1; - } - - if (0 == strncmp(file, "http", 4)) { - url = strdup(file); - } else if (!(url = clib_package_file_url(pkg->url, file))) { - return 1; - } - - _debug("file URL: %s", url); - - if (!(path = path_join(dir, basename(file)))) { - rc = 1; - goto cleanup; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (1 == opts.force || -1 == fs_exists(path)) { - if (verbose) { - logger_info("fetch", "%s:%s", pkg->repo, file); - fflush(stdout); - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - - rc = http_get_file_shared(url, path, clib_package_curl_share); - saved = 1; - } else { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (-1 == rc) { - if (verbose) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - logger_error("error", "unable to fetch %s:%s", pkg->repo, file); - fflush(stderr); - rc = 1; -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto cleanup; - } - } - - if (saved) { - if (verbose) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - logger_info("save", path); - fflush(stdout); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - } - -cleanup: - - free(url); - free(path); - return rc; -} - -#ifdef HAVE_PTHREADS -static void *fetch_package_file_thread(void *arg) { - fetch_package_file_thread_data_t *data = arg; - int *status = malloc(sizeof(int)); - int rc = - fetch_package_file_work(data->pkg, data->dir, data->file, data->verbose); - *status = rc; - (void)data->pkg->refs--; - pthread_exit((void *)status); - return (void *)(intptr_t)rc; -} -#endif - -/** - * Fetch a file associated with the given `pkg`. - * - * Returns 0 on success. - */ - -static int fetch_package_file(clib_package_t *pkg, const char *dir, char *file, - int verbose, void **data) { -#ifndef HAVE_PTHREADS - return fetch_package_file_work(pkg, dir, file, verbose); -#else - fetch_package_file_thread_data_t *fetch = malloc(sizeof(*fetch)); - int rc = 0; - - if (0 == fetch) { - return -1; - } - - *data = 0; - - memset(fetch, 0, sizeof(*fetch)); - - fetch->pkg = pkg; - fetch->dir = dir; - fetch->file = file; - fetch->verbose = verbose; - - rc = pthread_attr_init(&fetch->attr); - - if (0 != rc) { - free(fetch); - return rc; - } - - (void)pkg->refs++; - rc = pthread_create(&fetch->thread, NULL, fetch_package_file_thread, fetch); - - if (0 != rc) { - pthread_attr_destroy(&fetch->attr); - free(fetch); - return rc; - } - - rc = pthread_attr_destroy(&fetch->attr); - - if (0 != rc) { - pthread_cancel(fetch->thread); - free(fetch); - return rc; - } - - *data = fetch; - - return rc; -#endif -} static void set_prefix(clib_package_t *pkg, long path_max) { if (NULL != opts.prefix || NULL != pkg->prefix) { @@ -1159,7 +881,7 @@ int clib_package_install_executable(clib_package_t *pkg, const char *dir, E_FORMAT(&tarball, "%s/%s", tmp, file); - rc = http_get_file_shared(url, tarball, clib_package_curl_share); + rc = http_get_file_shared(url, tarball, clib_package_curl_share, NULL, 0); if (0 != rc) { if (verbose) { @@ -1342,20 +1064,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { #endif } -#ifdef HAVE_PTHREADS - fetch_package_file_thread_data_t **fetchs = 0; - if (NULL != pkg && NULL != pkg->src) { - if (pkg->src->len > 0) { - fetchs = malloc(pkg->src->len * sizeof(fetch_package_file_thread_data_t)); - } - } - - if (fetchs) { - memset(fetchs, 0, pkg->src->len * sizeof(fetch_package_file_thread_data_t)); - } - -#endif - if (!pkg || !dir) { rc = -1; goto cleanup; @@ -1377,6 +1085,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { } } + /* if (NULL == pkg->url) { pkg->url = clib_package_url(pkg->author, pkg->repo_name, pkg->version); @@ -1385,6 +1094,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { goto cleanup; } } + */ // write clib.json or package.json if (!(package_json = path_join(pkg_dir, pkg->filename))) { @@ -1419,28 +1129,14 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { // fetch makefile if (!opts.global && pkg->makefile) { _debug("fetch: %s/%s", pkg->repo, pkg->makefile); - void *fetch = 0; - rc = fetch_package_file(pkg, pkg_dir, pkg->makefile, verbose, &fetch); - if (0 != rc) { + repository_file_handle_t handle = repository_download_package_file(pkg->url, pkg_dir, pkg->version, pkg->makefile, pkg_dir); + if (handle == NULL) { goto cleanup; } #ifdef HAVE_PTHREADS - if (0 != fetch) { - fetch_package_file_thread_data_t *data = fetch; - int *status; - pthread_join(data->thread, (void **)&status); - if (NULL != status) { - rc = *status; - free(status); - status = 0; - if (0 != rc) { - rc = 0; - logger_warn("warning", "unable to fetch Makefile (%s) for '%s'", - pkg->makefile, pkg->name); - } - } - } + repository_file_finish_download(handle); + repository_file_free(handle); #endif } @@ -1488,12 +1184,11 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { iterator = list_iterator_new(pkg->src, LIST_HEAD); list_node_t *source; - + repository_file_handle_t* handles = malloc(pkg->src->len*sizeof(repository_file_handle_t)); while ((source = list_iterator_next(iterator))) { - void *fetch = NULL; - rc = fetch_package_file(pkg, pkg_dir, source->val, verbose, &fetch); + handles[i] = repository_download_package_file(pkg->url, pkg_dir, pkg->version, source->val, pkg_dir); - if (0 != rc) { + if (handles[i] == NULL) { list_iterator_destroy(iterator); iterator = NULL; rc = -1; @@ -1505,32 +1200,19 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { i = 0; } - fetchs[i] = fetch; - (void)pending++; if (i < (max - 1)) { (void)i++; } else { - for (int j = 0; j <= i; j++) { - fetch_package_file_thread_data_t *data = fetchs[j]; - int *status; - pthread_join(data->thread, (void **)&status); - free(data); - fetchs[j] = NULL; - + while (--i >= 0) { + repository_file_finish_download(handles[i]); + repository_file_free(handles[i]); (void)pending--; - if (NULL != status) { - rc = *status; - free(status); - status = 0; - } - - if (0 != rc) { - rc = -1; - goto cleanup; - } +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) + usleep(1024 * 10); +#endif } i = 0; } @@ -1538,27 +1220,11 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { } #ifdef HAVE_PTHREADS - // Here there are i-1 threads running. - for (int j = 0; j < i; j++) { - fetch_package_file_thread_data_t *data = fetchs[j]; - int *status; - - pthread_join(data->thread, (void **)&status); + while (--i >= 0) { + repository_file_finish_download(handles[i]); (void)pending--; - free(data); - fetchs[j] = NULL; - - if (NULL != status) { - rc = *status; - free(status); - status = 0; - } - - if (0 != rc) { - rc = -1; - goto cleanup; - } + repository_file_free(handles[i]); } #endif @@ -1598,16 +1264,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { list_iterator_destroy(iterator); if (command) free(command); -#ifdef HAVE_PTHREADS - if (NULL != pkg && NULL != pkg->src) { - if (pkg->src->len > 0) { - if (fetchs) { - free(fetchs); - } - } - } - fetchs = NULL; -#endif return rc; } diff --git a/src/common/clib-package.h b/src/common/clib-package.h index dab17fcb..35b50d08 100644 --- a/src/common/clib-package.h +++ b/src/common/clib-package.h @@ -57,7 +57,7 @@ void clib_package_set_opts(clib_package_opts_t opts); clib_package_t *clib_package_new(const char *, int); -clib_package_t *clib_package_new_from_slug(const char *, int); +clib_package_t *clib_package_new_from_slug_and_url(const char* slug, const char* url, int); clib_package_t *clib_package_load_from_manifest(const char *, int); diff --git a/src/common/clib-secrets.c b/src/common/clib-secrets.c new file mode 100644 index 00000000..c667be3c --- /dev/null +++ b/src/common/clib-secrets.c @@ -0,0 +1,81 @@ +#include "clib-secrets.h" +#include "fs/fs.h" +#include "list/list.h" +#include "logger/logger.h" +#include "parson/parson.h" +#include +#include +#include + +struct clib_secret { + char *hostname; + char *secret; +}; + +struct clib_secret_handle { + list_t *secrets; +}; + +clib_secrets_t clib_secrets_load_from_file(const char *file) { + if (-1 == fs_exists(file)) { + logger_error("error", "Secrets file %s does not exist.", file); + return NULL; + } + + //logger_info("info", "Reading secrets from %s.", file); + + char* json = NULL; + json = fs_read(file); + if (NULL == json) { + return NULL; + } + + JSON_Value *root = json_parse_string(json); + if (root == NULL) { + logger_error("error", "unable to parse JSON"); + return NULL; + } + + JSON_Object *json_object = NULL; + if (!(json_object = json_value_get_object(root))) { + logger_error("error", "Invalid json file, root is not an object."); + return NULL; + } + + clib_secrets_t handle = malloc(sizeof(struct clib_secret_handle)); + + if (!(handle->secrets = list_new())) { + free(json); + free(handle); + + return NULL; + } + + for (unsigned int i = 0; i < json_object_get_count(json_object); i++) { + char *domain = json_object_get_name(json_object, i); + char *secret = json_object_get_string(json_object, domain); + + struct clib_secret *secret_struct = malloc(sizeof(struct clib_secret)); + secret_struct->hostname = strdup(domain); + secret_struct->secret = strdup(secret); + + list_rpush(handle->secrets, list_node_new(secret_struct)); + } + + return handle; +} + +char *clib_secret_find_for_hostname(clib_secrets_t secrets, const char *hostname) { + list_iterator_t *iterator = list_iterator_new(secrets->secrets, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(iterator))) { + struct clib_secret *secret = node->val; + if (strcmp(hostname, secret->hostname) == 0) { + list_iterator_destroy(iterator); + return secret->secret; + } + } + + list_iterator_destroy(iterator); + return NULL; +} diff --git a/src/common/clib-secrets.h b/src/common/clib-secrets.h new file mode 100644 index 00000000..c788302e --- /dev/null +++ b/src/common/clib-secrets.h @@ -0,0 +1,10 @@ +#ifndef CLIB_SRC_COMMON_CLIB_SECRETS_H +#define CLIB_SRC_COMMON_CLIB_SECRETS_H + +typedef struct clib_secret_handle* clib_secrets_t; + +clib_secrets_t clib_secrets_load_from_file(const char* file); + +char* clib_secret_find_for_hostname(clib_secrets_t secrets, const char* hostname); + +#endif//CLIB_SRC_COMMON_CLIB_SECRETS_H diff --git a/src/common/url.c b/src/common/url.c new file mode 100644 index 00000000..797b7997 --- /dev/null +++ b/src/common/url.c @@ -0,0 +1,5 @@ +#define URL_H_IMPLEMENTATION +#include "url/url.h" + +// This is a bit of a hack to use url.h in multiple files. +// It is better if we just split url.h \ No newline at end of file diff --git a/src/registry/github-registry.c b/src/registry/github-registry.c index 037a570d..eb431f7a 100644 --- a/src/registry/github-registry.c +++ b/src/registry/github-registry.c @@ -1,38 +1,38 @@ // // github-registry.c // -// Copyright (c) 2020 Elbert van de Put +// Copyright (c) 2021 Elbert van de Put // Based on work by Stephen Mathieson // MIT licensed // -#include #include "github-registry.h" -#include "gumbo-text-content/gumbo-text-content.h" +#include "case/case.h" #include "gumbo-get-element-by-id/get-element-by-id.h" #include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" +#include "gumbo-text-content/gumbo-text-content.h" #include "http-get/http-get.h" -#include -#include "substr/substr.h" +#include "registry-internal.h" #include "strdup/strdup.h" -#include "case/case.h" +#include "substr/substr.h" #include "trim/trim.h" -#include "wiki-registry-internal.h" +#include +#include /** * Add `href` to the given `package`. */ -static void add_package_href(wiki_package_ptr_t self) { - size_t len = strlen(self->repo) + 20; // https://github.com/ \0 +static void add_package_href(registry_package_ptr_t self) { + size_t len = strlen(self->id) + 20; // https://github.com/ \0 self->href = malloc(len); if (self->href) - sprintf(self->href, "https://github.com/%s", self->repo); + sprintf(self->href, "https://github.com/%s", self->id); } /** * Parse the given wiki `li` into a package. */ -static wiki_package_ptr_t parse_li(GumboNode *li) { - wiki_package_ptr_t self = wiki_package_new(); +static registry_package_ptr_t parse_li(GumboNode *li) { + registry_package_ptr_t self = registry_package_new(); char *text = NULL; if (!self) goto cleanup; @@ -45,11 +45,11 @@ static wiki_package_ptr_t parse_li(GumboNode *li) { if (!tok) goto cleanup; int pos = tok - text; - self->repo = substr(text, 0, pos); + self->id = substr(text, 0, pos); self->description = substr(text, pos + 3, -1); - if (!self->repo || !self->description) goto cleanup; + if (!self->id || !self->description) goto cleanup; trim(self->description); - trim(self->repo); + trim(self->id); add_package_href(self); @@ -96,13 +96,14 @@ list_t *wiki_registry_parse(const char *html) { list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); list_node_t *li_node; while ((li_node = list_iterator_next(li_iterator))) { - wiki_package_ptr_t package = parse_li(li_node->val); + registry_package_ptr_t package = parse_li(li_node->val); if (package && package->description) { package->category = strdup(category); list_rpush(pkgs, list_node_new(package)); } else { // failed to parse package - if (package) wiki_package_free(package); + if (package) + registry_package_free(package); } } list_iterator_destroy(li_iterator); diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c index 446fd9e6..2c899f8a 100644 --- a/src/registry/gitlab-registry.c +++ b/src/registry/gitlab-registry.c @@ -1,167 +1,76 @@ // // gitlab-registry.c // -// Copyright (c) 2020 Elbert van de Put +// Copyright (c) 2021 Elbert van de Put // MIT licensed // #include "gitlab-registry.h" -#include -#include "gumbo-text-content/gumbo-text-content.h" #include "gumbo-get-element-by-id/get-element-by-id.h" -#include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" #include "http-get/http-get.h" +#include "registry-internal.h" #include -#include "substr/substr.h" -#include "strdup/strdup.h" -#include "case/case.h" -#include "trim/trim.h" -#include "wiki-registry-internal.h" - -/** - * Add `href` to the given `package`. - * We assume that all packages listed by a registry live on the same platform as the registry. - */ -static void add_package_href(wiki_package_ptr_t self, const char* hostname) { - size_t len = strlen(self->repo) + strlen(hostname); - self->href = malloc(len); - if (self->href) - sprintf(self->href, "https://%s/%s", hostname, self->repo); -} - -/** - * Parse the given wiki `li` into a package. - */ -static wiki_package_ptr_t parse_li(GumboNode *li, const char* hostname) { - wiki_package_ptr_t self = wiki_package_new(); - char *text = NULL; - - if (!self) goto cleanup; - - text = gumbo_text_content(li); - if (!text) goto cleanup; - - // TODO support unicode dashes - char *tok = strstr(text, " - "); - if (!tok) goto cleanup; - - int pos = tok - text; - self->repo = substr(text, 0, pos); - self->description = substr(text, pos + 3, -1); - if (!self->repo || !self->description) goto cleanup; - trim(self->description); - trim(self->repo); - - add_package_href(self, hostname); - - cleanup: - free(text); - return self; -} +#include /** * Parse a list of packages from the given `html` */ -static list_t *wiki_registry_parse(const char* hostname, const char *html) { - list_t *pkgs = list_new(); - - // Try to parse the markdown file. - char* input = strdup(html); - char* line; - char* category = NULL; - while ((line = strsep(&input, "\n"))) { - char* dash_position = strstr(line, "-"); - // The line starts with a dash, so we expect a package. - if (dash_position != NULL && dash_position - line < 4) { - - char* link_name_start = strstr(line, "[")+1; - char* link_name_end = strstr(link_name_start, "]"); - char* link_value_start = strstr(link_name_end, "(")+1; - char* link_value_end = strstr(link_value_start, ")"); - char* description_position = strstr(link_value_end, "-")+1; - - wiki_package_ptr_t package = wiki_package_new(); - package->href = strndup(link_value_start, link_value_end-link_value_start); - package->repo = strndup(link_name_start, link_name_end-link_name_start); - package->description = strdup(description_position); - package->category = strdup(category != NULL ? category: "unknown"); - list_rpush(pkgs, list_node_new(package)); - } - - char* header_position = strstr(line, "##"); - // The category starts with a ##. - if (header_position != NULL && header_position - line < 4) { - category = header_position+2; - } +static list_t *gitlab_registry_parse(const char *hostname, const char *html) { + list_t *pkgs = list_new(); + + // Try to parse the markdown file. + char *input = strdup(html); + char *line; + char *category = NULL; + while ((line = strsep(&input, "\n"))) { + char *dash_position = strstr(line, "-"); + // The line starts with a dash, so we expect a package. + if (dash_position != NULL && dash_position - line < 4) { + + char *link_name_start = strstr(line, "[") + 1; + char *link_name_end = strstr(link_name_start, "]"); + char *link_value_start = strstr(link_name_end, "(") + 1; + char *link_value_end = strstr(link_value_start, ")"); + char *description_position = strstr(link_value_end, "-") + 1; + + registry_package_ptr_t package = registry_package_new(); + package->href = strndup(link_value_start, link_value_end - link_value_start); + package->id = strndup(link_name_start, link_name_end - link_name_start); + package->description = strdup(description_position); + package->category = strdup(category != NULL ? category : "unknown"); + list_rpush(pkgs, list_node_new(package)); } - free(input); - - return pkgs; - - GumboOutput *output = gumbo_parse(html); - GumboNode *body = gumbo_get_element_by_id("content-body", output->root); - if (body) { - // grab all category `

`s - list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); - list_node_t *heading_node; - list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); - while ((heading_node = list_iterator_next(heading_iterator))) { - GumboNode *heading = (GumboNode *) heading_node->val; - char *category = gumbo_text_content(heading); - // die if we failed to parse a category, as it's - // almost certinaly a malloc error - if (!category) break; - trim(case_lower(category)); - GumboVector *siblings = &heading->parent->v.element.children; - size_t pos = heading->index_within_parent; - - // skip elements until the UL - // TODO: don't hardcode position here - // 2: - // 1 - whitespace - // 2 - actual node - GumboNode *ul = siblings->data[pos + 2]; - if (GUMBO_TAG_UL != ul->v.element.tag) { - free(category); - continue; - } - - list_t *lis = gumbo_get_elements_by_tag_name("li", ul); - list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); - list_node_t *li_node; - while ((li_node = list_iterator_next(li_iterator))) { - wiki_package_ptr_t package = parse_li(li_node->val, hostname); - if (package && package->description) { - package->category = strdup(category); - list_rpush(pkgs, list_node_new(package)); - } else { - // failed to parse package - if (package) wiki_package_free(package); - } - } - list_iterator_destroy(li_iterator); - list_destroy(lis); - free(category); - } - list_iterator_destroy(heading_iterator); - list_destroy(h2s); + char *header_position = strstr(line, "##"); + // The category starts with a ##. + if (header_position != NULL && header_position - line < 4) { + category = header_position + 2; } + } - gumbo_destroy_output(&kGumboDefaultOptions, output); - return pkgs; + free(input); + + return pkgs; } /** - * Get a list of packages from the given gitlab wiki `url`. - * TODO, get the secret from a secrets file. + * Get a list of packages from the given gitlab file `url`. */ -list_t *gitlab_registry_fetch(const char *url, const char* hostname) { - char* headers[1] = {"PRIVATE-TOKEN: SECRET"}; - http_get_response_t *res = http_get(url, headers, 1); - if (!res->ok) return NULL; - - list_t *list = wiki_registry_parse(hostname, res->data); - http_get_free(res); - return list; +list_t *gitlab_registry_fetch(const char *url, const char *hostname, const char *secret) { + http_get_response_t *res; + if (secret == NULL) { + return NULL; + } + + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + res = http_get(url, &authentication_header, 1); + if (!res->ok) { + return NULL; + } + + list_t *list = gitlab_registry_parse(hostname, res->data); + http_get_free(res); + return list; } - diff --git a/src/registry/gitlab-registry.h b/src/registry/gitlab-registry.h index 38d7de00..9075736e 100644 --- a/src/registry/gitlab-registry.h +++ b/src/registry/gitlab-registry.h @@ -3,6 +3,6 @@ #include "list/list.h" -list_t* gitlab_registry_fetch(const char* url, const char* hostname); +list_t* gitlab_registry_fetch(const char* url, const char* hostname, const char* secret); #endif //CLIB_GITLAB_REGISTRY_H diff --git a/src/registry/registry-internal.h b/src/registry/registry-internal.h new file mode 100644 index 00000000..770fba06 --- /dev/null +++ b/src/registry/registry-internal.h @@ -0,0 +1,23 @@ +// +// registry-internal.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +// DO NOT INCLUDE. THIS HEADER IS INTERNAL ONLY +#ifndef REGISTRY_INTERNAL_H +#define REGISTRY_INTERNAL_H +#include "registry.h" + +struct registry_package_t { + char *id; + char *href; + char *description; + char *category; +}; + +registry_package_ptr_t registry_package_new(); + +void registry_package_free(registry_package_ptr_t pkg); + +#endif diff --git a/src/registry/registry-manager.c b/src/registry/registry-manager.c new file mode 100644 index 00000000..d178e5e4 --- /dev/null +++ b/src/registry/registry-manager.c @@ -0,0 +1,78 @@ +// +// registry-manager.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "registry-manager.h" +#include +#include "url/url.h" + +#define CLIB_WIKI_URL "https://github.com/clibs/clib/wiki/Packages" + +registries_t registry_manager_init_registries(list_t* registry_urls, clib_secrets_t secrets) { + list_t* registries = list_new(); + + // Add all the registries that were provided. + list_iterator_t *registry_iterator = list_iterator_new(registry_urls, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(registry_iterator))) { + char* url = node->val; + url_data_t *parsed = url_parse(url); + char* hostname = strdup(parsed->hostname); + url_free(parsed); + char* secret = clib_secret_find_for_hostname(secrets, hostname); + registry_ptr_t registry = registry_create(url, secret); + list_rpush(registries, list_node_new(registry)); + } + list_iterator_destroy(registry_iterator); + + // And add the default registry. + registry_ptr_t registry = registry_create(CLIB_WIKI_URL, NULL); + list_rpush(registries, list_node_new(registry)); + + return registries; +} + +void registry_manager_fetch_registries(registries_t registries) { + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t reg; + while ((reg = registry_iterator_next(it))) { + if (!registry_fetch(reg)) { + printf("REGISTRY: could not list packages from. %s\n", registry_get_url(reg)); + } + } + registry_iterator_destroy(it); +} + +registry_package_ptr_t registry_manger_find_package(registries_t registries, const char* package_id) { + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t reg; + while ((reg = registry_iterator_next(it))) { + registry_package_ptr_t package = registry_find_package(reg, package_id); + if (package != NULL) { + registry_iterator_destroy(it); + return package; + } + } + registry_iterator_destroy(it); + + return NULL; +} + +registry_iterator_t registry_iterator_new(registries_t registries) { + return list_iterator_new(registries, LIST_HEAD); +} + +registry_ptr_t registry_iterator_next(registry_iterator_t iterator) { + list_node_t *node = list_iterator_next(iterator); + if (node == NULL) { + return NULL; + } + + return (registry_ptr_t) node->val; +} + +void registry_iterator_destroy(registry_iterator_t iterator) { + list_iterator_destroy(iterator); +} diff --git a/src/registry/registry-manager.h b/src/registry/registry-manager.h new file mode 100644 index 00000000..f717ca25 --- /dev/null +++ b/src/registry/registry-manager.h @@ -0,0 +1,44 @@ +// +// registry-manager.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H +#define CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H + +#include "clib-secrets.h" +#include "list/list.h" +#include "registry.h" + +// Contains an abstraction for handling multiple registries + +typedef list_t* registries_t; +typedef list_iterator_t* registry_iterator_t; + +/** + * Initializes all registies specified by the urls + * @param registry_urls + * @param secrets + * @return + */ +registries_t registry_manager_init_registries(list_t* registry_urls, clib_secrets_t secrets); + +void registry_manager_fetch_registries(registries_t registries); + +/** + * An iterator through the registries. + */ +registry_iterator_t registry_iterator_new(registries_t registry); +registry_ptr_t registry_iterator_next(registry_iterator_t iterator); +void registry_iterator_destroy(registry_iterator_t iterator); + +/** + * Search the registry for a package + * @param registry a registry handle + * @param package_id the identifier of the package "/" + * @return a pointer to the package if it could be found or NULL + */ +registry_package_ptr_t registry_manger_find_package(registries_t registries, const char* package_id); + +#endif//CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H diff --git a/src/registry/registry.c b/src/registry/registry.c new file mode 100644 index 00000000..238be3ea --- /dev/null +++ b/src/registry/registry.c @@ -0,0 +1,160 @@ +// +// registry.c +// +// Copyright (c) 2020 clib authors +// MIT licensed +// + +#include "github-registry.h" +#include "gitlab-registry.h" +#include "gumbo-parser/gumbo.h" +#include "list/list.h" +#include "registry-internal.h" +#include "url/url.h" +#include +#include + +enum registry_type_t { + REGISTRY_TYPE_GITHUB, + REGISTRY_TYPE_GITLAB, +}; + +struct registry_t { + enum registry_type_t type; + char *url; + char *hostname; + char *secret; + list_t *packages; +}; + +/** + * Create a new registry package. + */ +registry_package_ptr_t registry_package_new() { + registry_package_ptr_t pkg = malloc(sizeof(struct registry_package_t)); + if (pkg) { + pkg->id = NULL; + pkg->href = NULL; + pkg->description = NULL; + pkg->category = NULL; + } + return pkg; +} + +/** + * Release the memory held by the package. + */ +void registry_package_free(registry_package_ptr_t pkg) { + free(pkg->id); + free(pkg->href); + free(pkg->description); + free(pkg->category); + free(pkg); +} + +registry_ptr_t registry_create(const char *url, const char *secret) { + registry_ptr_t registry = malloc(sizeof(struct registry_t)); + registry->url = strdup(url); + registry->secret = strdup(secret); + + if (strstr(url, "github.com") != NULL) { + registry->type = REGISTRY_TYPE_GITHUB; + } else if (strstr(url, "gitlab") != NULL) { + registry->type = REGISTRY_TYPE_GITLAB; + } else { + return NULL; + } + + url_data_t *parsed = url_parse(url); + registry->hostname = strdup(parsed->hostname); + url_free(parsed); + + return registry; +} + +void registry_free(registry_ptr_t registry) { + free(registry->url); + free(registry->hostname); + if (registry->packages != NULL) { + list_iterator_t *it = list_iterator_new(registry->packages, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(it))) { + registry_package_free(node->val); + } + list_iterator_destroy(it); + list_destroy(registry->packages); + } + free(registry); +} + +const char *registry_get_url(registry_ptr_t registry) { + return registry->url; +} + +bool registry_fetch(registry_ptr_t registry) { + switch (registry->type) { + case REGISTRY_TYPE_GITLAB: + registry->packages = gitlab_registry_fetch(registry->url, registry->hostname, registry->secret); + if (registry->packages != NULL) { + return true; + } + break; + case REGISTRY_TYPE_GITHUB: + registry->packages = github_registry_fetch(registry->url); + if (registry->packages != NULL) { + return true; + } + break; + default: + return false; + } + + return false; +} + +registry_package_iterator_t registry_package_iterator_new(registry_ptr_t registry) { + return list_iterator_new(registry->packages, LIST_HEAD); +} + +registry_package_ptr_t registry_package_iterator_next(registry_package_iterator_t iterator) { + list_node_t *node = list_iterator_next(iterator); + if (node == NULL) { + return NULL; + } + + return (registry_package_ptr_t) node->val; +} + +void registry_package_iterator_destroy(registry_package_iterator_t iterator) { + list_iterator_destroy(iterator); +} + +registry_package_ptr_t registry_find_package(registry_ptr_t registry, const char *package_id) { + registry_package_iterator_t it = registry_package_iterator_new(registry); + registry_package_ptr_t pack; + while ((pack = registry_package_iterator_next(it))) { + if (0 == strcmp(package_id, pack->id)) { + registry_package_iterator_destroy(it); + return pack; + } + } + registry_package_iterator_destroy(it); + + return NULL; +} + +char *registry_package_get_id(registry_package_ptr_t package) { + return package->id; +} + +char *registry_package_get_href(registry_package_ptr_t package) { + return package->href; +} + +char *registry_package_get_description(registry_package_ptr_t package) { + return package->description; +} + +char *registry_package_get_category(registry_package_ptr_t package) { + return package->category; +} \ No newline at end of file diff --git a/src/registry/registry.h b/src/registry/registry.h new file mode 100644 index 00000000..531c0afa --- /dev/null +++ b/src/registry/registry.h @@ -0,0 +1,64 @@ +// +// registry.h +// +// Copyright (c) 2020 clib authors +// MIT licensed +// + +#ifndef REGISTRY_H +#define REGISTRY_H 1 + +#include + +typedef struct registry_package_t * registry_package_ptr_t; +typedef struct registry_t * registry_ptr_t; +typedef list_iterator_t* registry_package_iterator_t; + +/** + * Create a new registry for the given url. + * @param url the url of the registry. + * @param secret the secret to authenticate with this registry of NULL if no secret is required. + * @return a handle to the registry + */ +registry_ptr_t registry_create(const char* url, const char* secret); + +/** + * Free the memory held by the registry. + * @param registry + */ +void registry_free(registry_ptr_t registry); + +/** + * Fetch the list of packages from the registry. + * @param registry + */ +bool registry_fetch(registry_ptr_t registry); + +/** + * Get the url for the registry + * @param registry + * @return + */ +const char* registry_get_url(registry_ptr_t registry); + +/** + * An iterator through the packages in the registry. + */ +registry_package_iterator_t registry_package_iterator_new(registry_ptr_t registry); +registry_package_ptr_t registry_package_iterator_next(registry_package_iterator_t iterator); +void registry_package_iterator_destroy(registry_package_iterator_t iterator); + +/** + * Search the registry for a package + * @param registry a registry handle + * @param package_id the identifier of the package "/" + * @return a pointer to the package if it could be found or NULL + */ +registry_package_ptr_t registry_find_package(registry_ptr_t registry, const char* package_id); + +char* registry_package_get_id(registry_package_ptr_t package); +char* registry_package_get_href(registry_package_ptr_t package); +char* registry_package_get_description(registry_package_ptr_t package); +char* registry_package_get_category(registry_package_ptr_t package); + +#endif \ No newline at end of file diff --git a/src/registry/wiki-registry-internal.h b/src/registry/wiki-registry-internal.h deleted file mode 100644 index 62f94f10..00000000 --- a/src/registry/wiki-registry-internal.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef WIKI_REGISTRY_HELPER_H -#define WIKI_REGISTRY_HELPER_H -// DO NOT INCLUDE. THIS HEADER IS INTERNAL ONLY -#include "wiki-registry.h" - -struct wiki_package_t { - char *repo; - char *href; - char *description; - char *category; -}; - -wiki_package_ptr_t wiki_package_new(); - -void wiki_package_free(wiki_package_ptr_t pkg); - -#endif //WIKI_REGISTRY_HELPER_H diff --git a/src/registry/wiki-registry.c b/src/registry/wiki-registry.c deleted file mode 100644 index 2c842132..00000000 --- a/src/registry/wiki-registry.c +++ /dev/null @@ -1,147 +0,0 @@ - -// -// wiki-registry.c -// -// Copyright (c) 2014 Stephen Mathieson -// MIT licensed -// - -#include -#include -#include "strdup/strdup.h" -#include "gumbo-parser/gumbo.h" -#include "list/list.h" -#include "wiki-registry-internal.h" -#include "github-registry.h" -#include "gitlab-registry.h" -#include "url/url.h" - -enum wiki_registry_type_t { - REGISTRY_TYPE_GITHUB, - REGISTRY_TYPE_GITLAB, -}; - -struct wiki_registry_t { - enum wiki_registry_type_t type; - char* url; - char* hostname; - list_t *packages; -}; - -/** - * Create a new wiki package. - */ -wiki_package_ptr_t wiki_package_new() { - wiki_package_ptr_t pkg = malloc(sizeof(struct wiki_package_t)); - if (pkg) { - pkg->repo = NULL; - pkg->href = NULL; - pkg->description = NULL; - pkg->category = NULL; - } - return pkg; -} - -/** - * Free a wiki_package_t. - */ -void wiki_package_free(wiki_package_ptr_t pkg) { - free(pkg->repo); - free(pkg->href); - free(pkg->description); - free(pkg->category); - free(pkg); -} - -wiki_registry_ptr_t wiki_registry_create(const char *url) { - wiki_registry_ptr_t registry = malloc(sizeof(struct wiki_registry_t)); - registry->url = strdup(url); - - if (strstr(url, "github.com") != NULL) { - registry->type = REGISTRY_TYPE_GITHUB; - } else if (strstr(url, "gitlab") != NULL) { - registry->type = REGISTRY_TYPE_GITLAB; - } else { - return NULL; - } - - url_data_t *parsed = url_parse(url); - registry->hostname = strdup(parsed->hostname); - url_free(parsed); - - return registry; -} - -void wiki_registry_free(wiki_registry_ptr_t registry) { - free(registry->url); - free(registry->hostname); - if (registry->packages != NULL) { - list_iterator_t* it = list_iterator_new(registry->packages, LIST_HEAD); - list_node_t* node; - while ((node = list_iterator_next(it))) { - wiki_package_free(node->val); - } - list_iterator_destroy(it); - list_destroy(registry->packages); - } - free(registry); -} - -const char* wiki_registry_get_url(wiki_registry_ptr_t registry) { - return registry->url; -} - -bool wiki_registry_fetch(wiki_registry_ptr_t registry) { - switch (registry->type) { - case REGISTRY_TYPE_GITLAB: - registry->packages = gitlab_registry_fetch(registry->url, registry->hostname); - if (registry->packages != NULL) { - return true; - } - break; - case REGISTRY_TYPE_GITHUB: - registry->packages = github_registry_fetch(registry->url); - if (registry->packages != NULL) { - return true; - } - break; - default: - return false; - } - - return false; -} - - -wiki_registry_iterator_t wiki_registry_iterator_new(wiki_registry_ptr_t registry) { - return list_iterator_new(registry->packages, LIST_HEAD); -} - -wiki_package_ptr_t wiki_registry_iterator_next(wiki_registry_iterator_t iterator) { - list_node_t *node = list_iterator_next(iterator); - if (node == NULL) { - return NULL; - } - - return (wiki_package_ptr_t )node->val; -} - -void wiki_registry_iterator_destroy(wiki_registry_iterator_t iterator) { - list_iterator_destroy(iterator); -} - -char* wiki_package_get_repo(wiki_package_ptr_t package) { - return package->repo; -} - -char* wiki_package_get_href(wiki_package_ptr_t package) { - return package->href; -} - -char* wiki_package_get_description(wiki_package_ptr_t package) { - return package->description; -} - -char* wiki_package_get_category(wiki_package_ptr_t package) { - return package->category; -} \ No newline at end of file diff --git a/src/registry/wiki-registry.h b/src/registry/wiki-registry.h deleted file mode 100644 index 695e216e..00000000 --- a/src/registry/wiki-registry.h +++ /dev/null @@ -1,55 +0,0 @@ - -// -// wiki-registry.h -// -// Copyright (c) 2013 Stephen Mathieson -// MIT licensed -// - - -#ifndef WIKI_REGISTRY_H -#define WIKI_REGISTRY_H 1 - -typedef struct wiki_package_t* wiki_package_ptr_t; -typedef struct wiki_registry_t* wiki_registry_ptr_t; -typedef list_iterator_t* wiki_registry_iterator_t; - -/** - * Create a new wiki_registry for the given url. - * @param url - * @return - */ -wiki_registry_ptr_t wiki_registry_create(const char* url); - -/** - * Free the memory held by the registry. - * @param registry - */ -void wiki_registry_free(wiki_registry_ptr_t registry); - -/** - * Fetch the list of packages from the registry. - * @param registry - */ -bool wiki_registry_fetch(wiki_registry_ptr_t registry); - -/** - * Get the url for the registry - * @param registry - * @return - */ -const char* wiki_registry_get_url(wiki_registry_ptr_t registry); - -/** - * An iterator through the packages in the registry. - */ -wiki_registry_iterator_t wiki_registry_iterator_new(wiki_registry_ptr_t registry); -wiki_package_ptr_t wiki_registry_iterator_next(wiki_registry_iterator_t iterator); -void wiki_registry_iterator_destroy(wiki_registry_iterator_t iterator); - -char* wiki_package_get_repo(wiki_package_ptr_t package); -char* wiki_package_get_href(wiki_package_ptr_t package); -char* wiki_package_get_description(wiki_package_ptr_t package); -char* wiki_package_get_category(wiki_package_ptr_t package); - -#endif diff --git a/src/repository/github-repository.c b/src/repository/github-repository.c new file mode 100644 index 00000000..be72990f --- /dev/null +++ b/src/repository/github-repository.c @@ -0,0 +1,37 @@ +// +// github-repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "github-repository.h" +#include +#include +#include + +#define GITHUB_CONTENT_URL "https://raw.githubusercontent.com/" +#define GITHUB_CONTENT_URL_WITH_TOKEN "https://%s@raw.githubusercontent.com/" + +char* github_repository_get_url_for_file(const char* hostname, const char* slug, const char* version, const char *file_path, const char* secret) { + + int size = strlen(GITHUB_CONTENT_URL) + strlen(slug) + 1 // / + + strlen(version) + 1 // \0 + ; + + if (secret != NULL) { + size += strlen(secret); + size += 1; // @ + } + + char *url = malloc(size); + if (url) { + memset(url, '\0', size); + if (secret != NULL) { + sprintf(url, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s", secret, slug, version); + } else { + sprintf(url, GITHUB_CONTENT_URL "%s/%s", slug, version); + } + } + + return url; +} diff --git a/src/repository/github-repository.h b/src/repository/github-repository.h new file mode 100644 index 00000000..6a996c23 --- /dev/null +++ b/src/repository/github-repository.h @@ -0,0 +1,12 @@ +// +// github-repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H + +char* github_repository_get_url_for_file(const char* hostname, const char* slug, const char* version, const char *file, const char* secret); + +#endif//CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H diff --git a/src/repository/gitlab-repository.c b/src/repository/gitlab-repository.c new file mode 100644 index 00000000..fba873cc --- /dev/null +++ b/src/repository/gitlab-repository.c @@ -0,0 +1,26 @@ +// +// gitlab-repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "gitlab-repository.h" +#include +#include +#include +#include + +// GET :hostname/api/v4/projects/:id/repository/files/:file_path/raw +char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret) { + url_data_t *parsed = url_parse(package_url); + + int size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(file) + 64; + char *url = malloc(size); + if (url) { + snprintf(url, size, "https://%s/api/v4%s/repository/files/%s/raw?ref=master", parsed->hostname, parsed->pathname, file); + } + + url_free(parsed); + + return url; +} diff --git a/src/repository/gitlab-repository.h b/src/repository/gitlab-repository.h new file mode 100644 index 00000000..2adaaae3 --- /dev/null +++ b/src/repository/gitlab-repository.h @@ -0,0 +1,12 @@ +// +// gitlab-repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H + +char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret); + +#endif//CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H diff --git a/src/repository/repository.c b/src/repository/repository.c new file mode 100644 index 00000000..c922c34a --- /dev/null +++ b/src/repository/repository.c @@ -0,0 +1,264 @@ +// +// repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "repository.h" +#include + +#include "debug/debug.h" +#include "github-repository.h" +#include "gitlab-repository.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static debug_t _debugger; +#define _debug(...) \ + ({ \ + if (!(_debugger.name)) \ + debug_init(&_debugger, "clib-repository"); \ + debug(&_debugger, __VA_ARGS__); \ + }) + +struct repository_file_t { + const char *url; + const char *dir; + const char *file; + const char *secret; + pthread_t thread; + pthread_attr_t attr; + void *data; +}; + +static pthread_mutex_t mutex; + +static clib_secrets_t secrets; + +static int fetch_package_file(const char *url, const char *dir, const char *file, const char *secret, repository_file_handle_t *thread_data_ptr); + +static void curl_lock_callback(CURL *handle, curl_lock_data data, curl_lock_access access, void *userptr) { + pthread_mutex_lock(&mutex); +} + +static void curl_unlock_callback(CURL *handle, curl_lock_data data, curl_lock_access access, void *userptr) { + pthread_mutex_unlock(&mutex); +} + +static void init_curl_share() { + if (0 == clib_package_curl_share) { + pthread_mutex_lock(&mutex); + clib_package_curl_share = curl_share_init(); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_LOCKFUNC, curl_lock_callback); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback); + curl_share_setopt(clib_package_curl_share, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + pthread_mutex_unlock(&mutex); + } +} + +void repository_init(clib_secrets_t _secrets) { + secrets = _secrets; +} + +static char *repository_create_url_for_file(const char *package_url, const char *slug, const char *version, const char *file_path, const char *secret) { + if (strstr(package_url, "github.com") != NULL) { + return github_repository_get_url_for_file(package_url, slug, version, file_path, secret); + } else if (strstr(package_url, "gitlab") != NULL) { + return gitlab_repository_get_url_for_file(package_url, slug, version, file_path, secret); + } else { + return NULL; + } +} + +http_get_response_t *repository_fetch_package_manifest(const char *package_url, const char *package_id, const char *version) { + init_curl_share(); + + // Check if there is a secret for the requested repository. + url_data_t *parsed = url_parse(package_url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + char *manifest_name = "package.json"; + char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_name, secret); + + http_get_response_t *res; + if (strstr(package_url, "gitlab") != NULL) { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + + res = http_get(manifest_url, &authentication_header, 1); + } else { + res = http_get(manifest_url, NULL, 0); + } + + return res; +} + +repository_file_handle_t repository_download_package_file(const char *package_url, const char *package_id, const char *version, const char *file_path, const char *destination_path) { + init_curl_share(); + + // Check if there is a secret for the requested repository. + url_data_t *parsed = url_parse(package_url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + char *url = repository_create_url_for_file(package_url, package_id, version, file_path, secret); + + repository_file_handle_t handle; + fetch_package_file(url, destination_path, file_path, secret, &handle); + + return handle; +} + +void repository_file_finish_download(repository_file_handle_t file) { + void *rc; + pthread_join(file->thread, &rc); +} + +void repository_file_free(repository_file_handle_t file) { + // TODO, check what else should be freed. + free(file); +} + +static int fetch_package_file_work(const char *url, const char *dir, const char *file, const char *secret) { + char *path = NULL; + int saved = 0; + int rc = 0; + + if (NULL == url) { + return 1; + } + + _debug("file URL: %s", url); + + if (!(path = path_join(dir, basename(file)))) { + rc = 1; + goto cleanup; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + + // TODO, add force again. (1 == opts.force) + if (-1 == fs_exists(path)) { + logger_info("fetch", "%s:%s", url, file); + fflush(stdout); + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + + if (strstr(url, "gitlab") != NULL) { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + + rc = http_get_file_shared(url, path, clib_package_curl_share, &authentication_header, 1); + } else { + rc = http_get_file_shared(url, path, clib_package_curl_share, NULL, 0); + } + saved = 1; + } else { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + } + + if (-1 == rc) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + logger_error("error", "unable to fetch %s:%s", url, file); + fflush(stderr); + rc = 1; +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + goto cleanup; + } + + if (saved) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + logger_info("save", path); + fflush(stdout); +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + } + +cleanup: + + free(path); + return rc; +} + +#ifdef HAVE_PTHREADS +static void *fetch_package_file_thread(void *arg) { + repository_file_handle_t data = arg; + int *status = malloc(sizeof(int)); + int rc = fetch_package_file_work(data->url, data->dir, data->file, data->secret); + *status = rc; + pthread_exit((void *) status); +} +#endif + +static int fetch_package_file(const char *url, const char *dir, const char *file, const char *secret, repository_file_handle_t *thread_data_ptr) { +#ifndef HAVE_PTHREADS + return fetch_package_file_work(pkg, dir, file, secret); +#else + repository_file_handle_t fetch = malloc(sizeof(*fetch)); + int rc = 0; + + if (0 == fetch) { + return -1; + } + + *thread_data_ptr = 0; + + memset(fetch, 0, sizeof(*fetch)); + + fetch->url = url; + fetch->dir = dir; + fetch->file = file; + fetch->secret = secret; + + rc = pthread_attr_init(&fetch->attr); + + if (0 != rc) { + free(fetch); + return rc; + } + + rc = pthread_create(&fetch->thread, NULL, fetch_package_file_thread, fetch); + + if (0 != rc) { + pthread_attr_destroy(&fetch->attr); + free(fetch); + return rc; + } + + rc = pthread_attr_destroy(&fetch->attr); + + if (0 != rc) { + pthread_cancel(fetch->thread); + free(fetch); + return rc; + } + + *thread_data_ptr = fetch; + + return rc; +#endif +} diff --git a/src/repository/repository.h b/src/repository/repository.h new file mode 100644 index 00000000..f433afba --- /dev/null +++ b/src/repository/repository.h @@ -0,0 +1,50 @@ +// +// repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_REPOSITORY_H + +#include +#include + +typedef struct repository_file_t* repository_file_handle_t; + +/** + * Initialize with secrets to enable authentication. + */ +void repository_init(clib_secrets_t secrets); + +/** + * Start download of package manifest for the package with url. + * The file will be stored at destination_path. + * This function starts a thread to dowload the file, the thread can be joined with `repository_file_finish_download`. + * + * @return a handle on success and NULL on failure. + */ +http_get_response_t* repository_fetch_package_manifest(const char*package_url, const char* slug, const char* version); + +/** + * Start download of a file for the package with url. + * The file will be stored at destination_path. + * This function starts a thread to dowload the file, the thread can be joined with `repository_file_finish_download`. + * + * @return a handle on success and NULL on failure. + */ +repository_file_handle_t repository_download_package_file(const char*package_url, const char* slug, const char* version, const char *file_path, const char* destination_path); + +/** + * Waits until the download is finished. + * @param file + */ +void repository_file_finish_download(repository_file_handle_t file); + +/** + * Free the memory held by the file. + * @param file + */ +void repository_file_free(repository_file_handle_t file); + +#endif//CLIB_SRC_REPOSITORY_REPOSITORY_H From fdfb5a428bb720f0ec772bcf83416155e0759624 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Fri, 26 Mar 2021 18:08:29 +0100 Subject: [PATCH 05/15] installer: Separate install functions to improve clarity. misc: Fix some problems codacy detected. --- src/clib-build.c | 16 +- src/clib-configure.c | 12 +- src/clib-install.c | 54 +-- src/clib-update.c | 12 +- src/clib-upgrade.c | 25 +- src/common/clib-package-installer.c | 609 +++++++++++++++++++++++++++ src/common/clib-package-installer.h | 23 ++ src/common/clib-package.c | 614 ++-------------------------- src/common/clib-package.h | 11 +- src/registry/registry.c | 4 + 10 files changed, 724 insertions(+), 656 deletions(-) create mode 100644 src/common/clib-package-installer.c create mode 100644 src/common/clib-package-installer.h diff --git a/src/clib-build.c b/src/clib-build.c index a84125c0..752da5a1 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -79,7 +79,7 @@ struct options { #endif }; -clib_package_opts_t package_opts = {0}; +clib_package_opts_t build_package_opts = {0}; clib_package_t *root_package = 0; command_t program = {0}; @@ -231,8 +231,8 @@ int build_package_with_manifest_name(const char *dir, const char *file) { } if (root_package && root_package->prefix) { - package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); + build_package_opts.prefix = root_package->prefix; + clib_package_set_opts(build_package_opts); setenv("PREFIX", package_opts.prefix, 1); } else if (opts.prefix) { setenv("PREFIX", opts.prefix, 1); @@ -671,12 +671,12 @@ int main(int argc, char **argv) { clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; + build_package_opts.skip_cache = opts.skip_cache; + build_package_opts.prefix = opts.prefix; + build_package_opts.global = opts.global; + build_package_opts.force = opts.force; - clib_package_set_opts(package_opts); + clib_package_set_opts(build_package_opts); if (0 == program.argc || (argc == rest_offset + rest_argc)) { rc = build_package(CWD); diff --git a/src/clib-configure.c b/src/clib-configure.c index da3fcdc5..a8acf0f9 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -70,7 +70,7 @@ struct options { #endif }; -clib_package_opts_t package_opts = {0}; +clib_package_opts_t configure_package_opts = {0}; clib_package_t *root_package = 0; hash_t *configured = 0; @@ -603,12 +603,12 @@ int main(int argc, char **argv) { clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; + configure_package_opts.skip_cache = package_opts.skip_cache; + configure_package_opts.prefix = package_opts.prefix; + configure_package_opts.global = package_opts.global; + configure_package_opts.force = package_opts.force; - clib_package_set_opts(package_opts); + clib_package_set_opts(configure_package_opts); if (0 == program.argc || (argc == rest_offset + rest_argc)) { rc = configure_package(CWD); diff --git a/src/clib-install.c b/src/clib-install.c index 07dd3486..2e688774 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -12,20 +12,18 @@ #include "common/clib-validate.h" #include "debug/debug.h" #include "fs/fs.h" -#include "http-get/http-get.h" #include "logger/logger.h" #include "parson/parson.h" -#include "str-replace/str-replace.h" #include "version.h" #include #include -#include #include #include #include #include #include #include +#include "clib-package-installer.h" #define SX(s) #s #define S(s) SX(s) @@ -58,8 +56,9 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; static clib_package_t *root_package = NULL; +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; /** * Option setters. @@ -279,27 +278,18 @@ static int install_package(const char *slug) { } } - // Read local config files. - clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); - repository_init(secrets); // The repository requires the secrets for authentication. - clib_package_t *package = clib_package_load_local_manifest(0); - - registries_t registries = registry_manager_init_registries(package->registries, secrets); - registry_manager_fetch_registries(registries); registry_package_ptr_t package_info = registry_manger_find_package(registries, slug); if (!package_info) { debug(&debugger, "Package %s not found in any registry.", slug); return -1; } - pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), opts.verbose); if (NULL == pkg) return -1; if (root_package && root_package->prefix) { package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); } rc = clib_package_install(pkg, opts.dir, opts.verbose); @@ -418,38 +408,34 @@ int main(int argc, char *argv[]) { realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + memset((void *) opts.prefix, 0, size); + memcpy((void *) opts.prefix, prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; - package_opts.token = opts.token; + clib_package_opts_t install_package_opts = {0}; + install_package_opts.skip_cache = opts.skip_cache; + install_package_opts.prefix = opts.prefix; + install_package_opts.global = opts.global; + install_package_opts.force = opts.force; + install_package_opts.token = opts.token; #ifdef HAVE_PTHREADS - package_opts.concurrency = opts.concurrency; + install_package_opts.concurrency = opts.concurrency; #endif - clib_package_set_opts(package_opts); + clib_package_set_opts(install_package_opts); - if (!root_package) { - const char *name = NULL; - char *json = NULL; - unsigned int i = 0; + // Read local config files. + secrets = clib_secrets_load_from_file("clib_secrets.json"); + root_package = clib_package_load_local_manifest(0); - do { - name = manifest_names[i]; - json = fs_read(name); - } while (NULL != manifest_names[++i] && !json); + repository_init(secrets); // The repository requires the secrets for authentication. + registries = registry_manager_init_registries(root_package->registries, secrets); + registry_manager_fetch_registries(registries); - if (json) { - root_package = clib_package_new(json, opts.verbose); - } - } + clib_package_installer_init(registries, secrets); int code = 0 == program.argc ? install_local_packages() : install_packages(program.argc, program.argv); diff --git a/src/clib-update.c b/src/clib-update.c index 5eebeccb..b3367d0c 100755 --- a/src/clib-update.c +++ b/src/clib-update.c @@ -17,6 +17,7 @@ #include "parson/parson.h" #include "str-replace/str-replace.h" #include "version.h" +#include #include #include #include @@ -50,7 +51,6 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; static clib_package_t *root_package = NULL; /** @@ -331,14 +331,14 @@ int main(int argc, char *argv[]) { logger_error("error", "Failed to initialize cURL"); } - if (opts.prefix) { + if (package_opts.prefix) { char prefix[path_max]; memset(prefix, 0, path_max); - realpath(opts.prefix, prefix); + realpath(package_opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + package_opts.prefix = malloc(size); + memset((void *) package_opts.prefix, 0, size); + memcpy((void *) package_opts.prefix, prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); diff --git a/src/clib-upgrade.c b/src/clib-upgrade.c index 4046cf72..b9d0c39b 100755 --- a/src/clib-upgrade.c +++ b/src/clib-upgrade.c @@ -19,6 +19,7 @@ #include "tempdir/tempdir.h" #include "version.h" #include +#include #include #include #include @@ -54,7 +55,7 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; +static clib_package_opts_t upgrade_package_opts = {0}; static clib_package_t *root_package = NULL; /** @@ -148,8 +149,8 @@ static int install_package(const char *slug) { } if (root_package && root_package->prefix) { - package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); + upgrade_package_opts.prefix = root_package->prefix; + clib_package_set_opts(upgrade_package_opts); } char *tmp = gettempdir(); @@ -232,23 +233,23 @@ int main(int argc, char *argv[]) { realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + memset((void *) opts.prefix, 0, size); + memcpy((void *) opts.prefix, prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = 1; - package_opts.prefix = opts.prefix; - package_opts.global = 1; - package_opts.force = opts.force; - package_opts.token = opts.token; + upgrade_package_opts.skip_cache = 1; + upgrade_package_opts.prefix = opts.prefix; + upgrade_package_opts.global = 1; + upgrade_package_opts.force = opts.force; + upgrade_package_opts.token = opts.token; #ifdef HAVE_PTHREADS - package_opts.concurrency = opts.concurrency; + upgrade_package_opts.concurrency = opts.concurrency; #endif - clib_package_set_opts(package_opts); + clib_package_set_opts(upgrade_package_opts); char *slug = 0; diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c new file mode 100644 index 00000000..0087a504 --- /dev/null +++ b/src/common/clib-package-installer.c @@ -0,0 +1,609 @@ +// +// clib-package-installer.c +// +// Copyright (c) 2021 clib authors +// MIT licensed +// + +#include "clib-package-installer.h" +#include "asprintf/asprintf.h" +#include "clib-cache.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) +#include +#endif + +CURLSH *clib_package_curl_share; +//TODO, cleanup somewhere curl_share_cleanup(clib_package_curl_share); + +static debug_t _debugger; + +#define _debug(...) \ + ({ \ + if (!(_debugger.name)) \ + debug_init(&_debugger, "package-installer"); \ + debug(&_debugger, __VA_ARGS__); \ + }) + +#define E_FORMAT(...) \ + ({ \ + rc = asprintf(__VA_ARGS__); \ + if (-1 == rc) \ + goto cleanup; \ + }); + +static hash_t *visited_packages = 0; + +#ifdef HAVE_PTHREADS +typedef struct clib_package_lock clib_package_lock_t; +struct clib_package_lock { + pthread_mutex_t mutex; +}; + +static clib_package_lock_t lock = {PTHREAD_MUTEX_INITIALIZER}; + +#endif + +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; + +void clib_package_installer_init(registries_t _registries, clib_secrets_t _secrets) { + secrets = _secrets; + registries = _registries; +} + +static inline int install_packages(list_t *list, const char *dir, int verbose) { + list_node_t *node = NULL; + list_iterator_t *iterator = NULL; + int rc = -1; + list_t *freelist = NULL; + + if (!list || !dir) + goto cleanup; + + iterator = list_iterator_new(list, LIST_HEAD); + if (NULL == iterator) + goto cleanup; + + freelist = list_new(); + + while ((node = list_iterator_next(iterator))) { + clib_package_dependency_t *dep = NULL; + char *slug = NULL; + clib_package_t *pkg = NULL; + int error = 1; + + dep = (clib_package_dependency_t *) node->val; + char* package_id = clib_package_get_id(dep->author, dep->name); + slug = clib_package_slug(dep->author, dep->name, dep->version); + if (NULL == slug) + goto loop_cleanup; + + registry_package_ptr_t package_info = registry_manger_find_package(registries, package_id); + if (!package_info) { + debug(&_debugger, "Package %s not found in any registry.", slug); + return -1; + } + + pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), verbose); + if (NULL == pkg) + goto loop_cleanup; + + if (-1 == clib_package_install(pkg, dir, verbose)) + goto loop_cleanup; + + list_rpush(freelist, list_node_new(pkg)); + error = 0; + + loop_cleanup: + if (slug) + free(slug); + if (error) { + list_iterator_destroy(iterator); + iterator = NULL; + rc = -1; + goto cleanup; + } + } + + rc = 0; + +cleanup: + if (iterator) + list_iterator_destroy(iterator); + + if (freelist) { + iterator = list_iterator_new(freelist, LIST_HEAD); + while ((node = list_iterator_next(iterator))) { + clib_package_t *pkg = node->val; + if (pkg) + clib_package_free(pkg); + } + list_iterator_destroy(iterator); + list_destroy(freelist); + } + return rc; +} + +int clib_package_install_executable(clib_package_t *pkg, const char *dir, int verbose) { +#ifdef PATH_MAX + long path_max = PATH_MAX; +#elif defined(_PC_PATH_MAX) + long path_max = pathconf(dir, _PC_PATH_MAX); +#else + long path_max = 4096; +#endif + + int rc; + char *url = NULL; + char *file = NULL; + char *tarball = NULL; + char *command = NULL; + char *unpack_dir = NULL; + char *deps = NULL; + char *tmp = NULL; + char *reponame = NULL; + char dir_path[path_max]; + + _debug("install executable %s", pkg->repo); + + tmp = gettempdir(); + + if (NULL == tmp) { + if (verbose) { + logger_error("error", "gettempdir() out of memory"); + } + return -1; + } + + if (!pkg->repo) { + if (verbose) { + logger_error("error", "repo field required to install executable"); + } + return -1; + } + + reponame = strrchr(pkg->repo, '/'); + if (reponame && *reponame != '\0') + reponame++; + else { + if (verbose) { + logger_error("error", + "malformed repo field, must be in the form user/pkg"); + } + return -1; + } + + E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, + pkg->version); + + E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); + + E_FORMAT(&tarball, "%s/%s", tmp, file); + + rc = http_get_file_shared(url, tarball, clib_package_curl_share, NULL, 0); + + if (0 != rc) { + if (verbose) { + logger_error("error", "download failed for '%s@%s' - HTTP GET '%s'", + pkg->repo, pkg->version, url); + } + + goto cleanup; + } + + E_FORMAT(&command, "cd %s && gzip -dc %s | tar x", tmp, file); + + _debug("download url: %s", url); + _debug("file: %s", file); + _debug("tarball: %s", tarball); + _debug("command(extract): %s", command); + + // cheap untar + rc = system(command); + if (0 != rc) + goto cleanup; + + free(command); + command = NULL; + + clib_package_set_prefix(pkg, path_max); + + const char *configure = pkg->configure; + + if (0 == configure) { + configure = ":"; + } + + memset(dir_path, 0, path_max); + realpath(dir, dir_path); + + char *version = pkg->version; + if ('v' == version[0]) { + (void) version++; + } + + E_FORMAT(&unpack_dir, "%s/%s-%s", tmp, reponame, version); + + _debug("dir: %s", unpack_dir); + + if (pkg->dependencies) { + E_FORMAT(&deps, "%s/deps", unpack_dir); + _debug("deps: %s", deps); + rc = clib_package_install_dependencies(pkg, deps, verbose); + if (-1 == rc) + goto cleanup; + } + + if (!package_opts.global && pkg->makefile) { + E_FORMAT(&command, "cp -fr %s/%s/%s %s", dir_path, pkg->name, + basename(pkg->makefile), unpack_dir); + + rc = system(command); + if (0 != rc) { + goto cleanup; + } + + free(command); + } + + if (pkg->flags) { + char *flags = NULL; +#ifdef _GNU_SOURCE + char *cflags = secure_getenv("CFLAGS"); +#else + char *cflags = getenv("CFLAGS"); +#endif + + if (cflags) { + asprintf(&flags, "%s %s", cflags, pkg->flags); + } else { + asprintf(&flags, "%s", pkg->flags); + } + + setenv("CFLAGS", cflags, 1); + } + + E_FORMAT(&command, "cd %s && %s", unpack_dir, pkg->install); + + _debug("command(install): %s", command); + rc = system(command); + +cleanup: + free(tmp); + free(command); + free(tarball); + free(file); + free(url); + return rc; +} + +/** + * Install the given `pkg` in `dir` + */ + +int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { + list_iterator_t *iterator = NULL; + char *package_json = NULL; + char *pkg_dir = NULL; + char *command = NULL; + int pending = 0; + int rc = 0; + int i = 0; + +#ifdef PATH_MAX + long path_max = PATH_MAX; +#elif defined(_PC_PATH_MAX) + long path_max = pathconf(dir, _PC_PATH_MAX); +#else + long path_max = 4096; +#endif + +#ifdef HAVE_PTHREADS + int max = package_opts.concurrency; +#endif + +#ifdef CLIB_PACKAGE_PREFIX + if (0 == opts.prefix) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + opts.prefix = CLIB_PACKAGE_PREFIX; +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } +#endif + + if (0 == package_opts.prefix) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif +#ifdef _GNU_SOURCE + char *prefix = secure_getenv("PREFIX"); +#else + char *prefix = getenv("PREFIX"); +#endif + + if (prefix) { + package_opts.prefix = prefix; + } +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (0 == visited_packages) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + visited_packages = hash_new(); + // initial write because sometimes `hash_set()` crashes + hash_set(visited_packages, strdup(""), ""); + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (0 == package_opts.force && pkg && pkg->name) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + if (hash_has(visited_packages, pkg->name)) { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + return 0; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (!pkg || !dir) { + rc = -1; + goto cleanup; + } + + clib_package_set_prefix(pkg, path_max); + + if (!(pkg_dir = path_join(dir, pkg->name))) { + rc = -1; + goto cleanup; + } + + if (!package_opts.global) { + _debug("mkdir -p %s", pkg_dir); + // create directory for pkg + if (-1 == mkdirp(pkg_dir, 0777)) { + rc = -1; + goto cleanup; + } + } + + /* + if (NULL == pkg->url) { + pkg->url = clib_package_url(pkg->author, pkg->repo_name, pkg->version); + + if (NULL == pkg->url) { + rc = -1; + goto cleanup; + } + } + */ + + // write clib.json or package.json + if (!(package_json = path_join(pkg_dir, pkg->filename))) { + rc = -1; + goto cleanup; + } + + if (!package_opts.global && NULL != pkg->src) { + _debug("write: %s", package_json); + if (-1 == fs_write(package_json, pkg->json)) { + if (verbose) { + logger_error("error", "Failed to write %s", package_json); + } + + rc = -1; + goto cleanup; + } + } + + if (pkg->name) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + if (!hash_has(visited_packages, pkg->name)) { + hash_set(visited_packages, strdup(pkg->name), "t"); + } +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + // fetch makefile + if (!package_opts.global && pkg->makefile) { + _debug("fetch: %s/%s", pkg->repo, pkg->makefile); + repository_file_handle_t handle = repository_download_package_file(pkg->url, pkg_dir, pkg->version, pkg->makefile, pkg_dir); + if (handle == NULL) { + goto cleanup; + } + +#ifdef HAVE_PTHREADS + repository_file_finish_download(handle); + repository_file_free(handle); +#endif + } + + // if no sources are listed, just install + if (package_opts.global || NULL == pkg->src) + goto install; + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + if (clib_cache_has_package(pkg->author, pkg->name, pkg->version)) { + if (package_opts.skip_cache) { + clib_cache_delete_package(pkg->author, pkg->name, pkg->version); +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + goto download; + } + + if (0 != clib_cache_load_package(pkg->author, pkg->name, pkg->version, pkg_dir)) { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + goto download; + } + + if (verbose) { + logger_info("cache", pkg->repo); + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + + goto install; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + +download: + + iterator = list_iterator_new(pkg->src, LIST_HEAD); + list_node_t *source; + repository_file_handle_t *handles = malloc(pkg->src->len * sizeof(repository_file_handle_t)); + while ((source = list_iterator_next(iterator))) { + handles[i] = repository_download_package_file(pkg->url, pkg_dir, pkg->version, source->val, pkg_dir); + + if (handles[i] == NULL) { + list_iterator_destroy(iterator); + iterator = NULL; + rc = -1; + goto cleanup; + } + +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) + struct timespec ts = {0, 1000*1000*10}; + nanosleep(&ts, NULL); +#endif + +#ifdef HAVE_PTHREADS + if (i < 0) { + i = 0; + } + + (void) pending++; + + if (i < max) { + (void) i++; + } else { + while (--i >= 0) { + repository_file_finish_download(handles[i]); + repository_file_free(handles[i]); + (void) pending--; + +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) + usleep(1024 * 10); +#endif + } + } +#endif + } + +#ifdef HAVE_PTHREADS + while (--i >= 0) { + repository_file_finish_download(handles[i]); + + (void) pending--; + repository_file_free(handles[i]); + } +#endif + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + +install: + if (pkg->configure) { + E_FORMAT(&command, "cd %s/%s && %s", dir, pkg->name, pkg->configure); + + _debug("command(configure): %s", command); + + rc = system(command); + if (0 != rc) + goto cleanup; + } + + if (0 == rc && pkg->install) { + rc = clib_package_install_executable(pkg, dir, verbose); + } + + if (0 == rc) { + rc = clib_package_install_dependencies(pkg, dir, verbose); + } + +cleanup: + if (pkg_dir) + free(pkg_dir); + if (package_json) + free(package_json); + if (iterator) + list_iterator_destroy(iterator); + if (command) + free(command); + return rc; +} + +/** + * Install the given `pkg`'s dependencies in `dir` + */ +int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, + int verbose) { + if (!pkg || !dir) + return -1; + if (NULL == pkg->dependencies) + return 0; + + return install_packages(pkg->dependencies, dir, verbose); +} + +/** + * Install the given `pkg`'s development dependencies in `dir` + */ +int clib_package_install_development(clib_package_t *pkg, const char *dir, + int verbose) { + if (!pkg || !dir) + return -1; + if (NULL == pkg->development) + return 0; + + return install_packages(pkg->development, dir, verbose); +} diff --git a/src/common/clib-package-installer.h b/src/common/clib-package-installer.h new file mode 100644 index 00000000..65722e13 --- /dev/null +++ b/src/common/clib-package-installer.h @@ -0,0 +1,23 @@ +// +// clib-package-installer.h +// +// Copyright (c) 2021 Clib authors +// MIT licensed +// +#include "clib-package.h" +#include + +#ifndef CLIB_SRC_COMMON_CLIB_PACKAGE_INSTALLER_H +#define CLIB_SRC_COMMON_CLIB_PACKAGE_INSTALLER_H + +void clib_package_installer_init(registries_t registries, clib_secrets_t secrets); + +int clib_package_install(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_executable(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_development(clib_package_t *pkg, const char *dir, int verbose); + +#endif diff --git a/src/common/clib-package.c b/src/common/clib-package.c index 614980de..e6d29963 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -5,10 +5,6 @@ // MIT license // -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #include "asprintf/asprintf.h" #include "clib-cache.h" #include "clib-package.h" @@ -21,14 +17,7 @@ #include "mkdirp/mkdirp.h" #include "parse-repo/parse-repo.h" #include "parson/parson.h" -#include "path-join/path-join.h" #include "strdup/strdup.h" -#include "substr/substr.h" -#include "tempdir/tempdir.h" -#include -#include -#include -#include #include #include #include @@ -64,8 +53,7 @@ static clib_package_lock_t lock = {PTHREAD_MUTEX_INITIALIZER}; #endif -CURLSH *clib_package_curl_share; -debug_t _debugger; +static debug_t _debugger; #define _debug(...) \ ({ \ @@ -81,7 +69,7 @@ debug_t _debugger; goto cleanup; \ }); -static clib_package_opts_t opts = { +clib_package_opts_t package_opts = { #ifdef HAVE_PTHREADS .concurrency = MAX_THREADS, #endif @@ -102,57 +90,54 @@ static inline char *json_array_get_string_safe(JSON_Array *, int); static inline char *clib_package_file_url(const char *, const char *); -static inline char *clib_package_slug(const char *, const char *, const char *); - -static inline char *clib_package_repo(const char *, const char *); static inline list_t *parse_package_deps(JSON_Object *); static inline int install_packages(list_t *, const char *, int); void clib_package_set_opts(clib_package_opts_t o) { - if (1 == opts.skip_cache && 0 == o.skip_cache) { - opts.skip_cache = 0; - } else if (0 == opts.skip_cache && 1 == o.skip_cache) { - opts.skip_cache = 1; + if (1 == package_opts.skip_cache && 0 == o.skip_cache) { + package_opts.skip_cache = 0; + } else if (0 == package_opts.skip_cache && 1 == o.skip_cache) { + package_opts.skip_cache = 1; } - if (1 == opts.global && 0 == o.global) { - opts.global = 0; - } else if (0 == opts.global && 1 == o.global) { - opts.global = 1; + if (1 == package_opts.global && 0 == o.global) { + package_opts.global = 0; + } else if (0 == package_opts.global && 1 == o.global) { + package_opts.global = 1; } - if (1 == opts.force && 0 == o.force) { - opts.force = 0; - } else if (0 == opts.force && 1 == o.force) { - opts.force = 1; + if (1 == package_opts.force && 0 == o.force) { + package_opts.force = 0; + } else if (0 == package_opts.force && 1 == o.force) { + package_opts.force = 1; } if (0 != o.prefix) { if (0 == strlen(o.prefix)) { - opts.prefix = 0; + package_opts.prefix = 0; } else { - opts.prefix = o.prefix; + package_opts.prefix = o.prefix; } } if (0 != o.token) { if (0 == strlen(o.token)) { - opts.token = 0; + package_opts.token = 0; } else { - opts.token = o.token; + package_opts.token = o.token; } } if (o.concurrency) { - opts.concurrency = o.concurrency; + package_opts.concurrency = o.concurrency; } else if (o.concurrency < 0) { - opts.concurrency = 0; + package_opts.concurrency = 0; } - if (opts.concurrency < 0) { - opts.concurrency = 0; + if (package_opts.concurrency < 0) { + package_opts.concurrency = 0; } } @@ -207,7 +192,7 @@ static inline char *clib_package_file_url(const char *url, const char *file) { * Build a slug */ -static inline char *clib_package_slug(const char *author, const char *name, +char *clib_package_slug(const char *author, const char *name, const char *version) { int size = strlen(author) + 1 // / + strlen(name) + 1 // @ @@ -270,7 +255,7 @@ clib_package_t *clib_package_load_local_manifest(int verbose) { * Build a repo */ -static inline char *clib_package_repo(const char *author, const char *name) { +char *clib_package_get_id(const char *author, const char *name) { int size = strlen(author) + 1 // / + strlen(name) + 1 // \0 ; @@ -327,72 +312,6 @@ static inline list_t *parse_package_deps(JSON_Object *obj) { return list; } -static inline int install_packages(list_t *list, const char *dir, int verbose) { - list_node_t *node = NULL; - list_iterator_t *iterator = NULL; - int rc = -1; - list_t *freelist = NULL; - - if (!list || !dir) - goto cleanup; - - iterator = list_iterator_new(list, LIST_HEAD); - if (NULL == iterator) - goto cleanup; - - freelist = list_new(); - - while ((node = list_iterator_next(iterator))) { - clib_package_dependency_t *dep = NULL; - char *slug = NULL; - clib_package_t *pkg = NULL; - int error = 1; - - dep = (clib_package_dependency_t *)node->val; - slug = clib_package_slug(dep->author, dep->name, dep->version); - if (NULL == slug) - goto loop_cleanup; - - pkg = clib_package_new_from_slug_and_url(slug, "FIXME", verbose); - if (NULL == pkg) - goto loop_cleanup; - - if (-1 == clib_package_install(pkg, dir, verbose)) - goto loop_cleanup; - - list_rpush(freelist, list_node_new(pkg)); - error = 0; - - loop_cleanup: - if (slug) - free(slug); - if (error) { - list_iterator_destroy(iterator); - iterator = NULL; - rc = -1; - goto cleanup; - } - } - - rc = 0; - -cleanup: - if (iterator) - list_iterator_destroy(iterator); - - if (freelist) { - iterator = list_iterator_new(freelist, LIST_HEAD); - while ((node = list_iterator_next(iterator))) { - clib_package_t *pkg = node->val; - if (pkg) - clib_package_free(pkg); - } - list_iterator_destroy(iterator); - list_destroy(freelist); - } - return rc; -} - /** * Create a new clib package from the given `json` */ @@ -604,7 +523,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, #endif // fetch json if (clib_cache_has_json(author, name, version)) { - if (opts.skip_cache) { + if (package_opts.skip_cache) { clib_cache_delete_json(author, name, version); goto download; } @@ -687,7 +606,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, pkg->author = strdup(author); } - if (!(repo = clib_package_repo(pkg->author, pkg->name))) { + if (!(repo = clib_package_get_id(pkg->author, pkg->name))) { goto error; } @@ -807,13 +726,13 @@ clib_package_dependency_t *clib_package_dependency_new(const char *repo, } -static void set_prefix(clib_package_t *pkg, long path_max) { - if (NULL != opts.prefix || NULL != pkg->prefix) { +void clib_package_set_prefix(clib_package_t *pkg, long path_max) { + if (NULL != package_opts.prefix || NULL != pkg->prefix) { char path[path_max]; memset(path, 0, path_max); - if (opts.prefix) { - realpath(opts.prefix, path); + if (package_opts.prefix) { + realpath(package_opts.prefix, path); } else { realpath(pkg->prefix, path); } @@ -824,477 +743,6 @@ static void set_prefix(clib_package_t *pkg, long path_max) { } } -int clib_package_install_executable(clib_package_t *pkg, const char *dir, - int verbose) { -#ifdef PATH_MAX - long path_max = PATH_MAX; -#elif defined(_PC_PATH_MAX) - long path_max = pathconf(dir, _PC_PATH_MAX); -#else - long path_max = 4096; -#endif - - int rc; - char *url = NULL; - char *file = NULL; - char *tarball = NULL; - char *command = NULL; - char *unpack_dir = NULL; - char *deps = NULL; - char *tmp = NULL; - char *reponame = NULL; - char dir_path[path_max]; - - _debug("install executable %s", pkg->repo); - - tmp = gettempdir(); - - if (NULL == tmp) { - if (verbose) { - logger_error("error", "gettempdir() out of memory"); - } - return -1; - } - - if (!pkg->repo) { - if (verbose) { - logger_error("error", "repo field required to install executable"); - } - return -1; - } - - reponame = strrchr(pkg->repo, '/'); - if (reponame && *reponame != '\0') - reponame++; - else { - if (verbose) { - logger_error("error", - "malformed repo field, must be in the form user/pkg"); - } - return -1; - } - - E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, - pkg->version); - - E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); - - E_FORMAT(&tarball, "%s/%s", tmp, file); - - rc = http_get_file_shared(url, tarball, clib_package_curl_share, NULL, 0); - - if (0 != rc) { - if (verbose) { - logger_error("error", "download failed for '%s@%s' - HTTP GET '%s'", - pkg->repo, pkg->version, url); - } - - goto cleanup; - } - - E_FORMAT(&command, "cd %s && gzip -dc %s | tar x", tmp, file); - - _debug("download url: %s", url); - _debug("file: %s", file); - _debug("tarball: %s", tarball); - _debug("command(extract): %s", command); - - // cheap untar - rc = system(command); - if (0 != rc) - goto cleanup; - - free(command); - command = NULL; - - set_prefix(pkg, path_max); - - const char *configure = pkg->configure; - - if (0 == configure) { - configure = ":"; - } - - memset(dir_path, 0, path_max); - realpath(dir, dir_path); - - char *version = pkg->version; - if ('v' == version[0]) { - (void)version++; - } - - E_FORMAT(&unpack_dir, "%s/%s-%s", tmp, reponame, version); - - _debug("dir: %s", unpack_dir); - - if (pkg->dependencies) { - E_FORMAT(&deps, "%s/deps", unpack_dir); - _debug("deps: %s", deps); - rc = clib_package_install_dependencies(pkg, deps, verbose); - if (-1 == rc) - goto cleanup; - } - - if (!opts.global && pkg->makefile) { - E_FORMAT(&command, "cp -fr %s/%s/%s %s", dir_path, pkg->name, - basename(pkg->makefile), unpack_dir); - - rc = system(command); - if (0 != rc) { - goto cleanup; - } - - free(command); - } - - if (pkg->flags) { - char *flags = NULL; -#ifdef _GNU_SOURCE - char *cflags = secure_getenv("CFLAGS"); -#else - char *cflags = getenv("CFLAGS"); -#endif - - if (cflags) { - asprintf(&flags, "%s %s", cflags, pkg->flags); - } else { - asprintf(&flags, "%s", pkg->flags); - } - - setenv("CFLAGS", cflags, 1); - } - - E_FORMAT(&command, "cd %s && %s", unpack_dir, pkg->install); - - _debug("command(install): %s", command); - rc = system(command); - -cleanup: - free(tmp); - free(command); - free(tarball); - free(file); - free(url); - return rc; -} - -/** - * Install the given `pkg` in `dir` - */ - -int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { - list_iterator_t *iterator = NULL; - char *package_json = NULL; - char *pkg_dir = NULL; - char *command = NULL; - int pending = 0; - int rc = 0; - int i = 0; - -#ifdef PATH_MAX - long path_max = PATH_MAX; -#elif defined(_PC_PATH_MAX) - long path_max = pathconf(dir, _PC_PATH_MAX); -#else - long path_max = 4096; -#endif - -#ifdef HAVE_PTHREADS - int max = opts.concurrency; -#endif - -#ifdef CLIB_PACKAGE_PREFIX - if (0 == opts.prefix) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - opts.prefix = CLIB_PACKAGE_PREFIX; -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } -#endif - - if (0 == opts.prefix) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif -#ifdef _GNU_SOURCE - char *prefix = secure_getenv("PREFIX"); -#else - char *prefix = getenv("PREFIX"); -#endif - - if (prefix) { - opts.prefix = prefix; - } -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (0 == visited_packages) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - visited_packages = hash_new(); - // initial write because sometimes `hash_set()` crashes - hash_set(visited_packages, strdup(""), ""); - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (0 == opts.force && pkg && pkg->name) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (hash_has(visited_packages, pkg->name)) { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - return 0; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (!pkg || !dir) { - rc = -1; - goto cleanup; - } - - set_prefix(pkg, path_max); - - if (!(pkg_dir = path_join(dir, pkg->name))) { - rc = -1; - goto cleanup; - } - - if (!opts.global) { - _debug("mkdir -p %s", pkg_dir); - // create directory for pkg - if (-1 == mkdirp(pkg_dir, 0777)) { - rc = -1; - goto cleanup; - } - } - - /* - if (NULL == pkg->url) { - pkg->url = clib_package_url(pkg->author, pkg->repo_name, pkg->version); - - if (NULL == pkg->url) { - rc = -1; - goto cleanup; - } - } - */ - - // write clib.json or package.json - if (!(package_json = path_join(pkg_dir, pkg->filename))) { - rc = -1; - goto cleanup; - } - - if (!opts.global && NULL != pkg->src) { - _debug("write: %s", package_json); - if (-1 == fs_write(package_json, pkg->json)) { - if (verbose) { - logger_error("error", "Failed to write %s", package_json); - } - - rc = -1; - goto cleanup; - } - } - - if (pkg->name) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - if (!hash_has(visited_packages, pkg->name)) { - hash_set(visited_packages, strdup(pkg->name), "t"); - } -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - // fetch makefile - if (!opts.global && pkg->makefile) { - _debug("fetch: %s/%s", pkg->repo, pkg->makefile); - repository_file_handle_t handle = repository_download_package_file(pkg->url, pkg_dir, pkg->version, pkg->makefile, pkg_dir); - if (handle == NULL) { - goto cleanup; - } - -#ifdef HAVE_PTHREADS - repository_file_finish_download(handle); - repository_file_free(handle); -#endif - } - - // if no sources are listed, just install - if (opts.global || NULL == pkg->src) - goto install; - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (clib_cache_has_package(pkg->author, pkg->name, pkg->version)) { - if (opts.skip_cache) { - clib_cache_delete_package(pkg->author, pkg->name, pkg->version); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto download; - } - - if (0 != clib_cache_load_package(pkg->author, pkg->name, pkg->version, - pkg_dir)) { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto download; - } - - if (verbose) { - logger_info("cache", pkg->repo); - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - - goto install; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - -download: - - iterator = list_iterator_new(pkg->src, LIST_HEAD); - list_node_t *source; - repository_file_handle_t* handles = malloc(pkg->src->len*sizeof(repository_file_handle_t)); - while ((source = list_iterator_next(iterator))) { - handles[i] = repository_download_package_file(pkg->url, pkg_dir, pkg->version, source->val, pkg_dir); - - if (handles[i] == NULL) { - list_iterator_destroy(iterator); - iterator = NULL; - rc = -1; - goto cleanup; - } - -#ifdef HAVE_PTHREADS - if (i < 0) { - i = 0; - } - - (void)pending++; - - if (i < (max - 1)) { - (void)i++; - } else { - while (--i >= 0) { - repository_file_finish_download(handles[i]); - repository_file_free(handles[i]); - (void)pending--; - -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) - usleep(1024 * 10); -#endif - } - i = 0; - } -#endif - } - -#ifdef HAVE_PTHREADS - while (--i >= 0) { - repository_file_finish_download(handles[i]); - - (void)pending--; - repository_file_free(handles[i]); - } -#endif - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - -install: - if (pkg->configure) { - E_FORMAT(&command, "cd %s/%s && %s", dir, pkg->name, pkg->configure); - - _debug("command(configure): %s", command); - - rc = system(command); - if (0 != rc) - goto cleanup; - } - - if (0 == rc && pkg->install) { - rc = clib_package_install_executable(pkg, dir, verbose); - } - - if (0 == rc) { - rc = clib_package_install_dependencies(pkg, dir, verbose); - } - -cleanup: - if (pkg_dir) - free(pkg_dir); - if (package_json) - free(package_json); - if (iterator) - list_iterator_destroy(iterator); - if (command) - free(command); - return rc; -} - -/** - * Install the given `pkg`'s dependencies in `dir` - */ - -int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, - int verbose) { - if (!pkg || !dir) - return -1; - if (NULL == pkg->dependencies) - return 0; - - return install_packages(pkg->dependencies, dir, verbose); -} - -/** - * Install the given `pkg`'s development dependencies in `dir` - */ - -int clib_package_install_development(clib_package_t *pkg, const char *dir, - int verbose) { - if (!pkg || !dir) - return -1; - if (NULL == pkg->development) - return 0; - - return install_packages(pkg->development, dir, verbose); -} - /** * Free a clib package */ @@ -1362,6 +810,4 @@ void clib_package_cleanup() { hash_free(visited_packages); visited_packages = 0; } - - curl_share_cleanup(clib_package_curl_share); } diff --git a/src/common/clib-package.h b/src/common/clib-package.h index 35b50d08..42062a42 100644 --- a/src/common/clib-package.h +++ b/src/common/clib-package.h @@ -52,6 +52,8 @@ typedef struct { } clib_package_opts_t; extern CURLSH *clib_package_curl_share; +// TODO, move to a separate file. +extern clib_package_opts_t package_opts; void clib_package_set_opts(clib_package_opts_t opts); @@ -76,14 +78,11 @@ char *clib_package_parse_name(const char *); clib_package_dependency_t *clib_package_dependency_new(const char *, const char *); -int clib_package_install_executable(clib_package_t *pkg, const char *dir, - int verbose); +char *clib_package_get_id(const char *, const char *); -int clib_package_install(clib_package_t *, const char *, int); +char *clib_package_slug(const char *author, const char *name, const char* version); -int clib_package_install_dependencies(clib_package_t *, const char *, int); - -int clib_package_install_development(clib_package_t *, const char *, int); +void clib_package_set_prefix(clib_package_t *pkg, long path_max); void clib_package_free(clib_package_t *); diff --git a/src/registry/registry.c b/src/registry/registry.c index 238be3ea..0f812820 100644 --- a/src/registry/registry.c +++ b/src/registry/registry.c @@ -62,6 +62,8 @@ registry_ptr_t registry_create(const char *url, const char *secret) { } else if (strstr(url, "gitlab") != NULL) { registry->type = REGISTRY_TYPE_GITLAB; } else { + registry_free(registry); + return NULL; } @@ -106,9 +108,11 @@ bool registry_fetch(registry_ptr_t registry) { } break; default: + registry->packages = list_new(); return false; } + registry->packages = list_new(); return false; } From 81f76c5928882b045d080ba0bce9106ff121388a Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sat, 27 Mar 2021 11:42:01 +0100 Subject: [PATCH 06/15] update: Fix clib-update with the new way of fetching packages, fix a bug with downloading packages from gitlab, move common definitinos to clib-settings --- src/clib-build.c | 2 +- src/clib-configure.c | 2 +- src/clib-search.c | 228 ++++++++++++++-------------- src/clib-settings.h | 13 ++ src/clib-uninstall.c | 1 + src/clib-update.c | 13 ++ src/common/clib-package-installer.c | 46 ++---- src/common/clib-package.c | 6 +- src/repository/github-repository.c | 14 +- src/repository/github-repository.h | 2 +- src/repository/repository.c | 9 +- 11 files changed, 174 insertions(+), 162 deletions(-) create mode 100644 src/clib-settings.h diff --git a/src/clib-build.c b/src/clib-build.c index 752da5a1..0358210e 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -41,7 +41,7 @@ #include #include #include - +#include "clib-settings.h" #include "version.h" #define PROGRAM_NAME "clib-build" diff --git a/src/clib-configure.c b/src/clib-configure.c index a8acf0f9..6d0385ad 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -41,7 +41,7 @@ #include #include #include - +#include "clib-settings.h" #include "version.h" #define PROGRAM_NAME "clib-configure" diff --git a/src/clib-search.c b/src/clib-search.c index e7a7ec95..6974aaa3 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -49,50 +49,50 @@ static void setopt_nocache(command_t *self) { opt_cache = 0; } static void setopt_json(command_t *self) { opt_json = 1; } -#define COMPARE(v) \ - { \ - if (NULL == v) { \ - rc = 0; \ - goto cleanup; \ - } \ - case_lower(v); \ - for (int i = 0; i < count; i++) { \ - if (strstr(v, args[i])) { \ - rc = 1; \ - break; \ - } \ - } \ +#define COMPARE(v) \ + { \ + if (NULL == v) { \ + rc = 0; \ + goto cleanup; \ + } \ + case_lower(v); \ + for (int i = 0; i < count; i++) { \ + if (strstr(v, args[i])) { \ + rc = 1; \ + break; \ + } \ + } \ } static int matches(int count, char *args[], registry_package_ptr_t pkg) { - // Display all packages if there's no query - if (0 == count) - return 1; - - char *description = NULL; - char *name = NULL; - char *repo = NULL; - char *href = NULL; - int rc = 0; - - name = clib_package_parse_name(registry_package_get_id(pkg)); - COMPARE(name); - - description = strdup(registry_package_get_description(pkg)); - COMPARE(description); - - repo = strdup(registry_package_get_id(pkg)); - COMPARE(repo); - - href = strdup(registry_package_get_href(pkg)); - COMPARE(href); - - cleanup: - free(description); - free(name); - free(repo); - free(href); - return rc; + // Display all packages if there's no query + if (0 == count) + return 1; + + char *description = NULL; + char *name = NULL; + char *repo = NULL; + char *href = NULL; + int rc = 0; + + name = clib_package_parse_name(registry_package_get_id(pkg)); + COMPARE(name); + + description = strdup(registry_package_get_description(pkg)); + COMPARE(description); + + repo = strdup(registry_package_get_id(pkg)); + COMPARE(repo); + + href = strdup(registry_package_get_href(pkg)); + COMPARE(href); + +cleanup: + free(description); + free(name); + free(repo); + free(href); + return rc; } /* @@ -126,67 +126,67 @@ static char *wiki_html_cache() { static void display_package(const registry_package_ptr_t pkg, cc_color_t fg_color_highlight, cc_color_t fg_color_text) { - cc_fprintf(fg_color_highlight, stdout, " %s\n", registry_package_get_id(pkg)); - printf(" url: "); - cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_href(pkg)); - printf(" desc: "); - cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_description(pkg)); - printf("\n"); + cc_fprintf(fg_color_highlight, stdout, " %s\n", registry_package_get_id(pkg)); + printf(" url: "); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_href(pkg)); + printf(" desc: "); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_description(pkg)); + printf("\n"); } static void add_package_to_json(const registry_package_ptr_t pkg, JSON_Array *json_list) { - JSON_Value *json_pkg_root = json_value_init_object(); - JSON_Object *json_pkg = json_value_get_object(json_pkg_root); + JSON_Value *json_pkg_root = json_value_init_object(); + JSON_Object *json_pkg = json_value_get_object(json_pkg_root); - json_object_set_string(json_pkg, "repo", registry_package_get_id(pkg)); - json_object_set_string(json_pkg, "href", registry_package_get_href(pkg)); - json_object_set_string(json_pkg, "description", registry_package_get_description(pkg)); - json_object_set_string(json_pkg, "category", registry_package_get_category(pkg)); + json_object_set_string(json_pkg, "repo", registry_package_get_id(pkg)); + json_object_set_string(json_pkg, "href", registry_package_get_href(pkg)); + json_object_set_string(json_pkg, "description", registry_package_get_description(pkg)); + json_object_set_string(json_pkg, "category", registry_package_get_category(pkg)); - json_array_append_value(json_list, json_pkg_root); + json_array_append_value(json_list, json_pkg_root); } int main(int argc, char *argv[]) { - opt_color = 1; - opt_cache = 1; + opt_color = 1; + opt_cache = 1; - debug_init(&debugger, "clib-search"); + debug_init(&debugger, "clib-search"); - clib_cache_init(CLIB_SEARCH_CACHE_TIME); + clib_cache_init(CLIB_SEARCH_CACHE_TIME); - command_t program; - command_init(&program, "clib-search", CLIB_VERSION); - program.usage = "[options] [query ...]"; + command_t program; + command_init(&program, "clib-search", CLIB_VERSION); + program.usage = "[options] [query ...]"; - command_option(&program, "-n", "--no-color", "don't colorize output", - setopt_nocolor); + command_option(&program, "-n", "--no-color", "don't colorize output", + setopt_nocolor); - command_option(&program, "-c", "--skip-cache", "skip the search cache", - setopt_nocache); + command_option(&program, "-c", "--skip-cache", "skip the search cache", + setopt_nocache); - command_option(&program, "-j", "--json", "generate a serialized JSON output", - setopt_json); + command_option(&program, "-j", "--json", "generate a serialized JSON output", + setopt_json); - command_parse(&program, argc, argv); + command_parse(&program, argc, argv); - for (int i = 0; i < program.argc; i++) - case_lower(program.argv[i]); + for (int i = 0; i < program.argc; i++) + case_lower(program.argv[i]); - // set color theme - cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; - cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; + // set color theme + cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; + cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; - // We search the local manifest for extra registries. - // It is important to give the extra registries preference over the default registry. - clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + // We search the local manifest for extra registries. + // It is important to give the extra registries preference over the default registry. + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); - clib_package_t *package = clib_package_load_local_manifest(0); - registries_t registries = registry_manager_init_registries(package->registries, secrets); - registry_manager_fetch_registries(registries); + clib_package_t *package = clib_package_load_local_manifest(0); + registries_t registries = registry_manager_init_registries(package->registries, secrets); + registry_manager_fetch_registries(registries); - // TODO, implement caching for the new registries. - /* + // TODO, implement caching for the new registries. + /* char *html = wiki_html_cache(); if (NULL == html) { command_free(&program); @@ -198,45 +198,45 @@ int main(int argc, char *argv[]) { debug(&debugger, "found %zu packages", pkgs->len); */ - registry_iterator_t it = registry_iterator_new(registries); - registry_ptr_t registry = NULL; - while ((registry = registry_iterator_next(it))) { - printf("SEARCH: packages from %s\n", registry_get_url(registry)); - registry_package_ptr_t pkg; - registry_package_iterator_t it = registry_package_iterator_new(registry); + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t registry = NULL; + while ((registry = registry_iterator_next(it))) { + printf("SEARCH: packages from %s\n", registry_get_url(registry)); + registry_package_ptr_t pkg; + registry_package_iterator_t it = registry_package_iterator_new(registry); - JSON_Array *json_list = NULL; - JSON_Value *json_list_root = NULL; + JSON_Array *json_list = NULL; + JSON_Value *json_list_root = NULL; - if (opt_json) { - json_list_root = json_value_init_array(); - json_list = json_value_get_array(json_list_root); - } + if (opt_json) { + json_list_root = json_value_init_array(); + json_list = json_value_get_array(json_list_root); + } - printf("\n"); - - while ((pkg = registry_package_iterator_next(it))) { - if (matches(program.argc, program.argv, pkg)) { - if (opt_json) { - add_package_to_json(pkg, json_list); - } else { - display_package(pkg, fg_color_highlight, fg_color_text); - } - } else { - debug(&debugger, "skipped package %s", registry_package_get_id(pkg)); - } - } + printf("\n"); + while ((pkg = registry_package_iterator_next(it))) { + if (matches(program.argc, program.argv, pkg)) { if (opt_json) { - char *serialized = json_serialize_to_string_pretty(json_list_root); - puts(serialized); - - json_free_serialized_string(serialized); - json_value_free(json_list_root); + add_package_to_json(pkg, json_list); + } else { + display_package(pkg, fg_color_highlight, fg_color_text); } + } else { + debug(&debugger, "skipped package %s", registry_package_get_id(pkg)); + } + } - registry_package_iterator_destroy(it); + if (opt_json) { + char *serialized = json_serialize_to_string_pretty(json_list_root); + puts(serialized); + + json_free_serialized_string(serialized); + json_value_free(json_list_root); } - command_free(&program); - return 0; + + registry_package_iterator_destroy(it); + } + command_free(&program); + return 0; } diff --git a/src/clib-settings.h b/src/clib-settings.h new file mode 100644 index 00000000..bdd17902 --- /dev/null +++ b/src/clib-settings.h @@ -0,0 +1,13 @@ +#ifndef CLIB_SRC_CLIB_SETTINGS_H +#define CLIB_SRC_CLIB_SETTINGS_H + +// Shared settings +#define CLIB_PACKAGE_CACHE_TIME 30 * 24 * 60 * 60 + +#ifdef HAVE_PTHREADS +#define MAX_THREADS 12 +#endif + +const char *manifest_names[] = {"clib.json", "package.json", 0}; + +#endif//CLIB_SRC_CLIB_SETTINGS_H diff --git a/src/clib-uninstall.c b/src/clib-uninstall.c index 7dc37296..0e88a874 100755 --- a/src/clib-uninstall.c +++ b/src/clib-uninstall.c @@ -19,6 +19,7 @@ #include "version.h" #include #include +#include "clib-settings.h" #define CLIB_UNINSTALL_DEFAULT_TARGET "make uninstall" diff --git a/src/clib-update.c b/src/clib-update.c index b3367d0c..43405923 100755 --- a/src/clib-update.c +++ b/src/clib-update.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,8 @@ struct options { static struct options opts = {0}; static clib_package_t *root_package = NULL; +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; /** * Option setters. @@ -355,6 +358,16 @@ int main(int argc, char *argv[]) { clib_package_set_opts(package_opts); + // Read local config files. + secrets = clib_secrets_load_from_file("clib_secrets.json"); + root_package = clib_package_load_local_manifest(0); + + repository_init(secrets); // The repository requires the secrets for authentication. + registries = registry_manager_init_registries(root_package->registries, secrets); + registry_manager_fetch_registries(registries); + + clib_package_installer_init(registries, secrets); + int code = 0 == program.argc ? install_local_packages() : install_packages(program.argc, program.argv); diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index 0087a504..e05a5bde 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -20,28 +20,25 @@ #include #include #include - -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif +#include CURLSH *clib_package_curl_share; //TODO, cleanup somewhere curl_share_cleanup(clib_package_curl_share); static debug_t _debugger; -#define _debug(...) \ - ({ \ - if (!(_debugger.name)) \ +#define _debug(...) \ + ({ \ + if (!(_debugger.name)) \ debug_init(&_debugger, "package-installer"); \ - debug(&_debugger, __VA_ARGS__); \ + debug(&_debugger, __VA_ARGS__); \ }) -#define E_FORMAT(...) \ - ({ \ - rc = asprintf(__VA_ARGS__); \ - if (-1 == rc) \ - goto cleanup; \ +#define E_FORMAT(...) \ + ({ \ + rc = asprintf(__VA_ARGS__); \ + if (-1 == rc) \ + goto cleanup; \ }); static hash_t *visited_packages = 0; @@ -86,7 +83,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { int error = 1; dep = (clib_package_dependency_t *) node->val; - char* package_id = clib_package_get_id(dep->author, dep->name); + char *package_id = clib_package_get_id(dep->author, dep->name); slug = clib_package_slug(dep->author, dep->name, dep->version); if (NULL == slug) goto loop_cleanup; @@ -315,18 +312,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { int max = package_opts.concurrency; #endif -#ifdef CLIB_PACKAGE_PREFIX - if (0 == opts.prefix) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - opts.prefix = CLIB_PACKAGE_PREFIX; -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } -#endif - if (0 == package_opts.prefix) { #ifdef HAVE_PTHREADS pthread_mutex_lock(&lock.mutex); @@ -441,7 +426,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { // fetch makefile if (!package_opts.global && pkg->makefile) { _debug("fetch: %s/%s", pkg->repo, pkg->makefile); - repository_file_handle_t handle = repository_download_package_file(pkg->url, pkg_dir, pkg->version, pkg->makefile, pkg_dir); + repository_file_handle_t handle = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->name), pkg->version, pkg->makefile, pkg_dir); if (handle == NULL) { goto cleanup; } @@ -497,7 +482,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { list_node_t *source; repository_file_handle_t *handles = malloc(pkg->src->len * sizeof(repository_file_handle_t)); while ((source = list_iterator_next(iterator))) { - handles[i] = repository_download_package_file(pkg->url, pkg_dir, pkg->version, source->val, pkg_dir); + handles[i] = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->name), pkg->version, source->val, pkg_dir); if (handles[i] == NULL) { list_iterator_destroy(iterator); @@ -507,7 +492,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { } #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) - struct timespec ts = {0, 1000*1000*10}; + struct timespec ts = {0, 1000 * 1000 * 10}; nanosleep(&ts, NULL); #endif @@ -527,7 +512,8 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { (void) pending--; #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) - usleep(1024 * 10); + struct timespec ts = {0, 1000 * 1000 * 10}; + nanosleep(&ts, NULL); #endif } } diff --git a/src/common/clib-package.c b/src/common/clib-package.c index e6d29963..b9a87057 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -548,7 +548,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, } else { _debug("Fetching package manifest for %s", slug); // clean up when retrying - res = repository_fetch_package_manifest(url, slug, version); + res = repository_fetch_package_manifest(url, clib_package_get_id(author, name), version); json = res->data; _debug("status: %d", res->status); @@ -579,10 +579,10 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, if (!pkg) goto error; - // force version number + // force version number if the registry returned a different version from what we expected. if (pkg->version) { if (version) { - if (0 != strcmp(version, DEFAULT_REPO_VERSION)) { + if (0 != strcmp(version, DEFAULT_REPO_VERSION) && 0 != strcmp(pkg->version, version)) { _debug("forcing version number: %s (%s)", version, pkg->version); free(pkg->version); pkg->version = version; diff --git a/src/repository/github-repository.c b/src/repository/github-repository.c index be72990f..99a578d0 100644 --- a/src/repository/github-repository.c +++ b/src/repository/github-repository.c @@ -12,24 +12,24 @@ #define GITHUB_CONTENT_URL "https://raw.githubusercontent.com/" #define GITHUB_CONTENT_URL_WITH_TOKEN "https://%s@raw.githubusercontent.com/" -char* github_repository_get_url_for_file(const char* hostname, const char* slug, const char* version, const char *file_path, const char* secret) { +char *github_repository_get_url_for_file(const char *hostname, const char *package_id, const char *version, const char *file_path, const char *secret) { - int size = strlen(GITHUB_CONTENT_URL) + strlen(slug) + 1 // / - + strlen(version) + 1 // \0 - ; + int size = strlen(GITHUB_CONTENT_URL) + strlen(package_id) + 1// / + + strlen(version) + 1 + strlen(file_path) + 1 // \0 + ; if (secret != NULL) { size += strlen(secret); - size += 1; // @ + size += 1;// @ } char *url = malloc(size); if (url) { memset(url, '\0', size); if (secret != NULL) { - sprintf(url, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s", secret, slug, version); + sprintf(url, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s/%s", secret, package_id, version, file_path); } else { - sprintf(url, GITHUB_CONTENT_URL "%s/%s", slug, version); + sprintf(url, GITHUB_CONTENT_URL "%s/%s/%s", package_id, version, file_path); } } diff --git a/src/repository/github-repository.h b/src/repository/github-repository.h index 6a996c23..93579775 100644 --- a/src/repository/github-repository.h +++ b/src/repository/github-repository.h @@ -7,6 +7,6 @@ #ifndef CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H #define CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H -char* github_repository_get_url_for_file(const char* hostname, const char* slug, const char* version, const char *file, const char* secret); +char* github_repository_get_url_for_file(const char* hostname, const char*package_id, const char* version, const char *file, const char* secret); #endif//CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H diff --git a/src/repository/repository.c b/src/repository/repository.c index c922c34a..050421f6 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -68,11 +68,11 @@ void repository_init(clib_secrets_t _secrets) { secrets = _secrets; } -static char *repository_create_url_for_file(const char *package_url, const char *slug, const char *version, const char *file_path, const char *secret) { +static char *repository_create_url_for_file(const char *package_url, const char *package_id, const char *version, const char *file_path, const char *secret) { if (strstr(package_url, "github.com") != NULL) { - return github_repository_get_url_for_file(package_url, slug, version, file_path, secret); + return github_repository_get_url_for_file(package_url, package_id, version, file_path, secret); } else if (strstr(package_url, "gitlab") != NULL) { - return gitlab_repository_get_url_for_file(package_url, slug, version, file_path, secret); + return gitlab_repository_get_url_for_file(package_url, package_id, version, file_path, secret); } else { return NULL; } @@ -148,8 +148,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char pthread_mutex_lock(&mutex); #endif - // TODO, add force again. (1 == opts.force) - if (-1 == fs_exists(path)) { + if (package_opts.force || -1 == fs_exists(path)) { logger_info("fetch", "%s:%s", url, file); fflush(stdout); From c6aa0bb76000ec1cdc96439a79f6f571fc3a181c Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sat, 27 Mar 2021 23:04:38 +0100 Subject: [PATCH 07/15] tests: Start with getting the tests working. Incorporate some feedback. misc: Apply suggestions from code review Co-authored-by: Bruno Dias --- Makefile | 4 ++-- src/clib-install.c | 7 +++++-- src/common/clib-package-installer.c | 4 ++-- src/common/clib-secrets.c | 5 ++++- src/registry/github-registry.c | 5 +++-- src/registry/registry-manager.c | 29 ++++++++++++++++------------- src/registry/registry-manager.h | 2 +- src/registry/registry.c | 15 +++++++++------ src/repository/gitlab-repository.c | 4 +++- test/help.sh | 4 ++-- 10 files changed, 47 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 849ea295..1c31f6a1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -CC ?= gcc +CC ?= cc PREFIX ?= /usr/local BINS = clib clib-install clib-search clib-init clib-configure clib-build clib-update clib-upgrade clib-uninstall @@ -12,7 +12,7 @@ RM = rm -f MKDIR = mkdir -p SRC = $(wildcard src/*.c) -COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*) +COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*.c) ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c) SDEPS = $(wildcard deps/*/*.c) ODEPS = $(SDEPS:.c=.o) diff --git a/src/clib-install.c b/src/clib-install.c index 2e688774..9cb3d42b 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -278,7 +278,10 @@ static int install_package(const char *slug) { } } - registry_package_ptr_t package_info = registry_manger_find_package(registries, slug); + char* author = clib_package_parse_author(slug); + char* name = clib_package_parse_name(slug); + char* package_id = clib_package_get_id(author, name); + registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); if (!package_info) { debug(&debugger, "Package %s not found in any registry.", slug); return -1; @@ -432,7 +435,7 @@ int main(int argc, char *argv[]) { root_package = clib_package_load_local_manifest(0); repository_init(secrets); // The repository requires the secrets for authentication. - registries = registry_manager_init_registries(root_package->registries, secrets); + registries = registry_manager_init_registries(root_package ? root_package->registries : NULL, secrets); registry_manager_fetch_registries(registries); clib_package_installer_init(registries, secrets); diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index e05a5bde..b696c132 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -88,7 +88,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { if (NULL == slug) goto loop_cleanup; - registry_package_ptr_t package_info = registry_manger_find_package(registries, package_id); + registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); if (!package_info) { debug(&_debugger, "Package %s not found in any registry.", slug); return -1; @@ -482,7 +482,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { list_node_t *source; repository_file_handle_t *handles = malloc(pkg->src->len * sizeof(repository_file_handle_t)); while ((source = list_iterator_next(iterator))) { - handles[i] = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->name), pkg->version, source->val, pkg_dir); + handles[i] = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->repo_name), pkg->version, source->val, pkg_dir); if (handles[i] == NULL) { list_iterator_destroy(iterator); diff --git a/src/common/clib-secrets.c b/src/common/clib-secrets.c index c667be3c..84a9fd53 100644 --- a/src/common/clib-secrets.c +++ b/src/common/clib-secrets.c @@ -18,7 +18,7 @@ struct clib_secret_handle { clib_secrets_t clib_secrets_load_from_file(const char *file) { if (-1 == fs_exists(file)) { - logger_error("error", "Secrets file %s does not exist.", file); + logger_warn("warning", "Secrets file %s does not exist.", file); return NULL; } @@ -66,6 +66,9 @@ clib_secrets_t clib_secrets_load_from_file(const char *file) { } char *clib_secret_find_for_hostname(clib_secrets_t secrets, const char *hostname) { + if (secrets == NULL) { + return NULL; + } list_iterator_t *iterator = list_iterator_new(secrets->secrets, LIST_HEAD); list_node_t *node; while ((node = list_iterator_next(iterator))) { diff --git a/src/registry/github-registry.c b/src/registry/github-registry.c index eb431f7a..305d45ed 100644 --- a/src/registry/github-registry.c +++ b/src/registry/github-registry.c @@ -18,6 +18,8 @@ #include #include +#define GITHUB_BASE_URL "https://github.com/" + /** * Add `href` to the given `package`. */ @@ -25,7 +27,7 @@ static void add_package_href(registry_package_ptr_t self) { size_t len = strlen(self->id) + 20; // https://github.com/ \0 self->href = malloc(len); if (self->href) - sprintf(self->href, "https://github.com/%s", self->id); + sprintf(self->href, GITHUB_BASE_URL "%s", self->id); } /** @@ -129,4 +131,3 @@ list_t *github_registry_fetch(const char *url) { http_get_free(res); return list; } - diff --git a/src/registry/registry-manager.c b/src/registry/registry-manager.c index d178e5e4..eb6163e2 100644 --- a/src/registry/registry-manager.c +++ b/src/registry/registry-manager.c @@ -13,19 +13,22 @@ registries_t registry_manager_init_registries(list_t* registry_urls, clib_secrets_t secrets) { list_t* registries = list_new(); - // Add all the registries that were provided. - list_iterator_t *registry_iterator = list_iterator_new(registry_urls, LIST_HEAD); - list_node_t *node; - while ((node = list_iterator_next(registry_iterator))) { - char* url = node->val; - url_data_t *parsed = url_parse(url); - char* hostname = strdup(parsed->hostname); - url_free(parsed); - char* secret = clib_secret_find_for_hostname(secrets, hostname); - registry_ptr_t registry = registry_create(url, secret); - list_rpush(registries, list_node_new(registry)); + if (registry_urls != NULL) { + // Add all the registries that were provided. + list_iterator_t *registry_iterator = list_iterator_new(registry_urls, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(registry_iterator))) { + char *url = node->val; + url_data_t *parsed = url_parse(url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + registry_ptr_t registry = registry_create(url, secret); + if (registry != NULL) { + list_rpush(registries, list_node_new(registry)); + } + } + list_iterator_destroy(registry_iterator); } - list_iterator_destroy(registry_iterator); // And add the default registry. registry_ptr_t registry = registry_create(CLIB_WIKI_URL, NULL); @@ -45,7 +48,7 @@ void registry_manager_fetch_registries(registries_t registries) { registry_iterator_destroy(it); } -registry_package_ptr_t registry_manger_find_package(registries_t registries, const char* package_id) { +registry_package_ptr_t registry_manager_find_package(registries_t registries, const char* package_id) { registry_iterator_t it = registry_iterator_new(registries); registry_ptr_t reg; while ((reg = registry_iterator_next(it))) { diff --git a/src/registry/registry-manager.h b/src/registry/registry-manager.h index f717ca25..4eaa8962 100644 --- a/src/registry/registry-manager.h +++ b/src/registry/registry-manager.h @@ -39,6 +39,6 @@ void registry_iterator_destroy(registry_iterator_t iterator); * @param package_id the identifier of the package "/" * @return a pointer to the package if it could be found or NULL */ -registry_package_ptr_t registry_manger_find_package(registries_t registries, const char* package_id); +registry_package_ptr_t registry_manager_find_package(registries_t registries, const char* package_id); #endif//CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H diff --git a/src/registry/registry.c b/src/registry/registry.c index 0f812820..e2dc595c 100644 --- a/src/registry/registry.c +++ b/src/registry/registry.c @@ -11,6 +11,7 @@ #include "list/list.h" #include "registry-internal.h" #include "url/url.h" +#include #include #include @@ -56,21 +57,22 @@ registry_ptr_t registry_create(const char *url, const char *secret) { registry_ptr_t registry = malloc(sizeof(struct registry_t)); registry->url = strdup(url); registry->secret = strdup(secret); + registry->packages = NULL; + url_data_t *parsed = url_parse(url); + registry->hostname = strdup(parsed->hostname); + url_free(parsed); - if (strstr(url, "github.com") != NULL) { + if (strstr(registry->hostname, "github.com") != NULL) { registry->type = REGISTRY_TYPE_GITHUB; - } else if (strstr(url, "gitlab") != NULL) { + } else if (strstr(registry->hostname, "gitlab") != NULL) { registry->type = REGISTRY_TYPE_GITLAB; } else { + logger_error("error", "Registry type (%s) not supported, currently github.com, gitlab.com and self-hosted gitlab are supported.", registry->url); registry_free(registry); return NULL; } - url_data_t *parsed = url_parse(url); - registry->hostname = strdup(parsed->hostname); - url_free(parsed); - return registry; } @@ -112,6 +114,7 @@ bool registry_fetch(registry_ptr_t registry) { return false; } + logger_error("error", "Fetching package list from (%s) failed.", registry->url); registry->packages = list_new(); return false; } diff --git a/src/repository/gitlab-repository.c b/src/repository/gitlab-repository.c index fba873cc..0b272f0b 100644 --- a/src/repository/gitlab-repository.c +++ b/src/repository/gitlab-repository.c @@ -10,6 +10,8 @@ #include #include +#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=master" + // GET :hostname/api/v4/projects/:id/repository/files/:file_path/raw char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret) { url_data_t *parsed = url_parse(package_url); @@ -17,7 +19,7 @@ char* gitlab_repository_get_url_for_file(const char*package_url, const char* slu int size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(file) + 64; char *url = malloc(size); if (url) { - snprintf(url, size, "https://%s/api/v4%s/repository/files/%s/raw?ref=master", parsed->hostname, parsed->pathname, file); + snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, file); } url_free(parsed); diff --git a/test/help.sh b/test/help.sh index 9ceb1126..af69dca9 100755 --- a/test/help.sh +++ b/test/help.sh @@ -10,7 +10,7 @@ ACTUAL=$(clib help install) EXPECTED=$(clib install --help) [ "$ACTUAL" = "$EXPECTED" ] || { - echo >&2 "\`help install\` should ouput clib-install --help" + echo >&2 "\`help install\` should output clib-install --help" exit 1 } @@ -18,7 +18,7 @@ ACTUAL=$(clib help search) EXPECTED=$(clib search --help) [ "$ACTUAL" = "$EXPECTED" ] || { - echo >&2 "\`help search\` should ouput clib-search --help" + echo >&2 "\`help search\` should output clib-search --help" exit 1 } From bf1b5f263e7e92f98826f791b9d09b4b15d524ad Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Mon, 29 Mar 2021 12:20:06 +0200 Subject: [PATCH 08/15] makefile: Revert c version from gnu17 to c17 so that it can be built on other platforms as well. --- deps/url/url.h | 2 +- src/clib-build.c | 4 ---- src/clib-configure.c | 4 ---- src/clib-init.c | 3 +-- src/common/url.c | 1 + 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/deps/url/url.h b/deps/url/url.h index a9ea7597..bb7af258 100644 --- a/deps/url/url.h +++ b/deps/url/url.h @@ -156,7 +156,7 @@ url_data_inspect (url_data_t *data); #ifdef URL_H_IMPLEMENTATION // non C99 standard functions -#if _POSIX_C_SOURCE < 200809L +#if _POSIX_C_SOURCE < 200809L && !(defined URL_H_NO_STRDUP) char * strdup (const char *str) { int n = strlen(str) + 1; diff --git a/src/clib-build.c b/src/clib-build.c index 0358210e..4977eb4b 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -13,10 +13,6 @@ #include #include -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #ifdef HAVE_PTHREADS #include #endif diff --git a/src/clib-configure.c b/src/clib-configure.c index 6d0385ad..4e2ea077 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -12,10 +12,6 @@ #include #include -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #ifdef HAVE_PTHREADS #include #endif diff --git a/src/clib-init.c b/src/clib-init.c index fd4178b2..d067f803 100755 --- a/src/clib-init.c +++ b/src/clib-init.c @@ -7,9 +7,7 @@ #include "asprintf/asprintf.h" #include "commander/commander.h" -#include "common/clib-package.h" #include "debug/debug.h" -#include "fs/fs.h" #include "logger/logger.h" #include "parson/parson.h" #include "version.h" @@ -17,6 +15,7 @@ #include #include #include +#include #if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ defined(__MINGW64__) diff --git a/src/common/url.c b/src/common/url.c index 797b7997..f1e4ded6 100644 --- a/src/common/url.c +++ b/src/common/url.c @@ -1,4 +1,5 @@ #define URL_H_IMPLEMENTATION +#define URL_H_NO_STRDUP #include "url/url.h" // This is a bit of a hack to use url.h in multiple files. From 489424e7ad070bcd5fd672ad2fc764a6b3abb331 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Tue, 30 Mar 2021 02:07:01 +0200 Subject: [PATCH 09/15] tests: Fix existing tests except binary dependency tests. --- Makefile | 2 +- clib.json | 5 +- deps/http-get/http-get.c | 2 +- deps/http-get/http-get.h | 2 +- deps/url/{package.json => clib.json} | 7 +- deps/url/url.c | 456 ++++++++++++++++ deps/url/url.h | 502 +----------------- src/clib-install.c | 78 +-- src/clib-uninstall.c | 2 +- src/common/clib-package-installer.c | 23 +- src/common/clib-package.c | 94 ++-- src/common/clib-package.h | 4 - src/common/clib-secrets.c | 5 +- src/common/url.c | 6 - src/registry/github-registry.c | 180 +++---- src/registry/registry.c | 1 + src/repository/repository.c | 20 +- src/repository/repository.h | 2 +- test.sh | 9 - test/cache/cache-test.c | 11 +- test/install-deps-from-package-json.sh | 3 +- test/install-multiple-clibs-libs.sh | 10 +- test/install-save.sh | 20 +- test/package/Makefile | 4 +- test/package/package-install-dependencies.c | 14 +- .../package-install-dev-dependencies.c | 10 +- test/package/package-install.c | 36 +- test/package/package-new-from-slug.c | 15 +- test/package/package-url.c | 28 - 29 files changed, 720 insertions(+), 831 deletions(-) rename deps/url/{package.json => clib.json} (66%) create mode 100644 deps/url/url.c delete mode 100644 src/common/url.c delete mode 100644 test/package/package-url.c diff --git a/Makefile b/Makefile index 1c31f6a1..a7a300f9 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ OBJS = $(DEPS:.c=.o) export CC -CFLAGS += -std=c99 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -Wall -Werror=return--type -Wno-unused-function -U__STRICT_ANSI__ +CFLAGS += -std=c99 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -Wall -Werror=return-type -Werror=implicit-function-declaration -Wno-unused-function -U__STRICT_ANSI__ ifdef STATIC CFLAGS += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags) diff --git a/clib.json b/clib.json index ac853563..eb0db306 100644 --- a/clib.json +++ b/clib.json @@ -18,11 +18,10 @@ "which": "0.1.3", "stephenmathieson/str-flatten.c": "0.0.4", "commander": "1.3.2", - "stephenmathieson/wiki-registry.c": "0.0.4", "stephenmathieson/case.c": "0.1.3", "jwerle/fs.c": "0.2.0", "stephenmathieson/str-replace.c": "0.0.6", - "strdup": "*", + "strdup": "0.1.*", "Constellation/console-colors.c": "1.0.1", "littlstar/asprintf.c": "0.0.3", "logger": "0.0.1", @@ -44,7 +43,7 @@ "stephenmathieson/gumbo-get-element-by-id.c": "*", "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", "clibs/list": "*", - "jwerle/url.h": "0.1.*" + "jwerle/url.h": "0.2.*" }, "development": { "stephenmathieson/describe.h": "2.0.1" diff --git a/deps/http-get/http-get.c b/deps/http-get/http-get.c index e90a664c..8c860a83 100644 --- a/deps/http-get/http-get.c +++ b/deps/http-get/http-get.c @@ -133,7 +133,7 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share, const return (200 == status && CURLE_ABORTED_BY_CALLBACK != res) ? 0 : -1; } -int http_get_file(const char *url, const char *file) { +int http_get_file(const char *url, const char *file, const char** headers, int header_count) { return http_get_file_shared(url, file, NULL, NULL, 0); } diff --git a/deps/http-get/http-get.h b/deps/http-get/http-get.h index af713512..16dd3b4f 100644 --- a/deps/http-get/http-get.h +++ b/deps/http-get/http-get.h @@ -24,7 +24,7 @@ typedef struct { http_get_response_t *http_get(const char *url, const char** headers, int header_count); http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count); -int http_get_file(const char *, const char *); +int http_get_file(const char *, const char *, const char** headers, int header_count); int http_get_file_shared(const char *, const char *, void *, const char** headers, int header_count); void http_get_free(http_get_response_t *); diff --git a/deps/url/package.json b/deps/url/clib.json similarity index 66% rename from deps/url/package.json rename to deps/url/clib.json index c622b7ae..bcdb8de1 100644 --- a/deps/url/package.json +++ b/deps/url/clib.json @@ -1,9 +1,10 @@ { "name": "url", - "version": "0.0.2", + "version": "0.2.0", "repo": "jwerle/url.h", "description": "Parse URLs much like Node's url module", "keywords": ["url", "parse"], "license": "MIT", - "src": ["url.h"] -} + "makefile": "Makefile", + "src": ["url.h", "url.c"] +} \ No newline at end of file diff --git a/deps/url/url.c b/deps/url/url.c new file mode 100644 index 00000000..5e8f5488 --- /dev/null +++ b/deps/url/url.c @@ -0,0 +1,456 @@ +#include +#include "url.h" +#include + +/** + * URI Schemes + * http://en.wikipedia.org/wiki/URI_scheme + */ + +static const char *URL_SCHEMES[] = { + // official IANA registered schemes + "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw", + "beshare", "bitcoin", "bolo", "callto", "cap", "chrome", "crome-extension", "com-evenbrite-attendee", + "cid", "coap", "coaps","content", "crid", "cvs", "data", "dav", "dict", "lna-playsingle", "dln-playcontainer", + "dns", "dtn", "dvb", "ed2k", "facetime", "fax", "feed", "file", "finger", "fish","ftp", "geo", "gg","git", + "gizmoproject", "go", "gopher", "gtalk", "h323", "hcp", "http", "https", "iax", "icap", "icon","im", + "imap", "info", "ipn", "ipp", "irc", "irc6", "ircs", "iris", "iris.beep", "iris.xpc", "iris.xpcs","iris.lws", + "itms", "jabber", "jar", "jms", "keyparc", "lastfm", "ldap", "ldaps", "magnet", "mailserver","mailto", + "maps", "market", "message", "mid", "mms", "modem", "ms-help", "mssettings-power", "msnim", "msrp", + "msrps", "mtqp", "mumble", "mupdate", "mvn", "news", "nfs", "ni", "nih", "nntp", "notes","oid", + "paquelocktoken", "pack", "palm", "paparazzi", "pkcs11", "platform", "pop", "pres", "prospero", "proxy", + "psyc","query", "reload", "res", "resource", "rmi", "rsync", "rtmp","rtsp", "secondlife", "service","session", + "sftp", "sgn", "shttp", "sieve", "sip", "sips", "skype", "smb", "sms", "snews", "snmp", "soap.beep","soap.beeps", + "soldat", "spotify", "ssh", "steam", "svn", "tag", "teamspeak", "tel", "telnet", "tftp", "things","thismessage", + "tn3270", "tip", "tv", "udp", "unreal", "urn", "ut2004", "vemmi","ventrilo", "videotex", "view-source", "wais","webcal", + "ws", "wss", "wtai", "wyciwyg", "xcon", "xcon-userid", "xfire","xmlrpc.beep", "xmlrpc.beeps", "xmpp", "xri","ymsgr", + + // unofficial schemes + "javascript", "jdbc", "doi" +}; + + +static char * +strff (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + (void) *ptr++; + } + + return strdup(ptr); +} + +static char * +strrwd (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + (void) *ptr--; + } + + return strdup(ptr); +} + +static char * +get_part (char *url, const char *format, int l) { + bool has = false; + char *tmp = strdup(url); + char *tmp_url = strdup(url); + char *fmt_url = strdup(url); + char *ret; + + if (!tmp || !tmp_url || !fmt_url) + return NULL; + + strcpy(tmp, ""); + strcpy(fmt_url, ""); + + // move pointer exactly the amount + // of characters in the `prototcol` char + // plus 3 characters that represent the `://` + // part of the url + char* fmt_url_new = strff(fmt_url, l); + free(fmt_url); + fmt_url = fmt_url_new; + + sscanf(fmt_url, format, tmp); + + if (0 != strcmp(tmp, tmp_url)) { + has = true; + ret = strdup(tmp); + } + + free(tmp); + free(tmp_url); + free(fmt_url); + + return has? ret : NULL; +} + +url_data_t * +url_parse (char *url) { + url_data_t *data = (url_data_t *) malloc(sizeof(url_data_t)); + if (!data) return NULL; + + data->href = url; + char *tmp_url = strdup(url); + bool is_ssh = false; + + char *protocol = url_get_protocol(tmp_url); + if (!protocol) { + free(tmp_url); + return NULL; + } + // length of protocol plus :// + int protocol_len = (int) strlen(protocol) + 3; + data->protocol = protocol; + + is_ssh = url_is_ssh(protocol); + + char *auth = (char *) malloc(sizeof(char)); + int auth_len = 0; + if (strstr(tmp_url, "@")) { + auth = get_part(tmp_url, "%[^@]", protocol_len); + auth_len = strlen(auth); + if (auth) auth_len++; + } + + data->auth = auth; + + char *hostname; + + hostname = (is_ssh) + ? get_part(tmp_url, "%[^:]", protocol_len + auth_len) + : get_part(tmp_url, "%[^/]", protocol_len + auth_len); + + if (!hostname) { + free(tmp_url); + url_free(data); + return NULL; + } + int hostname_len = (int) strlen(hostname); + char *tmp_hostname = strdup(hostname); + data->hostname = hostname; + + char *host = (char *) malloc((strlen(tmp_hostname)+1) * sizeof(char)); + sscanf(tmp_hostname, "%[^:]", host); + free(tmp_hostname); + if (!host) { + free(tmp_url); + url_free(data); + return NULL; + } + data->host = host; + + int host_len = (int)strlen(host); + if (hostname_len > host_len) { + data->port = strff(hostname, host_len + 1); // +1 for ':' char; + } else { + data->port = NULL; + } + + char *tmp_path; + + tmp_path = (is_ssh) + ? get_part(tmp_url, ":%s", protocol_len + auth_len + hostname_len) + : get_part(tmp_url, "/%s", protocol_len + auth_len + hostname_len); + + char *path = (char *) malloc((strlen(tmp_path) + 2) * sizeof(char)); + if (!path) { + free(tmp_url); + url_free(data); + return NULL; + } + const char *fmt = (is_ssh)? "%s" : "/%s"; + sprintf(path, fmt, tmp_path); + data->path = path; + + char *pathname = (char *) malloc((strlen(tmp_path) + 2) * sizeof(char)); + free(tmp_path); + if (!pathname) { + free(tmp_url); + url_free(data); + return NULL; + } + strcat(pathname, ""); + tmp_path = strdup(path); + sscanf(tmp_path, "%[^? | ^#]", pathname); + int pathname_len = (int)strlen(pathname); + data->pathname = pathname; + + char *search = (char *) malloc(sizeof(search)); + if (!search) { + free(tmp_url); + url_free(data); + return NULL; + } + char* tmp_path_new = strff(tmp_path, pathname_len); + free(tmp_path); + tmp_path = tmp_path_new; + strcpy(search, ""); + sscanf(tmp_path, "%[^#]", search); + data->search = search; + int search_len = (int)strlen(search); + free(tmp_path); + + char *query = (char *) malloc(sizeof(char)); + if (!query) { + free(tmp_url); + url_free(data); + return NULL; + } + sscanf(search, "?%s", query); + data->query = query; + + char *hash = (char *) malloc(sizeof(char)); + if (!hash) { + free(tmp_url); + url_free(data); + return NULL; + } + tmp_path = strff(path, pathname_len + search_len); + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + data->hash = hash; + free(tmp_path); + free(tmp_url); + + return data; +} + +bool +url_is_protocol (char *str) { + int count = sizeof(URL_SCHEMES) / sizeof(URL_SCHEMES[0]); + + for (int i = 0; i < count; ++i) { + if (0 == strcmp(URL_SCHEMES[i], str)) { + return true; + } + } + + return false; +} + +bool +url_is_ssh (char *str) { + str = strdup(str); + if (0 == strcmp(str, "ssh") || 0 == strcmp(str, "git")) { + free(str); + return true; + + } + free(str); + + return false; +} + +char * +url_get_protocol (char *url) { + char *protocol = (char *) malloc(URL_PROTOCOL_MAX_LENGTH * sizeof(char)); + if (!protocol) return NULL; + sscanf(url, "%[^://]", protocol); + if (url_is_protocol(protocol)) return protocol; + return NULL; +} + + +char * +url_get_auth (char *url) { + char *protocol = url_get_protocol(url); + if (!protocol) return NULL; + int l = (int) strlen(protocol) + 3; + return get_part(url, "%[^@]", l); +} + +char * +url_get_hostname (char *url) { + int l = 3; + char *protocol = url_get_protocol(url); + char *tmp_protocol = strdup(protocol); + char *auth = url_get_auth(url); + + if (!protocol) return NULL; + if (auth) l += strlen(auth) + 1; // add one @ symbol + if (auth) free(auth); + + l += (int) strlen(protocol); + + free(protocol); + + char * hostname = url_is_ssh(tmp_protocol) + ? get_part(url, "%[^:]", l) + : get_part(url, "%[^/]", l); + free(tmp_protocol); + return hostname; +} + +char * +url_get_host (char *url) { + char *host = (char *) malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + + if (!host || !hostname) return NULL; + + sscanf(hostname, "%[^:]", host); + + free(hostname); + + return host; +} + +char * +url_get_pathname (char *url) { + char *path = url_get_path(url); + char *pathname = (char *) malloc(sizeof(char)); + + if (!path || !pathname) return NULL; + + strcat(pathname, ""); + sscanf(path, "%[^?]", pathname); + + free(path); + + return pathname; +} + +char * +url_get_path (char *url) { + int l = 3; + char *tmp_path; + char *protocol = url_get_protocol(url); + char *auth = url_get_auth(url); + char *hostname = url_get_hostname(url); + + + if (!protocol || !hostname) + return NULL; + + bool is_ssh = url_is_ssh(protocol); + + l += (int) strlen(protocol) + (int) strlen(hostname); + + if (auth) l+= (int) strlen(auth) +1; // @ symbol + + tmp_path = (is_ssh) + ? get_part(url, ":%s", l) + : get_part(url, "/%s", l); + + const char *fmt = (is_ssh)? "%s" : "/%s"; + char *path = (char *) malloc(strlen(tmp_path) * sizeof(char)); + sprintf(path, fmt, tmp_path); + + if (auth) free(auth); + free(protocol); + free(hostname); + free(tmp_path); + + return path; + +} + +char * +url_get_search (char *url) { + char *path = url_get_path(url); + char *pathname = url_get_pathname(url); + char *search = (char *) malloc(sizeof(char)); + + if (!path || !search) return NULL; + + char *tmp_path = strff(path, (int)strlen(pathname)); + strcat(search, ""); + sscanf(tmp_path, "%[^#]", search); + + tmp_path = strrwd(tmp_path, (int)strlen(pathname)); + + free(path); + free(pathname); + + return search; +} + +char * +url_get_query (char *url) { + char *search = url_get_search(url); + char *query = (char *) malloc(sizeof(char)); + if (!search) return NULL; + sscanf(search, "?%s", query); + free(search); + return query; +} + +char * +url_get_hash (char *url) { + char *hash = (char *) malloc(sizeof(char)); + if (!hash) return NULL; + + char *path = url_get_path(url); + if (!path) return NULL; + + char *pathname = url_get_pathname(url); + if (!pathname) return NULL; + char *search = url_get_search(url); + + int pathname_len = (int) strlen(pathname); + int search_len = (int) strlen(search); + char *tmp_path = strff(path, pathname_len + search_len); + + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + tmp_path = strrwd(tmp_path, pathname_len + search_len); + free(tmp_path); + free(pathname); + free(path); + if (search) free(search); + + return hash; +} + +char * +url_get_port (char *url) { + char *port = (char *) malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + char *host = url_get_host(url); + if (!port || !hostname) return NULL; + + char *tmp_hostname = strff(hostname, strlen(host) +1); + sscanf(tmp_hostname, "%s", port); + + free(hostname); + free(tmp_hostname); + return port; +} + +void +url_inspect (char *url) { + url_data_inspect(url_parse(url)); +} + + +void +url_data_inspect (url_data_t *data) { + printf("#url =>\n"); + printf(" .href: \"%s\"\n", data->href); + printf(" .protocol: \"%s\"\n", data->protocol); + printf(" .host: \"%s\"\n", data->host); + printf(" .auth: \"%s\"\n", data->auth); + printf(" .hostname: \"%s\"\n", data->hostname); + printf(" .pathname: \"%s\"\n", data->pathname); + printf(" .search: \"%s\"\n", data->search); + printf(" .path: \"%s\"\n", data->path); + printf(" .hash: \"%s\"\n", data->hash); + printf(" .query: \"%s\"\n", data->query); + printf(" .port: \"%s\"\n", data->port); +} + +void +url_free (url_data_t *data) { + if (!data) return; + if (data->auth) free(data->auth); + if (data->protocol) free(data->protocol); + if (data->hostname) free(data->hostname); + if (data->host) free(data->host); + if (data->pathname) free(data->pathname); + if (data->path) free(data->path); + if (data->hash) free(data->hash); + if (data->port) free(data->port); + if (data->search) free(data->search); + if (data->query) free(data->query); + free(data); +} \ No newline at end of file diff --git a/deps/url/url.h b/deps/url/url.h index bb7af258..986489c0 100644 --- a/deps/url/url.h +++ b/deps/url/url.h @@ -8,14 +8,12 @@ #include #include #include -#include - /** * url.h version */ -#define URL_VERSION 0.0.2 +#define URL_VERSION 0.2.0 /** @@ -46,53 +44,23 @@ #define URL_AUTH_MAX_LENGTH 32 -/** - * URI Schemes - * http://en.wikipedia.org/wiki/URI_scheme - */ - -#ifdef URL_H_IMPLEMENTATION -char *URL_SCHEMES[] = { - // official IANA registered schemes - "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw", - "beshare", "bitcoin", "bolo", "callto", "cap", "chrome", "crome-extension", "com-evenbrite-attendee", - "cid", "coap", "coaps","content", "crid", "cvs", "data", "dav", "dict", "lna-playsingle", "dln-playcontainer", - "dns", "dtn", "dvb", "ed2k", "facetime", "fax", "feed", "file", "finger", "fish","ftp", "geo", "gg","git", - "gizmoproject", "go", "gopher", "gtalk", "h323", "hcp", "http", "https", "iax", "icap", "icon","im", - "imap", "info", "ipn", "ipp", "irc", "irc6", "ircs", "iris", "iris.beep", "iris.xpc", "iris.xpcs","iris.lws", - "itms", "jabber", "jar", "jms", "keyparc", "lastfm", "ldap", "ldaps", "magnet", "mailserver","mailto", - "maps", "market", "message", "mid", "mms", "modem", "ms-help", "mssettings-power", "msnim", "msrp", - "msrps", "mtqp", "mumble", "mupdate", "mvn", "news", "nfs", "ni", "nih", "nntp", "notes","oid", - "paquelocktoken", "pack", "palm", "paparazzi", "pkcs11", "platform", "pop", "pres", "prospero", "proxy", - "psyc","query", "reload", "res", "resource", "rmi", "rsync", "rtmp","rtsp", "secondlife", "service","session", - "sftp", "sgn", "shttp", "sieve", "sip", "sips", "skype", "smb", "sms", "snews", "snmp", "soap.beep","soap.beeps", - "soldat", "spotify", "ssh", "steam", "svn", "tag", "teamspeak", "tel", "telnet", "tftp", "things","thismessage", - "tn3270", "tip", "tv", "udp", "unreal", "urn", "ut2004", "vemmi","ventrilo", "videotex", "view-source", "wais","webcal", - "ws", "wss", "wtai", "wyciwyg", "xcon", "xcon-userid", "xfire","xmlrpc.beep", "xmlrpc.beeps", "xmpp", "xri","ymsgr", - - // unofficial schemes - "javascript", "jdbc", "doi" -}; -#endif - - /** * `url_data` struct that defines parts * of a parsed URL such as host and protocol */ typedef struct url_data { - char *href; - char *protocol; - char *host; - char *auth; - char *hostname; - char *pathname; - char *search; - char *path; - char *hash; - char *query; - char *port; + char *href; + char *protocol; + char *host; + char *auth; + char *hostname; + char *pathname; + char *search; + char *path; + char *hash; + char *query; + char *port; } url_data_t; @@ -104,7 +72,7 @@ typedef struct url_data { */ url_data_t * -url_parse (const char *url); +url_parse (char *url); char * url_get_protocol (char *url); @@ -152,446 +120,4 @@ void url_data_inspect (url_data_t *data); -// implementation -#ifdef URL_H_IMPLEMENTATION - -// non C99 standard functions -#if _POSIX_C_SOURCE < 200809L && !(defined URL_H_NO_STRDUP) -char * -strdup (const char *str) { - int n = strlen(str) + 1; - char *dup = malloc(n); - if (dup) strcpy(dup, str); - return dup; -} -#endif - - -static char * -strff (char *ptr, int n) { - for (int i = 0; i < n; ++i) { - *ptr++; - } - - return strdup(ptr); -} - -static char * -strrwd (char *ptr, int n) { - for (int i = 0; i < n; ++i) { - *ptr--; - } - - return strdup(ptr); -} - -static char * -get_part (char *url, const char *format, int l) { - bool has = false; - char *tmp = strdup(url); - char *tmp_url = strdup(url); - char *fmt_url = strdup(url); - char *ret; - - if (!tmp || !tmp_url || !fmt_url) - return NULL; - - strcpy(tmp, ""); - strcpy(fmt_url, ""); - - // move pointer exactly the amount - // of characters in the `prototcol` char - // plus 3 characters that represent the `://` - // part of the url - char* fmt_url_new = strff(fmt_url, l); - free(fmt_url); - fmt_url = fmt_url_new; - - sscanf(fmt_url, format, tmp); - - if (0 != strcmp(tmp, tmp_url)) { - has = true; - ret = strdup(tmp); - } - - free(tmp); - free(tmp_url); - free(fmt_url); - - return has? ret : NULL; -} - -url_data_t * -url_parse (const char *url) { - url_data_t *data = malloc(sizeof(url_data_t)); - if (!data) return NULL; - - data->href = url; - char *tmp_url = strdup(url); - bool is_ssh = false; - - char *protocol = url_get_protocol(tmp_url); - if (!protocol) { - free(tmp_url); - return NULL; - } - // length of protocol plus :// - int protocol_len = (int) strlen(protocol) + 3; - data->protocol = protocol; - - is_ssh = url_is_ssh(protocol); - - char *auth = malloc(sizeof(char)); - int auth_len = 0; - if (strstr(tmp_url, "@")) { - auth = get_part(tmp_url, "%[^@]", protocol_len); - auth_len = strlen(auth); - if (auth) auth_len++; - } - - data->auth = auth; - - char *hostname; - - hostname = (is_ssh) - ? get_part(tmp_url, "%[^:]", protocol_len + auth_len) - : get_part(tmp_url, "%[^/]", protocol_len + auth_len); - - if (!hostname) { - free(tmp_url); - url_free(data); - return NULL; - } - int hostname_len = (int) strlen(hostname); - char *tmp_hostname = strdup(hostname); - data->hostname = hostname; - - char *host = malloc((strlen(tmp_hostname)+1) * sizeof(char)); - sscanf(tmp_hostname, "%[^:]", host); - free(tmp_hostname); - if (!host) { - free(tmp_url); - url_free(data); - return NULL; - } - data->host = host; - - int host_len = (int)strlen(host); - if (hostname_len > host_len) { - data->port = strff(hostname, host_len + 1); // +1 for ':' char; - } else { - data->port = NULL; - } - - char *tmp_path; - - tmp_path = (is_ssh) - ? get_part(tmp_url, ":%s", protocol_len + auth_len + hostname_len) - : get_part(tmp_url, "/%s", protocol_len + auth_len + hostname_len); - - char *path = malloc((strlen(tmp_path) + 2) * sizeof(char)); - if (!path) { - free(tmp_url); - url_free(data); - return NULL; - } - char *fmt = (is_ssh)? "%s" : "/%s"; - sprintf(path, fmt, tmp_path); - data->path = path; - - char *pathname = malloc((strlen(tmp_path) + 2) * sizeof(char)); - free(tmp_path); - if (!pathname) { - free(tmp_url); - url_free(data); - return NULL; - } - strcat(pathname, ""); - tmp_path = strdup(path); - sscanf(tmp_path, "%[^? | ^#]", pathname); - int pathname_len = (int)strlen(pathname); - data->pathname = pathname; - - char *search = malloc(sizeof(search)); - if (!search) { - free(tmp_url); - url_free(data); - return NULL; - } - char* tmp_path_new = strff(tmp_path, pathname_len); - free(tmp_path); - tmp_path = tmp_path_new; - strcpy(search, ""); - sscanf(tmp_path, "%[^#]", search); - data->search = search; - int search_len = (int)strlen(search); - free(tmp_path); - - char *query = malloc(sizeof(char)); - if (!query) { - free(tmp_url); - url_free(data); - return NULL; - } - sscanf(search, "?%s", query); - data->query = query; - - char *hash = malloc(sizeof(char)); - if (!hash) { - free(tmp_url); - url_free(data); - return NULL; - } - tmp_path = strff(path, pathname_len + search_len); - strcat(hash, ""); - sscanf(tmp_path, "%s", hash); - data->hash = hash; - free(tmp_path); - free(tmp_url); - - return data; -} - -bool -url_is_protocol (char *str) { - int count = sizeof(URL_SCHEMES) / sizeof(URL_SCHEMES[0]); - - for (int i = 0; i < count; ++i) { - if (0 == strcmp(URL_SCHEMES[i], str)) { - return true; - } - } - - return false; -} - -bool -url_is_ssh (char *str) { - str = strdup(str); - if (0 == strcmp(str, "ssh") || 0 == strcmp(str, "git")) { - free(str); - return true; - - } - free(str); - - return false; -} - -char * -url_get_protocol (char *url) { - char *protocol = malloc(URL_PROTOCOL_MAX_LENGTH * sizeof(char)); - if (!protocol) return NULL; - sscanf(url, "%[^://]", protocol); - if (url_is_protocol(protocol)) return protocol; - return NULL; -} - - -char * -url_get_auth (char *url) { - char *protocol = url_get_protocol(url); - if (!protocol) return NULL; - int l = (int) strlen(protocol) + 3; - return get_part(url, "%[^@]", l); -} - -char * -url_get_hostname (char *url) { - int l = 3; - char *protocol = url_get_protocol(url); - char *tmp_protocol = strdup(protocol); - char *auth = url_get_auth(url); - - if (!protocol) return NULL; - if (auth) l += strlen(auth) + 1; // add one @ symbol - if (auth) free(auth); - - l += (int) strlen(protocol); - - free(protocol); - - char * hostname = url_is_ssh(tmp_protocol) - ? get_part(url, "%[^:]", l) - : get_part(url, "%[^/]", l); - free(tmp_protocol); - return hostname; -} - -char * -url_get_host (char *url) { - char *host = malloc(sizeof(char)); - char *hostname = url_get_hostname(url); - - if (!host || !hostname) return NULL; - - sscanf(hostname, "%[^:]", host); - - free(hostname); - - return host; -} - -char * -url_get_pathname (char *url) { - char *path = url_get_path(url); - char *pathname = malloc(sizeof(char)); - - if (!path || !pathname) return NULL; - - strcat(pathname, ""); - sscanf(path, "%[^?]", pathname); - - free(path); - - return pathname; -} - -char * -url_get_path (char *url) { - int l = 3; - char *tmp_path; - char *protocol = url_get_protocol(url); - char *auth = url_get_auth(url); - char *hostname = url_get_hostname(url); - - - if (!protocol || !hostname) - return NULL; - - bool is_ssh = url_is_ssh(protocol); - - l += (int) strlen(protocol) + (int) strlen(hostname); - - if (auth) l+= (int) strlen(auth) +1; // @ symbol - - tmp_path = (is_ssh) - ? get_part(url, ":%s", l) - : get_part(url, "/%s", l); - - char *fmt = (is_ssh)? "%s" : "/%s"; - char *path = malloc(strlen(tmp_path) * sizeof(char)); - sprintf(path, fmt, tmp_path); - - if (auth) free(auth); - free(protocol); - free(hostname); - free(tmp_path); - - return path; - -} - -char * -url_get_search (char *url) { - char *path = url_get_path(url); - char *pathname = url_get_pathname(url); - char *search = malloc(sizeof(char)); - - if (!path || !search) return NULL; - - char *tmp_path = strff(path, (int)strlen(pathname)); - strcat(search, ""); - sscanf(tmp_path, "%[^#]", search); - - tmp_path = strrwd(tmp_path, (int)strlen(pathname)); - - free(path); - free(pathname); - - return search; -} - -char * -url_get_query (char *url) { - char *search = url_get_search(url); - char *query = malloc(sizeof(char)); - if (!search) return NULL; - sscanf(search, "?%s", query); - free(search); - return query; -} - -char * -url_get_hash (char *url) { - char *hash = malloc(sizeof(char)); - if (!hash) return NULL; - - char *path = url_get_path(url); - if (!path) return NULL; - - char *pathname = url_get_pathname(url); - if (!pathname) return NULL; - char *search = url_get_search(url); - - int pathname_len = (int) strlen(pathname); - int search_len = (int) strlen(search); - char *tmp_path = strff(path, pathname_len + search_len); - - strcat(hash, ""); - sscanf(tmp_path, "%s", hash); - tmp_path = strrwd(tmp_path, pathname_len + search_len); - free(tmp_path); - free(pathname); - free(path); - if (search) free(search); - - return hash; -} - -char * -url_get_port (char *url) { - char *port = malloc(sizeof(char)); - char *hostname = url_get_hostname(url); - char *host = url_get_host(url); - if (!port || !hostname) return NULL; - - char *tmp_hostname = strff(hostname, strlen(host) +1); - sscanf(tmp_hostname, "%s", port); - - free(hostname); - free(tmp_hostname); - return port; -} - -void -url_inspect (char *url) { - url_data_inspect(url_parse(url)); -} - - -void -url_data_inspect (url_data_t *data) { - printf("#url =>\n"); - printf(" .href: \"%s\"\n", data->href); - printf(" .protocol: \"%s\"\n", data->protocol); - printf(" .host: \"%s\"\n", data->host); - printf(" .auth: \"%s\"\n", data->auth); - printf(" .hostname: \"%s\"\n", data->hostname); - printf(" .pathname: \"%s\"\n", data->pathname); - printf(" .search: \"%s\"\n", data->search); - printf(" .path: \"%s\"\n", data->path); - printf(" .hash: \"%s\"\n", data->hash); - printf(" .query: \"%s\"\n", data->query); - printf(" .port: \"%s\"\n", data->port); -} - -void -url_free (url_data_t *data) { - if (!data) return; - if (data->auth) free(data->auth); - if (data->protocol) free(data->protocol); - if (data->hostname) free(data->hostname); - if (data->host) free(data->host); - if (data->pathname) free(data->pathname); - if (data->path) free(data->path); - if (data->hash) free(data->hash); - if (data->port) free(data->port); - if (data->search) free(data->search); - if (data->query) free(data->query); - free(data); -} - -#endif - -#endif +#endif \ No newline at end of file diff --git a/src/clib-install.c b/src/clib-install.c index 9cb3d42b..009d7fc8 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -24,6 +24,7 @@ #include #include #include "clib-package-installer.h" +#include "strdup/strdup.h" #define SX(s) #s #define S(s) SX(s) @@ -56,7 +57,6 @@ struct options { static struct options opts = {0}; -static clib_package_t *root_package = NULL; static clib_secrets_t secrets = NULL; static registries_t registries = NULL; @@ -123,59 +123,25 @@ static void setopt_skip_cache(command_t *self) { debug(&debugger, "set skip cache flag"); } -static int install_local_packages_with_package_name(const char *file) { - if (0 != clib_validate(file)) { - return 1; - } - - debug(&debugger, "reading local clib.json or package.json"); - char *json = fs_read(file); - if (NULL == json) - return 1; - - clib_package_t *pkg = clib_package_new(json, opts.verbose); - if (NULL == pkg) - goto e1; - - if (pkg->prefix) { - setenv("PREFIX", pkg->prefix, 1); +/** + * Install dependency packages at `pwd`. + */ +static int install_local_packages(clib_package_t* root_package) { + if (root_package && root_package->prefix) { + setenv("PREFIX", root_package->prefix, 1); } - int rc = clib_package_install_dependencies(pkg, opts.dir, opts.verbose); + int rc = clib_package_install_dependencies(root_package, opts.dir, opts.verbose); if (-1 == rc) - goto e2; + return 1; if (opts.dev) { - rc = clib_package_install_development(pkg, opts.dir, opts.verbose); + rc = clib_package_install_development(root_package, opts.dir, opts.verbose); if (-1 == rc) - goto e2; + return 1; } - free(json); - clib_package_free(pkg); return 0; - -e2: - clib_package_free(pkg); -e1: - free(json); - return 1; -} - -/** - * Install dependency packages at `pwd`. - */ -static int install_local_packages() { - const char *name = NULL; - unsigned int i = 0; - int rc = 0; - - do { - name = manifest_names[i]; - rc = install_local_packages_with_package_name(name); - } while (NULL != manifest_names[++i] && 0 != rc); - - return rc; } static int write_dependency_with_package_name(clib_package_t *pkg, char *prefix, @@ -240,8 +206,7 @@ static int save_dev_dependency(clib_package_t *pkg) { /** * Create and install a package from `slug`. */ - -static int install_package(const char *slug) { +static int install_package(clib_package_t* root_package, const char *slug) { clib_package_t *pkg = NULL; int rc; @@ -258,7 +223,7 @@ static int install_package(const char *slug) { char dir[path_max]; realpath(slug, dir); slug = dir; - return install_local_packages(); + return install_local_packages(root_package); } } @@ -270,7 +235,7 @@ static int install_package(const char *slug) { #endif )) { free(stats); - return install_local_packages_with_package_name(slug); + return install_local_packages(root_package); } if (stats) { @@ -307,10 +272,6 @@ static int install_package(const char *slug) { } } - if (0 == pkg->repo || 0 != strcmp(slug, pkg->repo)) { - pkg->repo = strdup(slug); - } - if (opts.save) save_dependency(pkg); if (opts.savedev) @@ -325,10 +286,10 @@ static int install_package(const char *slug) { * Install the given `pkgs`. */ -static int install_packages(int n, char *pkgs[]) { +static int install_packages(clib_package_t* root_package, int n, char *pkgs[]) { for (int i = 0; i < n; i++) { debug(&debugger, "install %s (%d)", pkgs[i], i); - if (-1 == install_package(pkgs[i])) { + if (-1 == install_package(root_package, pkgs[i])) { logger_error("error", "Unable to install package %s", pkgs[i]); return 1; } @@ -432,7 +393,7 @@ int main(int argc, char *argv[]) { // Read local config files. secrets = clib_secrets_load_from_file("clib_secrets.json"); - root_package = clib_package_load_local_manifest(0); + clib_package_t* root_package = clib_package_load_local_manifest(0); repository_init(secrets); // The repository requires the secrets for authentication. registries = registry_manager_init_registries(root_package ? root_package->registries : NULL, secrets); @@ -440,8 +401,9 @@ int main(int argc, char *argv[]) { clib_package_installer_init(registries, secrets); - int code = 0 == program.argc ? install_local_packages() - : install_packages(program.argc, program.argv); + // TODO, move argument parsing here. + int code = 0 == program.argc ? install_local_packages(root_package) + : install_packages(root_package, program.argc, program.argv); curl_global_cleanup(); clib_package_cleanup(); diff --git a/src/clib-uninstall.c b/src/clib-uninstall.c index 0e88a874..2d3a934c 100755 --- a/src/clib-uninstall.c +++ b/src/clib-uninstall.c @@ -161,7 +161,7 @@ static int clib_uninstall(const char *owner, const char *name, goto done; logger_info("fetch", tarball); - if (-1 == http_get_file(tarball, tarpath)) { + if (-1 == http_get_file(tarball, tarpath, NULL, 0)) { logger_error("error", "failed to fetch tarball"); goto done; } diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index b696c132..5b7888fc 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -21,6 +21,7 @@ #include #include #include +#include CURLSH *clib_package_curl_share; //TODO, cleanup somewhere curl_share_cleanup(clib_package_curl_share); @@ -90,7 +91,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); if (!package_info) { - debug(&_debugger, "Package %s not found in any registry.", slug); + logger_error("package-installer", "Package %s not found in any registry.", package_id); return -1; } @@ -183,14 +184,11 @@ int clib_package_install_executable(clib_package_t *pkg, const char *dir, int ve return -1; } - E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, - pkg->version); - + E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, pkg->version); E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); - E_FORMAT(&tarball, "%s/%s", tmp, file); - rc = http_get_file_shared(url, tarball, clib_package_curl_share, NULL, 0); + rc = http_get_file(url, tarball, NULL, 0); if (0 != rc) { if (verbose) { @@ -426,7 +424,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { // fetch makefile if (!package_opts.global && pkg->makefile) { _debug("fetch: %s/%s", pkg->repo, pkg->makefile); - repository_file_handle_t handle = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->name), pkg->version, pkg->makefile, pkg_dir); + repository_file_handle_t handle = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->repo_name), pkg->version, pkg->makefile, pkg_dir); if (handle == NULL) { goto cleanup; } @@ -532,7 +530,9 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { #ifdef HAVE_PTHREADS pthread_mutex_lock(&lock.mutex); #endif - clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); + if (!package_opts.skip_cache) { + clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); + } #ifdef HAVE_PTHREADS pthread_mutex_unlock(&lock.mutex); #endif @@ -548,9 +548,10 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { goto cleanup; } - if (0 == rc && pkg->install) { - rc = clib_package_install_executable(pkg, dir, verbose); - } + // TODO, check if we want to enable this. + //if (pkg->install) { + // rc = clib_package_install_executable(pkg, dir, verbose); + //} if (0 == rc) { rc = clib_package_install_dependencies(pkg, dir, verbose); diff --git a/src/common/clib-package.c b/src/common/clib-package.c index b9a87057..ed87d9a9 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -88,8 +88,6 @@ static inline char *json_object_get_string_safe(JSON_Object *, const char *); static inline char *json_array_get_string_safe(JSON_Array *, int); -static inline char *clib_package_file_url(const char *, const char *); - static inline list_t *parse_package_deps(JSON_Object *); @@ -168,26 +166,6 @@ static inline char *json_array_get_string_safe(JSON_Array *array, int index) { return strdup(val); } -/** - * Build a URL for `file` of the package belonging to `url` - */ - -static inline char *clib_package_file_url(const char *url, const char *file) { - if (!url || !file) - return NULL; - - int size = strlen(url) + 1 // / - + strlen(file) + 1 // \0 - ; - - char *res = malloc(size); - if (res) { - memset(res, 0, size); - sprintf(res, "%s/%s", url, file); - } - return res; -} - /** * Build a slug */ @@ -216,11 +194,11 @@ clib_package_t *clib_package_load_from_manifest(const char *manifest, clib_package_t *pkg = NULL; if (-1 == fs_exists(manifest)) { - logger_error("error", "Missing %s", manifest); + _debug("clib-package", "%s not found", manifest); return NULL; } - logger_info("info", "reading local %s", manifest); + _debug("clib-package", "reading local %s", manifest); char *json = fs_read(manifest); if (NULL == json) @@ -490,13 +468,11 @@ clib_package_t *clib_package_new(const char *json, int verbose) { return pkg; } -static clib_package_t * -clib_package_new_from_slug_with_package_name(const char *slug, const char* url, int verbose, const char *file) { +static clib_package_t * clib_package_new_from_slug_with_package_name(const char *slug, const char* url, int verbose, const char *manifest_file) { char *author = NULL; char *name = NULL; char *version = NULL; char *json_url = NULL; - char *repo = NULL; char *json = NULL; char *log = NULL; http_get_response_t *res = NULL; @@ -548,65 +524,59 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, } else { _debug("Fetching package manifest for %s", slug); // clean up when retrying - res = repository_fetch_package_manifest(url, clib_package_get_id(author, name), version); + res = repository_fetch_package_manifest(url, clib_package_get_id(author, name), version, manifest_file); - json = res->data; _debug("status: %d", res->status); - if (!res || !res->ok) { + if (!res || (res->status != 200 && res->status != 404)) { goto download; + } else if (res->status == 404) { + goto error; } + json = res->data; log = "fetch"; } } if (verbose) { - logger_info(log, "%s/%s:%s", author, name, file); + logger_info(log, "%s/%s:%s", author, name, manifest_file); } free(json_url); json_url = NULL; - free(name); - name = NULL; - if (json) { - // build package - pkg = clib_package_new(json, verbose); - } - - // Set the url so that we can download the other files. - pkg->url = url; + // build package + pkg = clib_package_new(json, verbose); if (!pkg) goto error; - // force version number if the registry returned a different version from what we expected. + // Set the url so that we can download the other files. + pkg->url = strdup(url); + + // force the supplied version number if the registry returned a different version from what we expected. if (pkg->version) { - if (version) { if (0 != strcmp(version, DEFAULT_REPO_VERSION) && 0 != strcmp(pkg->version, version)) { _debug("forcing version number: %s (%s)", version, pkg->version); free(pkg->version); pkg->version = version; } else { free(version); + version = NULL; } - } } else { pkg->version = version; } - // force package author (don't know how this could fail) - if (author && pkg->author) { - if (0 != strcmp(author, pkg->author)) { - free(pkg->author); - pkg->author = author; - } else { - free(author); - } - } else { + if (!pkg->author) { pkg->author = strdup(author); } + if (!pkg->repo_name) { + pkg->repo_name = strdup(name); + } + free(name); + name = NULL; - if (!(repo = clib_package_get_id(pkg->author, pkg->name))) { + if (!(pkg->repo = clib_package_get_id(pkg->author, pkg->repo_name))) { goto error; } @@ -614,7 +584,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, pthread_mutex_lock(&lock.mutex); #endif // cache json - if (pkg && pkg->author && pkg->name && pkg->version) { + if (pkg->author && pkg->name && pkg->version) { if (-1 == clib_cache_save_json(pkg->author, pkg->name, pkg->version, json)) { _debug("failed to cache JSON for: %s/%s@%s", pkg->author, pkg->name, @@ -640,16 +610,17 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, error: if (0 == retries) { - if (verbose && author && name && file) { - logger_warn("warning", "unable to fetch %s/%s:%s", author, name, file); + if (verbose && author && name && manifest_file) { + logger_warn("warning", "unable to fetch %s/%s:%s", author, name, manifest_file); } } free(author); free(name); - free(version); + if (version != NULL) { + free(version); + } free(json_url); - free(repo); if (!res && json) free(json); if (res) @@ -664,14 +635,13 @@ clib_package_new_from_slug_with_package_name(const char *slug, const char* url, */ clib_package_t *clib_package_new_from_slug_and_url(const char *slug, const char* url, int verbose) { clib_package_t *package = NULL; - const char *name = NULL; unsigned int i = 0; do { - name = manifest_names[i]; - package = clib_package_new_from_slug_with_package_name(slug, url, verbose, name); + const char *manifest_name = manifest_names[i]; + package = clib_package_new_from_slug_with_package_name(slug, url, verbose, manifest_name); if (NULL != package) { - package->filename = (char *)name; + package->filename = (char *) manifest_name; } } while (NULL != manifest_names[++i] && NULL == package); diff --git a/src/common/clib-package.h b/src/common/clib-package.h index 42062a42..c6e4ce9d 100644 --- a/src/common/clib-package.h +++ b/src/common/clib-package.h @@ -65,10 +65,6 @@ clib_package_t *clib_package_load_from_manifest(const char *, int); clib_package_t *clib_package_load_local_manifest(int); -char *clib_package_url(const char *, const char *, const char *); - -char *clib_package_url_from_repo(const char *repo, const char *version); - char *clib_package_parse_version(const char *); char *clib_package_parse_author(const char *); diff --git a/src/common/clib-secrets.c b/src/common/clib-secrets.c index 84a9fd53..21be938a 100644 --- a/src/common/clib-secrets.c +++ b/src/common/clib-secrets.c @@ -6,6 +6,7 @@ #include #include #include +#include struct clib_secret { char *hostname; @@ -18,7 +19,7 @@ struct clib_secret_handle { clib_secrets_t clib_secrets_load_from_file(const char *file) { if (-1 == fs_exists(file)) { - logger_warn("warning", "Secrets file %s does not exist.", file); + //logger_warn("warning", "Secrets file %s does not exist.", file); return NULL; } @@ -32,7 +33,7 @@ clib_secrets_t clib_secrets_load_from_file(const char *file) { JSON_Value *root = json_parse_string(json); if (root == NULL) { - logger_error("error", "unable to parse JSON"); + logger_error("error", "unable to parse secrets JSON"); return NULL; } diff --git a/src/common/url.c b/src/common/url.c deleted file mode 100644 index f1e4ded6..00000000 --- a/src/common/url.c +++ /dev/null @@ -1,6 +0,0 @@ -#define URL_H_IMPLEMENTATION -#define URL_H_NO_STRDUP -#include "url/url.h" - -// This is a bit of a hack to use url.h in multiple files. -// It is better if we just split url.h \ No newline at end of file diff --git a/src/registry/github-registry.c b/src/registry/github-registry.c index 305d45ed..a19b42d0 100644 --- a/src/registry/github-registry.c +++ b/src/registry/github-registry.c @@ -16,118 +16,120 @@ #include "substr/substr.h" #include "trim/trim.h" #include +#include #include #define GITHUB_BASE_URL "https://github.com/" -/** - * Add `href` to the given `package`. - */ -static void add_package_href(registry_package_ptr_t self) { - size_t len = strlen(self->id) + 20; // https://github.com/ \0 - self->href = malloc(len); - if (self->href) - sprintf(self->href, GITHUB_BASE_URL "%s", self->id); -} - /** * Parse the given wiki `li` into a package. */ static registry_package_ptr_t parse_li(GumboNode *li) { registry_package_ptr_t self = registry_package_new(); - char *text = NULL; - - if (!self) goto cleanup; - - text = gumbo_text_content(li); - if (!text) goto cleanup; - - // TODO support unicode dashes - char *tok = strstr(text, " - "); - if (!tok) goto cleanup; - - int pos = tok - text; - self->id = substr(text, 0, pos); - self->description = substr(text, pos + 3, -1); - if (!self->id || !self->description) goto cleanup; - trim(self->description); - trim(self->id); - - add_package_href(self); - - cleanup: - free(text); - return self; + char *text = NULL; + + if (!self) + goto cleanup; + + text = gumbo_text_content(li); + if (!text) + goto cleanup; + + // TODO support unicode dashes + char *tok = strstr(text, " - "); + if (!tok) + goto cleanup; + + int pos = tok - text; + self->id = substr(text, 0, pos); + self->description = substr(text, pos + 3, -1); + if (!self->id || !self->description) + goto cleanup; + trim(self->description); + trim(self->id); + + size_t len = strlen(self->id) + 20;// https://github.com/ \0 + self->href = malloc(len); + if (!self->href) + goto cleanup; + sprintf(self->href, GITHUB_BASE_URL "%s", self->id); + +cleanup: + free(text); + return self; } /** * Parse a list of packages from the given `html` */ list_t *wiki_registry_parse(const char *html) { - GumboOutput *output = gumbo_parse(html); - list_t *pkgs = list_new(); - - GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); - if (body) { - // grab all category `

`s - list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); - list_node_t *heading_node; - list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); - while ((heading_node = list_iterator_next(heading_iterator))) { - GumboNode *heading = (GumboNode *) heading_node->val; - char *category = gumbo_text_content(heading); - // die if we failed to parse a category, as it's - // almost certinaly a malloc error - if (!category) break; - trim(case_lower(category)); - GumboVector *siblings = &heading->parent->v.element.children; - size_t pos = heading->index_within_parent; - - // skip elements until the UL - // TODO: don't hardcode position here - // 2: - // 1 - whitespace - // 2 - actual node - GumboNode *ul = siblings->data[pos + 2]; - if (GUMBO_TAG_UL != ul->v.element.tag) { - free(category); - continue; - } - - list_t *lis = gumbo_get_elements_by_tag_name("li", ul); - list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); - list_node_t *li_node; - while ((li_node = list_iterator_next(li_iterator))) { - registry_package_ptr_t package = parse_li(li_node->val); - if (package && package->description) { - package->category = strdup(category); - list_rpush(pkgs, list_node_new(package)); - } else { - // failed to parse package - if (package) - registry_package_free(package); - } - } - list_iterator_destroy(li_iterator); - list_destroy(lis); - free(category); + GumboOutput *output = gumbo_parse(html); + list_t *pkgs = list_new(); + + GumboNode *body = gumbo_get_element_by_id("wiki-body", output->root); + if (body) { + // grab all category `

`s + list_t *h2s = gumbo_get_elements_by_tag_name("h2", body); + list_node_t *heading_node; + list_iterator_t *heading_iterator = list_iterator_new(h2s, LIST_HEAD); + while ((heading_node = list_iterator_next(heading_iterator))) { + GumboNode *heading = (GumboNode *) heading_node->val; + char *category = gumbo_text_content(heading); + // die if we failed to parse a category, as it's + // almost certinaly a malloc error + if (!category) + break; + trim(case_lower(category)); + GumboVector *siblings = &heading->parent->v.element.children; + size_t pos = heading->index_within_parent; + + // skip elements until the UL + // TODO: don't hardcode position here + // 2: + // 1 - whitespace + // 2 - actual node + GumboNode *ul = siblings->data[pos + 2]; + if (GUMBO_TAG_UL != ul->v.element.tag) { + free(category); + continue; + } + + list_t *lis = gumbo_get_elements_by_tag_name("li", ul); + list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); + list_node_t *li_node; + while ((li_node = list_iterator_next(li_iterator))) { + registry_package_ptr_t package = parse_li(li_node->val); + if (package && package->description) { + package->category = strdup(category); + list_rpush(pkgs, list_node_new(package)); + } else { + // failed to parse package + if (package) + logger_error("error", "Github registry could not parse entry:", li_node->val); + registry_package_free(package); } - list_iterator_destroy(heading_iterator); - list_destroy(h2s); + } + list_iterator_destroy(li_iterator); + list_destroy(lis); + free(category); } + list_iterator_destroy(heading_iterator); + list_destroy(h2s); + } - gumbo_destroy_output(&kGumboDefaultOptions, output); - return pkgs; + gumbo_destroy_output(&kGumboDefaultOptions, output); + return pkgs; } /** * Get a list of packages from the given GitHub wiki `url`. */ list_t *github_registry_fetch(const char *url) { - http_get_response_t *res = http_get(url, NULL, 0); - if (!res->ok) return NULL; + http_get_response_t *res = http_get(url, NULL, 0); + if (!res->ok) + return NULL; - list_t *list = wiki_registry_parse(res->data); - http_get_free(res); - return list; + list_t *list = wiki_registry_parse(res->data); + http_get_free(res); + return list; } diff --git a/src/registry/registry.c b/src/registry/registry.c index e2dc595c..c450d28a 100644 --- a/src/registry/registry.c +++ b/src/registry/registry.c @@ -12,6 +12,7 @@ #include "registry-internal.h" #include "url/url.h" #include +#include #include #include diff --git a/src/repository/repository.c b/src/repository/repository.c index 050421f6..968eb62e 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -65,6 +65,7 @@ static void init_curl_share() { } void repository_init(clib_secrets_t _secrets) { + init_curl_share(); secrets = _secrets; } @@ -78,15 +79,12 @@ static char *repository_create_url_for_file(const char *package_url, const char } } -http_get_response_t *repository_fetch_package_manifest(const char *package_url, const char *package_id, const char *version) { - init_curl_share(); - +http_get_response_t *repository_fetch_package_manifest(const char *package_url, const char *package_id, const char *version, const char* manifest_file) { // Check if there is a secret for the requested repository. url_data_t *parsed = url_parse(package_url); char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); url_free(parsed); - char *manifest_name = "package.json"; - char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_name, secret); + char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_file, secret); http_get_response_t *res; if (strstr(package_url, "gitlab") != NULL) { @@ -95,17 +93,15 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url, char *authentication_header = malloc(size); snprintf(authentication_header, size, "%s:%s", key, secret); - res = http_get(manifest_url, &authentication_header, 1); + res = http_get_shared(manifest_url, clib_package_curl_share, &authentication_header, 1); } else { - res = http_get(manifest_url, NULL, 0); + res = http_get_shared(manifest_url, clib_package_curl_share, NULL, 0); } return res; } repository_file_handle_t repository_download_package_file(const char *package_url, const char *package_id, const char *version, const char *file_path, const char *destination_path) { - init_curl_share(); - // Check if there is a secret for the requested repository. url_data_t *parsed = url_parse(package_url); char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); @@ -149,7 +145,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char #endif if (package_opts.force || -1 == fs_exists(path)) { - logger_info("fetch", "%s:%s", url, file); + _debug("repository", "fetching %s", url); fflush(stdout); #ifdef HAVE_PTHREADS @@ -177,7 +173,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char #ifdef HAVE_PTHREADS pthread_mutex_lock(&mutex); #endif - logger_error("error", "unable to fetch %s:%s", url, file); + logger_error("error", "unable to fetch %s", url); fflush(stderr); rc = 1; #ifdef HAVE_PTHREADS @@ -190,7 +186,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char #ifdef HAVE_PTHREADS pthread_mutex_lock(&mutex); #endif - logger_info("save", path); + _debug("repository", "saved %s", path); fflush(stdout); #ifdef HAVE_PTHREADS pthread_mutex_unlock(&mutex); diff --git a/src/repository/repository.h b/src/repository/repository.h index f433afba..063a7227 100644 --- a/src/repository/repository.h +++ b/src/repository/repository.h @@ -24,7 +24,7 @@ void repository_init(clib_secrets_t secrets); * * @return a handle on success and NULL on failure. */ -http_get_response_t* repository_fetch_package_manifest(const char*package_url, const char* slug, const char* version); +http_get_response_t* repository_fetch_package_manifest(const char*package_url, const char* slug, const char* version, const char* manifest_file); /** * Start download of a file for the package with url. diff --git a/test.sh b/test.sh index 3e002f91..633985da 100755 --- a/test.sh +++ b/test.sh @@ -27,13 +27,4 @@ fi cd ../../ -printf "\nRunning clib cache tests\n\n" -cd test/cache - -if ! make test; then - EXIT_CODE=1 -fi - -cd ../../ - exit $EXIT_CODE diff --git a/test/cache/cache-test.c b/test/cache/cache-test.c index bb2a42aa..48332796 100644 --- a/test/cache/cache-test.c +++ b/test/cache/cache-test.c @@ -30,24 +30,23 @@ int main() { rimraf(clib_cache_dir()); - describe("clib-cache opearions") { + describe("clib-cache operations") { char *author = "author"; char *name = "pkg"; char *version = "1.2.0"; char pkg_dir[BUFSIZ]; - int expiraton = 1; + int expiration = 2; it("should initialize succesfully") { - assert_equal(0, clib_cache_init(expiraton)); + assert_equal(0, clib_cache_init(expiration)); } sprintf(pkg_dir, "%s/author_pkg_1.2.0", clib_cache_dir()); it("should manage the package cache") { - assert_equal( - 0, clib_cache_save_package(author, name, version, "../../deps/copy")); + assert_equal(0, clib_cache_save_package(author, name, version, "./deps/copy")); assert_equal(1, clib_cache_has_package(author, name, version)); assert_equal(0, clib_cache_is_expired_package(author, name, version)); @@ -97,7 +96,7 @@ int main() { assert_equal(13, clib_cache_save_search("")); assert_equal(1, clib_cache_has_search()); - sleep(expiraton + 1); + sleep(expiration + 1); assert_equal(0, clib_cache_has_search()); } diff --git a/test/install-deps-from-package-json.sh b/test/install-deps-from-package-json.sh index e32e13e6..08fafab6 100755 --- a/test/install-deps-from-package-json.sh +++ b/test/install-deps-from-package-json.sh @@ -10,7 +10,8 @@ mkdir -p tmp cd tmp || exit # see https://github.com/clibs/clib/issues/45 -cat > package.json << EOF +# emtter.c does not exist, we test that clib gives an error for it. +cat > clib.json << EOF { "dependencies": { "linenoise": "*", diff --git a/test/install-multiple-clibs-libs.sh b/test/install-multiple-clibs-libs.sh index 85a66c43..471f666e 100755 --- a/test/install-multiple-clibs-libs.sh +++ b/test/install-multiple-clibs-libs.sh @@ -5,16 +5,16 @@ throw() { exit 1 } -clib install -c -o tmp ms file hash > /dev/null || +clib install -c -o tmp ms jwerle/fs.c hash > /dev/null || throw "expecting successful exit code" -[ -d ./tmp/ms ] && [ -f ./tmp/ms/package.json ] || +{ [ -d ./tmp/ms ] && [ -f ./tmp/ms/package.json ]; } || throw "failed to install ms" -[ -d ./tmp/file ] && [ -f ./tmp/file/package.json ] || - throw "failed to install file" +{ [ -d ./tmp/fs ] && [ -f ./tmp/fs/clib.json ]; } || + throw "failed to install fs" -[ -d ./tmp/hash ] && [ -f ./tmp/hash/package.json ] || +{ [ -d ./tmp/hash ] && [ -f ./tmp/hash/package.json ]; } || throw "failed to install hash" rm -rf ./tmp diff --git a/test/install-save.sh b/test/install-save.sh index 4534fe2e..97645346 100755 --- a/test/install-save.sh +++ b/test/install-save.sh @@ -3,19 +3,19 @@ mkdir -p tmp/test-save cp test/data/test-save-package.json tmp/test-save/package.json cd tmp/test-save || exit -../../clib-install -c --save stephenmathieson/tabs-to-spaces@1.0.0 >/dev/null -../../clib-install -c -S darthtrevino/str-concat@0.0.2 >/dev/null +../../clib-install -c --save clibs/buffer@0.4.0 >/dev/null +../../clib-install -c -S clibs/strdup@0.1.0 >/dev/null ../../clib-install -c --save-dev jwerle/fs.c@0.1.1 >/dev/null -../../clib-install -c -D clibs/parson@1.0.2 >/dev/null -cd - || exit +../../clib-install -c -D clibs/list@0.2.0 >/dev/null +cd - >/dev/null || exit -if ! grep --quiet "stephenmathieson/tabs-to-spaces" tmp/test-save/package.json; then - echo >&2 "Failed to find stephenmathieson/tabs-to-spaces saved in package.json" +if ! grep --quiet "clibs/buffer" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/buffer.json" exit 1 fi -if ! grep --quiet "darthtrevino/str-concat" tmp/test-save/package.json; then - echo >&2 "Failed to find darthtrevino/strconcat saved in package.json" +if ! grep --quiet "clibs/strdup" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/strdup saved in package.json" exit 1 fi @@ -24,7 +24,7 @@ if ! grep --quiet "jwerle/fs.c" tmp/test-save/package.json; then exit 1 fi -if ! grep --quiet "clibs/parson" tmp/test-save/package.json; then - echo >&2 "Failed to find clibs/parson saved in package.json" +if ! grep --quiet "clibs/list" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/list saved in package.json" exit 1 fi diff --git a/test/package/Makefile b/test/package/Makefile index da68c925..47f36ccd 100644 --- a/test/package/Makefile +++ b/test/package/Makefile @@ -3,14 +3,14 @@ CC ?= cc VALGRIND ?= valgrind TEST_RUNNER ?= -SRC = ../../src/common/clib-package.c ../../src/common/clib-cache.c ../../src/common/clib-release-info.c ../../src/common/clib-settings.c +SRC = $(wildcard ../../src/*/*.c) DEPS += $(wildcard ../../deps/*/*.c) OBJS = $(SRC:.c=.o) $(DEPS:.c=.o) TEST_SRC = $(wildcard *.c) TEST_OBJ = $(TEST_SRC:.c=.o) TEST_BIN = $(TEST_SRC:.c=) -CFLAGS += -std=c99 -Wall -I../../src/common -I../../deps -DHAVE_PTHREADS -pthread -g +CFLAGS += -std=c99 -Wall -I../../src/common -I../../src/registry -I../../src/repository -I../../deps -DHAVE_PTHREADS -pthread -g LDFLAGS = -lcurl VALGRIND_OPTS ?= --leak-check=full --error-exitcode=3 diff --git a/test/package/package-install-dependencies.c b/test/package/package-install-dependencies.c index 0904917d..911d154c 100644 --- a/test/package/package-install-dependencies.c +++ b/test/package/package-install-dependencies.c @@ -3,6 +3,8 @@ #include "describe/describe.h" #include "fs/fs.h" #include "rimraf/rimraf.h" +#include "registry-manager.h" +#include "repository.h" int main() { curl_global_init(CURL_GLOBAL_ALL); @@ -12,6 +14,10 @@ int main() { .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + describe("clib_package_install_dependencies") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install_dependencies(NULL, "./deps", 0)); @@ -19,7 +25,7 @@ int main() { it("should install the dep in its own directory") { clib_package_t *dep = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(dep); assert(0 == clib_package_install_dependencies(dep, "./test/fixtures/", 0)); @@ -31,7 +37,7 @@ int main() { it("should install the dependency's package.json") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); @@ -42,7 +48,7 @@ int main() { it("should install the dependency's sources") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); @@ -54,7 +60,7 @@ int main() { it("should install the dependency's dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/rimraf.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/rimraf.c", "https://github.com/stephanmathieson/rimraf.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); diff --git a/test/package/package-install-dev-dependencies.c b/test/package/package-install-dev-dependencies.c index d23bab78..184070b8 100644 --- a/test/package/package-install-dev-dependencies.c +++ b/test/package/package-install-dev-dependencies.c @@ -3,6 +3,8 @@ #include "describe/describe.h" #include "fs/fs.h" #include "rimraf/rimraf.h" +#include "registry-manager.h" +#include "repository.h" int main() { curl_global_init(CURL_GLOBAL_ALL); @@ -12,6 +14,10 @@ int main() { .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + describe("clib_package_install_development") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install_development(NULL, "./deps", 0)); @@ -19,7 +25,7 @@ int main() { it("should return -1 when given a bad dir") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(-1 == clib_package_install_development(pkg, NULL, 0)); clib_package_free(pkg); @@ -27,7 +33,7 @@ int main() { it("should install the package's development dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/trim.c@0.0.2", 0); + clib_package_new_from_slug_and_url("stephenmathieson/trim.c@0.0.2", "https://github.com/stephanmathieson/trim.c", 0); assert(pkg); assert(0 == clib_package_install_development(pkg, "./test/fixtures", 0)); assert(0 == fs_exists("./test/fixtures/describe")); diff --git a/test/package/package-install.c b/test/package/package-install.c index 1ef464f9..fc931111 100644 --- a/test/package/package-install.c +++ b/test/package/package-install.c @@ -4,15 +4,24 @@ #include "fs/fs.h" #include "rimraf/rimraf.h" #include +#include "registry-manager.h" +#include "repository.h" +#include "clib-package-installer.h" +#include "strdup/strdup.h" int main() { curl_global_init(CURL_GLOBAL_ALL); - clib_package_set_opts((clib_package_opts_t){ + clib_package_set_opts((clib_package_opts_t) { .skip_cache = 1, .prefix = 0, .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + repository_init(NULL); + describe("clib_package_install") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install(NULL, "./deps", 0)); @@ -20,7 +29,7 @@ int main() { it("should install the pkg in its own directory") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/")); @@ -31,7 +40,7 @@ int main() { it("should install the package's clib.json or package.json") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 1)); assert(0 == fs_exists("./test/fixtures/case/package.json") || @@ -42,7 +51,7 @@ int main() { it("should install the package's sources") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/case/case.c")); @@ -53,7 +62,7 @@ int main() { it("should install the package's dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c@master", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c@master", "https://github.com/stephenmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/path-normalize/")); @@ -67,7 +76,7 @@ int main() { it("should not install the package's development dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/trim.c@0.0.2", 0); + clib_package_new_from_slug_and_url("stephenmathieson/trim.c@0.0.2", "https://github.com/stephenmathieson/trim.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(-1 == fs_exists("./test/fixtures/describe/")); @@ -79,18 +88,19 @@ int main() { rimraf("./test/fixtures"); } + /* This test is currently not feasible because clib has too many dependencies with issues. I propose we re-enable this test at the next release. it("should install itself") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/clib-package@0.4.2", 0); + clib_package_new_from_slug_and_url("clibs/clib@2.7.0", "https://github.com/clibs/clib", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/")); - assert(0 == fs_exists("./test/fixtures/clib-package/")); - assert(0 == fs_exists("./test/fixtures/clib-package/clib-package.c")); - assert(0 == fs_exists("./test/fixtures/clib-package/clib-package.h")); - assert(0 == fs_exists("./test/fixtures/clib-package/package.json") || - 0 == fs_exists("./test/fixtures/clib-package/clib.json")); + assert(0 == fs_exists("./test/fixtures/clib/")); + assert(0 == fs_exists("./test/fixtures/clib/clib-package.c")); + assert(0 == fs_exists("./test/fixtures/clib/clib-package.h")); + assert(0 == fs_exists("./test/fixtures/clib/package.json") || + 0 == fs_exists("./test/fixtures/clib/clib.json")); assert(0 == fs_exists("./test/fixtures/http-get/")); assert(0 == fs_exists("./test/fixtures/http-get/http-get.c")); @@ -144,7 +154,7 @@ int main() { clib_package_free(pkg); rimraf("./test/fixtures"); - } + } */ } curl_global_cleanup(); diff --git a/test/package/package-new-from-slug.c b/test/package/package-new-from-slug.c index 60830f45..adda0bcb 100644 --- a/test/package/package-new-from-slug.c +++ b/test/package/package-new-from-slug.c @@ -7,20 +7,20 @@ int main() { describe("clib_package_new_from_slug") { it("should return NULL when given a bad slug") { - assert(NULL == clib_package_new_from_slug(NULL, 0)); + assert(NULL == clib_package_new_from_slug_and_url(NULL, NULL, 0)); } it("should return NULL when given a slug missing a name") { - assert(NULL == clib_package_new_from_slug("author/@version", 0)); + assert(NULL == clib_package_new_from_slug_and_url("author/@version", "https://github.com/author/", 0)); } it("should return NULL when given slug which doesn't resolve") { - assert(NULL == clib_package_new_from_slug("abc11234", 0)); + assert(NULL == clib_package_new_from_slug_and_url("abc11234", "https://github.com/abc11234", 0)); } it("should build the correct package") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephanmathieson/case.c", 0); assert(pkg); assert_str_equal("case", pkg->name); assert_str_equal("0.1.0", pkg->version); @@ -32,7 +32,7 @@ int main() { it("should force package version numbers") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c@0.0.1", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c@0.0.1", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert_str_equal("0.0.1", pkg->version); clib_package_free(pkg); @@ -40,15 +40,14 @@ int main() { it("should use package version if version not provided") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert_str_equal("0.1.5", pkg->version); clib_package_free(pkg); } it("should save the package's json") { - clib_package_t *pkg = clib_package_new_from_slug( - "stephenmathieson/str-replace.c@8ca90fb", 0); + clib_package_t *pkg = clib_package_new_from_slug_and_url("stephenmathieson/str-replace.c@8ca90fb", "https://github.com/stephanmathieson/str-replace.c", 0); assert(pkg); assert(pkg->json); diff --git a/test/package/package-url.c b/test/package/package-url.c deleted file mode 100644 index bed9b60c..00000000 --- a/test/package/package-url.c +++ /dev/null @@ -1,28 +0,0 @@ - -#include "clib-package.h" -#include "describe/describe.h" - -int main() { - describe("clib_package_url") { - it("should return NULL when given a bad author") { - assert(NULL == clib_package_url(NULL, "name", "version")); - } - - it("should return NULL when given a bad name") { - assert(NULL == clib_package_url("author", NULL, "version")); - } - - it("should return NULL when given a bad version") { - assert(NULL == clib_package_url("author", "name", NULL)); - } - - it("should build a GitHub url") { - char *url = clib_package_url("author", "name", "version"); - assert_str_equal("https://raw.githubusercontent.com/author/name/version", - url); - free(url); - } - } - - return assert_failures(); -} From be8efe85edb48c2692a93b06d6f72007537107c8 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sat, 3 Apr 2021 23:53:17 +0200 Subject: [PATCH 10/15] build: Fix building for windows. --- .gitignore | 1 + deps/strdup/strdup.c | 15 ++++++++++++++- deps/strdup/strdup.h | 6 +++--- src/clib-search.c | 2 +- src/common/clib-package-installer.c | 7 +++++++ src/registry/gitlab-registry.c | 6 +++--- src/repository/repository.c | 1 + 7 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index fa21d235..36c912cf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ clib-build clib-update clib-upgrade clib-uninstall +*.exe test/package/package-* !test/package/package-*.c diff --git a/deps/strdup/strdup.c b/deps/strdup/strdup.c index f34bd5d9..4abda28e 100644 --- a/deps/strdup/strdup.c +++ b/deps/strdup/strdup.c @@ -8,9 +8,9 @@ #ifndef HAVE_STRDUP +#include "strdup.h" #include #include -#include "strdup.h" #ifndef strdup @@ -30,6 +30,19 @@ strdup(const char *str) { return buf; } +char* strndup(const char *str, size_t len) { + if (NULL == (char *) str) { + return NULL; + } + + char *buf = malloc(len+1); + + if (buf) { + memset(buf, 0, len+1); + memcpy(buf, str, len); + } + return buf; +} #endif #endif /* HAVE_STRDUP */ diff --git a/deps/strdup/strdup.h b/deps/strdup/strdup.h index cd75dd94..6a3bae38 100644 --- a/deps/strdup/strdup.h +++ b/deps/strdup/strdup.h @@ -17,10 +17,10 @@ * Returns a pointer to the newly allocated * copy of `str`, or `NULL` on failure. */ - +#include #ifndef strdup -char * -strdup(const char *str); +char * strdup(const char *str); +char * strndup(const char *str, size_t len); #endif #endif /* HAVE_STRDUP */ diff --git a/src/clib-search.c b/src/clib-search.c index 6974aaa3..0351f95c 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -182,7 +182,7 @@ int main(int argc, char *argv[]) { clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); clib_package_t *package = clib_package_load_local_manifest(0); - registries_t registries = registry_manager_init_registries(package->registries, secrets); + registries_t registries = registry_manager_init_registries(package ? package->registries : NULL, secrets); registry_manager_fetch_registries(registries); // TODO, implement caching for the new registries. diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index 5b7888fc..c8003329 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -23,6 +23,12 @@ #include #include +#if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ + defined(__MINGW64__) || defined(__CYGWIN__) +#define setenv(k, v, _) _putenv_s(k, v) +#define realpath(a, b) _fullpath(a, b, strlen(a)) +#endif + CURLSH *clib_package_curl_share; //TODO, cleanup somewhere curl_share_cleanup(clib_package_curl_share); @@ -375,6 +381,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { _debug("mkdir -p %s", pkg_dir); // create directory for pkg if (-1 == mkdirp(pkg_dir, 0777)) { + logger_error("error", "Could not create directory %s", pkg_dir); rc = -1; goto cleanup; } diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c index 2c899f8a..07cf4203 100644 --- a/src/registry/gitlab-registry.c +++ b/src/registry/gitlab-registry.c @@ -5,11 +5,11 @@ // MIT licensed // #include "gitlab-registry.h" -#include "gumbo-get-element-by-id/get-element-by-id.h" #include "http-get/http-get.h" #include "registry-internal.h" #include #include +#include /** * Parse a list of packages from the given `html` @@ -19,9 +19,9 @@ static list_t *gitlab_registry_parse(const char *hostname, const char *html) { // Try to parse the markdown file. char *input = strdup(html); - char *line; + char *line = input; char *category = NULL; - while ((line = strsep(&input, "\n"))) { + while ((line = strtok(line, "\n"))) { char *dash_position = strstr(line, "-"); // The line starts with a dash, so we expect a package. if (dash_position != NULL && dash_position - line < 4) { diff --git a/src/repository/repository.c b/src/repository/repository.c index 968eb62e..28197667 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -206,6 +206,7 @@ static void *fetch_package_file_thread(void *arg) { int rc = fetch_package_file_work(data->url, data->dir, data->file, data->secret); *status = rc; pthread_exit((void *) status); + return status; } #endif From d0c4e7c39e1f870dc3af35fc682972ee4e69aa8f Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Mon, 5 Apr 2021 01:20:49 +0200 Subject: [PATCH 11/15] docs: Add documentation on how to fetch packages from other sources, add build instructions, cleanup Readme. --- Building.md | 40 ++++++++++++++++ Readme.md | 90 +++++++++++++++--------------------- BEST_PRACTICE.md => Usage.md | 53 +++++++++++++++++---- 3 files changed, 121 insertions(+), 62 deletions(-) create mode 100644 Building.md rename BEST_PRACTICE.md => Usage.md (71%) diff --git a/Building.md b/Building.md new file mode 100644 index 00000000..fc41f0c4 --- /dev/null +++ b/Building.md @@ -0,0 +1,40 @@ +# Building clib from source + +## OSx + +```sh +$ git clone https://github.com/clibs/clib.git /tmp/clib +$ cd /tmp/clib +$ make install +``` + +## Ubuntu or debian + +```sh +# install libcurl +$ sudo apt install libcurl4-gnutls-dev -qq +# clone +$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib +# build +$ make +# put on path +$ sudo make install +``` + +## Windows (crosscompiling from linux) +The docker image contains the mingw toolchain which is used to compile for windows. +Curl needs to be built from source. +```shell +# Download and compile curl +$ docker run --rm dockcross/windows-static-64-posix > dockcross-windows-x64 +$ cat dockcross-windows-x64 +$ chmod +x dockcross-windows-x64 +$ wget https://curl.haxx.se/download/curl-7.76.0.tar.gz +$ tar xzf curl-* +$ CURL_SRC=curl-* +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && ./configure --prefix="/work/deps/curl" --host=x86_64-w64-mingw32.static --with-winssl --disable-dependency-tracking --disable-pthreads --enable-threaded-resolver --disable-imap --disable-pop3 --disable-smpt --disable-ldap --disable-mqtt --disable-smb' +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make' +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make install' +$ git clone https://github.com/clibs/clib.git && cd clib +$ ./dockcross-windows-x64 make all NO_PTHREADS=1 EXE=true +``` \ No newline at end of file diff --git a/Readme.md b/Readme.md index 2bbd882d..f1404303 100644 --- a/Readme.md +++ b/Readme.md @@ -7,58 +7,28 @@ ![c package manager screenshot](https://i.cloudup.com/GwqOU2hh9Y.png) -## Installation - - Expects [libcurl](http://curl.haxx.se/libcurl/) to be installed and linkable. - - With [homebrew](https://github.com/Homebrew/homebrew): - -```sh -$ brew install clib -``` - - Or [MacPorts](https://www.macports.org): - -```sh -$ sudo port selfupdate -$ sudo port install clib -``` - - With git: - -```sh -$ git clone https://github.com/clibs/clib.git /tmp/clib -$ cd /tmp/clib -$ make install -``` +## About - Ubuntu: +Basically the lazy-man's copy/paste promoting smaller C utilities, also +serving as a nice way to discover these sort of libraries. From my experience +C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide +stand-alone "micro" C libraries for developers to quickly install without coupling +to large frameworks. -```sh -# install libcurl -$ sudo apt-get install libcurl4-gnutls-dev -qq -# clone -$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib -# build -$ make -# put on path -$ sudo make install -``` +You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction. -## About +The [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry". The registry is used by `clib(1)` when searching for packages. - Basically the lazy-man's copy/paste promoting smaller C utilities, also - serving as a nice way to discover these sort of libraries. From my experience - C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide - stand-alone "micro" C libraries for developers to quickly install without coupling - to large frameworks. +## Installation and building +Binaries for `clib(1)` releases can be found at [releases](https://github.com/clibs/clib/releases/). +For OSx and linux [libcurl](http://curl.haxx.se/libcurl/) should be installed and linkable. +The windows binaries do not require any libraries to be installed. - You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction. +See [Building](Building.md) for instructions on how to build clib. - The wiki [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry" and populates the `clib-search(1)` results. ## Usage - +More detailed information on how to use `clib` can be found in [Usage](Usage.md). ``` clib [options] @@ -78,13 +48,9 @@ $ sudo make install search [query] Search for packages help Display help for cmd ``` +More information about the Command Line Interface can be found [here](https://github.com/clibs/clib/wiki/Command-Line-Interface). -More about the Command Line Interface [here](https://github.com/clibs/clib/wiki/Command-Line-Interface). - -## Examples - - More examples and best practices at [BEST_PRACTICE.md](https://github.com/clibs/clib/blob/master/BEST_PRACTICE.md). - +### Example usage Install a few dependencies to `./deps`: ```sh @@ -109,10 +75,26 @@ $ clib install ms file hash $ clib install visionmedia/mon visionmedia/every visionmedia/watch ``` -## clib.json - - Example of a clib.json explicitly listing the source: +## Clib.json +Information about a clib project or a package is stored in `clib.json`. +In a project that uses `clib` to install dependencies a typical `clib.json` will only contain the required dependencies. +It may look something like: +```json +{ + "name": "Copy and Paste-Inator", + "version": "0.4.2", + "description": "Creates copies of yourself to do all of his waiting in lines for you.", + "dependencies": { + "clibs/buffer": "0.0.1", + "clibs/term": "0.0.1", + "jwerle/throw.h": "0.0.0" + } +} +``` +Packages that can be installed by `clib` should also provide a `clib.json`. +It contains the files that should be installed by `clib` in `"src"` +An example of a clib.json for a package may look like: ```json { "name": "term", @@ -125,7 +107,7 @@ $ clib install visionmedia/mon visionmedia/every visionmedia/watch } ``` - Example of a clib.json for an executable: +Example of a clib.json for an executable: ```json { diff --git a/BEST_PRACTICE.md b/Usage.md similarity index 71% rename from BEST_PRACTICE.md rename to Usage.md index c2064a60..2bf29c21 100644 --- a/BEST_PRACTICE.md +++ b/Usage.md @@ -3,13 +3,12 @@ This page will cover: - [How to use libraries](#how-to-use-installed-libraries-for-your-project). - - [Example Makefile](#example-makefile). - - [Example `clib.json` for executables](#example-clibjson-for-executable-project). + - [Example Makefile](#example-makefile). + - [Example `clib.json` for executables](#example-packagejson-for-executable-project). - [Making your own library package](#making-your-own-libraries). - - [Example `clib.json` for libraries](#example-clibjson-for-libraries). + - [Example `clib.json` for libraries](#example-packagejson-for-libraries). - [How to install/uninstall executables](#install-and-uninstall-executables-packages). - -For instructions on installation, check out the [README](https://github.com/clibs/clib#installation). +- [How to fetch packages from other sources](#how-to-fetch-packages-from-other-sources). ## How to use installed libraries for your project @@ -116,7 +115,7 @@ at a example `clib.json` file for your project: (executable package) "dependencies": { "stephenmathieson/trim.c": "0.0.2", "clibs/commander": "1.3.2", - "clibs/logger": "0.0.1", + "clibs/logger": "0.0.1" }, "install": "make install", "uninstall": "make uninstall" @@ -129,9 +128,9 @@ _**NOTE:** Make sure you have a release as the same version in your `clib.json` ## Making your own libraries -Now that you know how to use libraries, heres how to make your own: +Now that you know how to use libraries, here is how to make your own: -Like before, heres a typical project directory tree: +Like before, a typical project directory tree: ``` your-library-c/ @@ -230,3 +229,41 @@ $ sudo clib-uninstall visionmedia/mon ```
+ +## How to fetch packages from other sources +By default `clib` uses the [listing of packages](https://github.com/clibs/clib/wiki/Packages) as the place to look for packages, the registry, and [github.com](https://github.com) for downloading packages. +You can specify additional registries and download from other repositories than github. +This might be useful when using `clib` to install a mix of private and public packages. + +### Adding additional registries +Additional registries can be provided in the `clib.json` of a project. +Currently github wiki's and gitlab — both [gitlab.com](https://www.gitlab.com) and self hosted — are supported. +For gitlab the format is a bit complicated as it has to conform to the gitlab api. +You should use the same format as the default registry but in a file in a repository instead of in a wiki. +```json +{ + // other definitions + "registries": [ + "https://gitlab.com/api/v4/projects/25447829/repository/files/README.md/raw?ref=master", + "https://github.com/awesome-org/clib/wiki/Packages" + ] +} +``` + +_**CAUTION:** For gitlab, the url should be of the form `/project/` and not `/group/repo` check [my-gitlab-registry](https://gitlab.com/nouwaarom/my-clib-registry) for an example._ + +### Downloading from gitlab or private sources +To download from some sources, authentication might be required. +To facilitate this `clib_secrets.json` is used to store the credentials. + +```json +{ + "github.com": "GITHUB_API_TOKEN", + "github.example.com": "GITLAB_USER_TOKEN" +} +``` + +Gitlab always requires a secret in order to use the API. +The secret can be obtained by clicking your profile and then (Preferences -> Access Tokens) and create a token with only `read_repository` rights. + +_**TIP:** To prevent accidentally commiting your secrets add `clib_secrets.json` to `.gitignore` and use `clib_secrets.json.dist` to specify for which domains a secret is required._ From d6c4f5690e608f7501ac9115b175284a20142f7b Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Fri, 9 Apr 2021 21:37:42 +0200 Subject: [PATCH 12/15] registry: Fix a bug with parsing gitlab registries, fix downloading from public gitlab repositories misc: Cleanup, download from the correct version when downloading from gitlab. --- src/clib-install.c | 5 ++--- src/clib-update.c | 5 ++--- src/clib-upgrade.c | 5 ++--- src/common/clib-package-installer.c | 32 +++++++---------------------- src/registry/gitlab-registry.c | 27 ++++++++++++++++-------- src/registry/registry-manager.c | 2 +- src/registry/registry.c | 4 ++++ src/registry/registry.h | 9 +++++++- src/repository/gitlab-repository.c | 11 ++++++---- src/repository/repository.c | 20 ++++++++++-------- 10 files changed, 63 insertions(+), 57 deletions(-) diff --git a/src/clib-install.c b/src/clib-install.c index 009d7fc8..ca362e39 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -371,9 +371,7 @@ int main(int argc, char *argv[]) { memset(prefix, 0, path_max); realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *) opts.prefix, 0, size); - memcpy((void *) opts.prefix, prefix, size); + opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); @@ -407,6 +405,7 @@ int main(int argc, char *argv[]) { curl_global_cleanup(); clib_package_cleanup(); + clib_package_free(root_package); command_free(&program); return code; diff --git a/src/clib-update.c b/src/clib-update.c index 43405923..f78db21c 100755 --- a/src/clib-update.c +++ b/src/clib-update.c @@ -16,6 +16,7 @@ #include "logger/logger.h" #include "parson/parson.h" #include "str-replace/str-replace.h" +#include "strdup/strdup.h" #include "version.h" #include #include @@ -339,9 +340,7 @@ int main(int argc, char *argv[]) { memset(prefix, 0, path_max); realpath(package_opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - package_opts.prefix = malloc(size); - memset((void *) package_opts.prefix, 0, size); - memcpy((void *) package_opts.prefix, prefix, size); + package_opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); diff --git a/src/clib-upgrade.c b/src/clib-upgrade.c index b9d0c39b..cb3ccb68 100755 --- a/src/clib-upgrade.c +++ b/src/clib-upgrade.c @@ -16,6 +16,7 @@ #include "logger/logger.h" #include "parson/parson.h" #include "str-replace/str-replace.h" +#include "strdup/strdup.h" #include "tempdir/tempdir.h" #include "version.h" #include @@ -232,9 +233,7 @@ int main(int argc, char *argv[]) { memset(prefix, 0, path_max); realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *) opts.prefix, 0, size); - memcpy((void *) opts.prefix, prefix, size); + opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index c8003329..c6cc1b6e 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -98,7 +98,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); if (!package_info) { logger_error("package-installer", "Package %s not found in any registry.", package_id); - return -1; + goto loop_cleanup; } pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), verbose); @@ -112,6 +112,7 @@ static inline int install_packages(list_t *list, const char *dir, int verbose) { error = 0; loop_cleanup: + free(package_id); if (slug) free(slug); if (error) { @@ -300,7 +301,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { char *package_json = NULL; char *pkg_dir = NULL; char *command = NULL; - int pending = 0; int rc = 0; int i = 0; @@ -485,10 +485,11 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { iterator = list_iterator_new(pkg->src, LIST_HEAD); list_node_t *source; - repository_file_handle_t *handles = malloc(pkg->src->len * sizeof(repository_file_handle_t)); + repository_file_handle_t *handles = malloc(max * sizeof(repository_file_handle_t)); + char* package_id = clib_package_get_id(pkg->author, pkg->repo_name); + // TODO, refactor this. while ((source = list_iterator_next(iterator))) { - handles[i] = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->repo_name), pkg->version, source->val, pkg_dir); - + handles[i] = repository_download_package_file(pkg->url, package_id, pkg->version, source->val, pkg_dir); if (handles[i] == NULL) { list_iterator_destroy(iterator); iterator = NULL; @@ -496,30 +497,13 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { goto cleanup; } -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) - struct timespec ts = {0, 1000 * 1000 * 10}; - nanosleep(&ts, NULL); -#endif - #ifdef HAVE_PTHREADS - if (i < 0) { - i = 0; - } - - (void) pending++; - if (i < max) { - (void) i++; + i++; } else { while (--i >= 0) { repository_file_finish_download(handles[i]); repository_file_free(handles[i]); - (void) pending--; - -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) - struct timespec ts = {0, 1000 * 1000 * 10}; - nanosleep(&ts, NULL); -#endif } } #endif @@ -528,8 +512,6 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { #ifdef HAVE_PTHREADS while (--i >= 0) { repository_file_finish_download(handles[i]); - - (void) pending--; repository_file_free(handles[i]); } #endif diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c index 07cf4203..9aefd92c 100644 --- a/src/registry/gitlab-registry.c +++ b/src/registry/gitlab-registry.c @@ -8,8 +8,17 @@ #include "http-get/http-get.h" #include "registry-internal.h" #include -#include #include +#include + +static char *string_split(char *in, char sep) { + char *next_sep = strchr(in, sep); + if (next_sep == NULL) { + return next_sep; + } + *next_sep = '\0'; + return next_sep + sizeof(char); +} /** * Parse a list of packages from the given `html` @@ -21,7 +30,7 @@ static list_t *gitlab_registry_parse(const char *hostname, const char *html) { char *input = strdup(html); char *line = input; char *category = NULL; - while ((line = strtok(line, "\n"))) { + while ((line = string_split(line, '\n'))) { char *dash_position = strstr(line, "-"); // The line starts with a dash, so we expect a package. if (dash_position != NULL && dash_position - line < 4) { @@ -58,14 +67,14 @@ static list_t *gitlab_registry_parse(const char *hostname, const char *html) { list_t *gitlab_registry_fetch(const char *url, const char *hostname, const char *secret) { http_get_response_t *res; if (secret == NULL) { - return NULL; + res = http_get(url, NULL, 0); + } else { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + res = http_get(url, &authentication_header, 1); } - - char *key = "PRIVATE-TOKEN"; - unsigned int size = strlen(key) + strlen(secret) + 2; - char *authentication_header = malloc(size); - snprintf(authentication_header, size, "%s:%s", key, secret); - res = http_get(url, &authentication_header, 1); if (!res->ok) { return NULL; } diff --git a/src/registry/registry-manager.c b/src/registry/registry-manager.c index eb6163e2..ed78d92d 100644 --- a/src/registry/registry-manager.c +++ b/src/registry/registry-manager.c @@ -42,7 +42,7 @@ void registry_manager_fetch_registries(registries_t registries) { registry_ptr_t reg; while ((reg = registry_iterator_next(it))) { if (!registry_fetch(reg)) { - printf("REGISTRY: could not list packages from. %s\n", registry_get_url(reg)); + printf("REGISTRY: could not list packages from. %s secret is: %s\n", registry_get_url(reg), registry_get_secret(reg)); } } registry_iterator_destroy(it); diff --git a/src/registry/registry.c b/src/registry/registry.c index c450d28a..77084fca 100644 --- a/src/registry/registry.c +++ b/src/registry/registry.c @@ -96,6 +96,10 @@ const char *registry_get_url(registry_ptr_t registry) { return registry->url; } +const char* registry_get_secret(registry_ptr_t registry) { + return registry->secret; +} + bool registry_fetch(registry_ptr_t registry) { switch (registry->type) { case REGISTRY_TYPE_GITLAB: diff --git a/src/registry/registry.h b/src/registry/registry.h index 531c0afa..237e850a 100644 --- a/src/registry/registry.h +++ b/src/registry/registry.h @@ -37,10 +37,17 @@ bool registry_fetch(registry_ptr_t registry); /** * Get the url for the registry * @param registry - * @return + * @return the url */ const char* registry_get_url(registry_ptr_t registry); +/** + * Get the secret for this registry + * @param registry + * @return the secret or NULL if there is no secret. + */ +const char* registry_get_secret(registry_ptr_t registry); + /** * An iterator through the packages in the registry. */ diff --git a/src/repository/gitlab-repository.c b/src/repository/gitlab-repository.c index 0b272f0b..666282c9 100644 --- a/src/repository/gitlab-repository.c +++ b/src/repository/gitlab-repository.c @@ -9,17 +9,20 @@ #include #include #include +#include "str-replace/str-replace.h" -#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=master" +#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=%s" // GET :hostname/api/v4/projects/:id/repository/files/:file_path/raw -char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret) { +char *gitlab_repository_get_url_for_file(const char *package_url, const char *slug, const char *version, const char *file, const char *secret) { url_data_t *parsed = url_parse(package_url); - int size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(file) + 64; + char* encoded_filename = str_replace(file, "/", "%2F"); + + size_t size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(encoded_filename) + strlen(GITLAB_API_V4_URL) + strlen(version) + 1; char *url = malloc(size); if (url) { - snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, file); + snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, encoded_filename, version); } url_free(parsed); diff --git a/src/repository/repository.c b/src/repository/repository.c index 28197667..6fe87b39 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -29,7 +29,7 @@ static debug_t _debugger; }) struct repository_file_t { - const char *url; + char *url; const char *dir; const char *file; const char *secret; @@ -87,7 +87,7 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url, char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_file, secret); http_get_response_t *res; - if (strstr(package_url, "gitlab") != NULL) { + if (secret && strstr(package_url, "gitlab") != NULL) { char *key = "PRIVATE-TOKEN"; unsigned int size = strlen(key) + strlen(secret) + 2; char *authentication_header = malloc(size); @@ -98,6 +98,8 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url, res = http_get_shared(manifest_url, clib_package_curl_share, NULL, 0); } + free(manifest_url); + return res; } @@ -116,7 +118,11 @@ repository_file_handle_t repository_download_package_file(const char *package_ur void repository_file_finish_download(repository_file_handle_t file) { void *rc; - pthread_join(file->thread, &rc); + int success = pthread_join(file->thread, &rc); + if (success != 0) { + printf("Failed to join thread.\n"); + } + free(rc); } void repository_file_free(repository_file_handle_t file) { @@ -133,8 +139,6 @@ static int fetch_package_file_work(const char *url, const char *dir, const char return 1; } - _debug("file URL: %s", url); - if (!(path = path_join(dir, basename(file)))) { rc = 1; goto cleanup; @@ -145,14 +149,14 @@ static int fetch_package_file_work(const char *url, const char *dir, const char #endif if (package_opts.force || -1 == fs_exists(path)) { - _debug("repository", "fetching %s", url); + _debug("fetching %s", url); fflush(stdout); #ifdef HAVE_PTHREADS pthread_mutex_unlock(&mutex); #endif - if (strstr(url, "gitlab") != NULL) { + if (secret && strstr(url, "gitlab") != NULL) { char *key = "PRIVATE-TOKEN"; unsigned int size = strlen(key) + strlen(secret) + 2; char *authentication_header = malloc(size); @@ -186,7 +190,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char #ifdef HAVE_PTHREADS pthread_mutex_lock(&mutex); #endif - _debug("repository", "saved %s", path); + _debug("saved %s", path); fflush(stdout); #ifdef HAVE_PTHREADS pthread_mutex_unlock(&mutex); From d80c2f4f52f8829fa233454d7acb5195215ef2a7 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sun, 25 Apr 2021 15:01:47 +0200 Subject: [PATCH 13/15] test: Add test for installing a dependency from gitlab and loading secrets from a JSON file. --- src/repository/repository.c | 5 +---- test/install-from-gitlab.sh | 30 +++++++++++++++++++++++++++++ test/package/clib_secrets.json | 4 ++++ test/package/package-secrets-load.c | 22 +++++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) create mode 100755 test/install-from-gitlab.sh create mode 100644 test/package/clib_secrets.json create mode 100644 test/package/package-secrets-load.c diff --git a/src/repository/repository.c b/src/repository/repository.c index 6fe87b39..f288ceaa 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -118,10 +118,7 @@ repository_file_handle_t repository_download_package_file(const char *package_ur void repository_file_finish_download(repository_file_handle_t file) { void *rc; - int success = pthread_join(file->thread, &rc); - if (success != 0) { - printf("Failed to join thread.\n"); - } + pthread_join(file->thread, &rc); free(rc); } diff --git a/test/install-from-gitlab.sh b/test/install-from-gitlab.sh new file mode 100755 index 00000000..b171850f --- /dev/null +++ b/test/install-from-gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +throw() { + echo >&2 "$1" + exit 1 +} + +rm -rf tmp +mkdir -p tmp +cd tmp || exit + +cat > clib.json << EOF +{ + "registries": [ + "https://gitlab.com/api/v4/projects/25447829/repository/files/README.md/raw?ref=master" + ], + "dependencies": { + "nouwaarom/clib-dependency": "0.0.1" + } +} +EOF + +clib install -c -o tmp > /dev/null || + throw "expecting exit code of 0"; + +{ [ -d ./tmp/clib-dependency ] && [ -f ./tmp/clib-dependency/package.json ]; } || + throw "failed to install clib-dependency" + +cd - > /dev/null || exit +rm -rf ./tmp diff --git a/test/package/clib_secrets.json b/test/package/clib_secrets.json new file mode 100644 index 00000000..4b93818d --- /dev/null +++ b/test/package/clib_secrets.json @@ -0,0 +1,4 @@ +{ + "gitlab.com": "GitlabSecret", + "github.com": "GithubSecret" +} \ No newline at end of file diff --git a/test/package/package-secrets-load.c b/test/package/package-secrets-load.c new file mode 100644 index 00000000..8df8afcb --- /dev/null +++ b/test/package/package-secrets-load.c @@ -0,0 +1,22 @@ +#include "describe/describe.h" +#include +#include "clib-secrets.h" + +int main() { + describe("clib_secrets") { + it("should load secrets from a valid json.") { + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + assert(secrets != NULL); + } + + it("should provide secrets for a domain.") { + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + char *github_secret = clib_secret_find_for_hostname(secrets, "github.com"); + assert(strcmp(github_secret, "GithubSecret") == 0); + char *gitlab_secret = clib_secret_find_for_hostname(secrets, "gitlab.com"); + assert(strcmp(gitlab_secret, "GitlabSecret") == 0); + } + + return assert_failures(); + } +} From 22fa313598c41ea4523841f07b8287695a3fbec0 Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sun, 25 Apr 2021 16:29:56 +0200 Subject: [PATCH 14/15] repository: Fix a bug in the download function where not all threads are properly joined. Fixed some compiler warnings. --- deps/http-get/http-get.c | 8 ++++---- deps/url/url.c | 9 +++++---- deps/url/url.h | 2 +- src/common/clib-package-installer.c | 24 +++++++++++++----------- src/common/clib-secrets.c | 4 ++-- src/registry/gitlab-registry.c | 2 +- src/repository/repository.c | 13 ++++++++----- 7 files changed, 34 insertions(+), 28 deletions(-) diff --git a/deps/http-get/http-get.c b/deps/http-get/http-get.c index 8c860a83..98f1accf 100644 --- a/deps/http-get/http-get.c +++ b/deps/http-get/http-get.c @@ -40,7 +40,7 @@ static size_t http_get_cb(void *contents, size_t size, size_t nmemb, void *userp return realsize; } -http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char** headers, int header_count) { +http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char** const headers, int header_count) { CURL *req = curl_easy_init(); http_get_response_t *res = malloc(sizeof(http_get_response_t)); @@ -79,7 +79,7 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char* * Perform an HTTP(S) GET on `url` */ -http_get_response_t *http_get(const char *url, const char** headers, int header_count) { +http_get_response_t *http_get(const char *url, const char** const headers, int header_count) { return http_get_shared(url, NULL, headers, header_count); } @@ -97,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea * Request `url` and save to `file` */ -int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** headers, int header_count) { +int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** const headers, int header_count) { CURL *req = curl_easy_init(); if (!req) return -1; @@ -133,7 +133,7 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share, const return (200 == status && CURLE_ABORTED_BY_CALLBACK != res) ? 0 : -1; } -int http_get_file(const char *url, const char *file, const char** headers, int header_count) { +int http_get_file(const char *url, const char *file, const char** const headers, int header_count) { return http_get_file_shared(url, file, NULL, NULL, 0); } diff --git a/deps/url/url.c b/deps/url/url.c index 5e8f5488..5a738bce 100644 --- a/deps/url/url.c +++ b/deps/url/url.c @@ -85,11 +85,11 @@ get_part (char *url, const char *format, int l) { } url_data_t * -url_parse (char *url) { +url_parse (const char *url) { url_data_t *data = (url_data_t *) malloc(sizeof(url_data_t)); if (!data) return NULL; - data->href = url; + data->href = strdup(url); char *tmp_url = strdup(url); bool is_ssh = false; @@ -108,7 +108,7 @@ url_parse (char *url) { int auth_len = 0; if (strstr(tmp_url, "@")) { auth = get_part(tmp_url, "%[^@]", protocol_len); - auth_len = strlen(auth); + auth_len = (int)strlen(auth); if (auth) auth_len++; } @@ -267,7 +267,7 @@ url_get_hostname (char *url) { char *auth = url_get_auth(url); if (!protocol) return NULL; - if (auth) l += strlen(auth) + 1; // add one @ symbol + if (auth) l += (int)strlen(auth) + 1; // add one @ symbol if (auth) free(auth); l += (int) strlen(protocol); @@ -442,6 +442,7 @@ url_data_inspect (url_data_t *data) { void url_free (url_data_t *data) { if (!data) return; + if (data->href) free(data->href); if (data->auth) free(data->auth); if (data->protocol) free(data->protocol); if (data->hostname) free(data->hostname); diff --git a/deps/url/url.h b/deps/url/url.h index 986489c0..a6b572eb 100644 --- a/deps/url/url.h +++ b/deps/url/url.h @@ -72,7 +72,7 @@ typedef struct url_data { */ url_data_t * -url_parse (char *url); +url_parse (const char *url); char * url_get_protocol (char *url); diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index c6cc1b6e..a6978ce3 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -302,7 +302,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { char *pkg_dir = NULL; char *command = NULL; int rc = 0; - int i = 0; + int thread_index = 0; #ifdef PATH_MAX long path_max = PATH_MAX; @@ -489,8 +489,8 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { char* package_id = clib_package_get_id(pkg->author, pkg->repo_name); // TODO, refactor this. while ((source = list_iterator_next(iterator))) { - handles[i] = repository_download_package_file(pkg->url, package_id, pkg->version, source->val, pkg_dir); - if (handles[i] == NULL) { + handles[thread_index] = repository_download_package_file(pkg->url, package_id, pkg->version, source->val, pkg_dir); + if (handles[thread_index] == NULL) { list_iterator_destroy(iterator); iterator = NULL; rc = -1; @@ -498,21 +498,23 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { } #ifdef HAVE_PTHREADS - if (i < max) { - i++; + if (thread_index < (max-1)) { + thread_index++; } else { - while (--i >= 0) { - repository_file_finish_download(handles[i]); - repository_file_free(handles[i]); + for (int j = 0; j <= thread_index; j++) { + repository_file_finish_download(handles[j]); + repository_file_free(handles[j]); } + thread_index = 0; } #endif } #ifdef HAVE_PTHREADS - while (--i >= 0) { - repository_file_finish_download(handles[i]); - repository_file_free(handles[i]); + // Here thread_index is one higher than the actual thread index. + for (int j = 0; j < thread_index; j++) { + repository_file_finish_download(handles[j]); + repository_file_free(handles[j]); } #endif diff --git a/src/common/clib-secrets.c b/src/common/clib-secrets.c index 21be938a..faef7f9d 100644 --- a/src/common/clib-secrets.c +++ b/src/common/clib-secrets.c @@ -53,8 +53,8 @@ clib_secrets_t clib_secrets_load_from_file(const char *file) { } for (unsigned int i = 0; i < json_object_get_count(json_object); i++) { - char *domain = json_object_get_name(json_object, i); - char *secret = json_object_get_string(json_object, domain); + const char *domain = json_object_get_name(json_object, i); + const char *secret = json_object_get_string(json_object, domain); struct clib_secret *secret_struct = malloc(sizeof(struct clib_secret)); secret_struct->hostname = strdup(domain); diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c index 9aefd92c..4e23850b 100644 --- a/src/registry/gitlab-registry.c +++ b/src/registry/gitlab-registry.c @@ -73,7 +73,7 @@ list_t *gitlab_registry_fetch(const char *url, const char *hostname, const char unsigned int size = strlen(key) + strlen(secret) + 2; char *authentication_header = malloc(size); snprintf(authentication_header, size, "%s:%s", key, secret); - res = http_get(url, &authentication_header, 1); + res = http_get(url, (const char **) &authentication_header, 1); } if (!res->ok) { return NULL; diff --git a/src/repository/repository.c b/src/repository/repository.c index f288ceaa..03341a39 100644 --- a/src/repository/repository.c +++ b/src/repository/repository.c @@ -18,6 +18,7 @@ #include #include #include +#include #include static debug_t _debugger; @@ -93,7 +94,7 @@ http_get_response_t *repository_fetch_package_manifest(const char *package_url, char *authentication_header = malloc(size); snprintf(authentication_header, size, "%s:%s", key, secret); - res = http_get_shared(manifest_url, clib_package_curl_share, &authentication_header, 1); + res = http_get_shared(manifest_url, clib_package_curl_share, (const char **) &authentication_header, 1); } else { res = http_get_shared(manifest_url, clib_package_curl_share, NULL, 0); } @@ -136,7 +137,8 @@ static int fetch_package_file_work(const char *url, const char *dir, const char return 1; } - if (!(path = path_join(dir, basename(file)))) { + char* file_copy = strdup(file); + if (!(path = path_join(dir, basename(file_copy)))) { rc = 1; goto cleanup; } @@ -159,7 +161,7 @@ static int fetch_package_file_work(const char *url, const char *dir, const char char *authentication_header = malloc(size); snprintf(authentication_header, size, "%s:%s", key, secret); - rc = http_get_file_shared(url, path, clib_package_curl_share, &authentication_header, 1); + rc = http_get_file_shared(url, path, clib_package_curl_share, (const char **) &authentication_header, 1); } else { rc = http_get_file_shared(url, path, clib_package_curl_share, NULL, 0); } @@ -195,8 +197,9 @@ static int fetch_package_file_work(const char *url, const char *dir, const char } cleanup: - free(path); + free(file_copy); + return rc; } @@ -226,7 +229,7 @@ static int fetch_package_file(const char *url, const char *dir, const char *file memset(fetch, 0, sizeof(*fetch)); - fetch->url = url; + fetch->url = strdup(url); fetch->dir = dir; fetch->file = file; fetch->secret = secret; From eae186bc1dad370bb38a0498872f06988b8610ba Mon Sep 17 00:00:00 2001 From: Elbert van de Put Date: Sun, 2 May 2021 12:51:17 +0200 Subject: [PATCH 15/15] install: Enabled installing binary dependencies. Fix some memory issues. --- deps/url/url.c | 6 +++--- src/clib-build.c | 2 +- src/clib-configure.c | 2 +- src/clib-settings.h | 13 ------------- src/common/clib-package-installer.c | 9 +++++---- 5 files changed, 10 insertions(+), 22 deletions(-) delete mode 100644 src/clib-settings.h diff --git a/deps/url/url.c b/deps/url/url.c index 5a738bce..c6262464 100644 --- a/deps/url/url.c +++ b/deps/url/url.c @@ -175,7 +175,7 @@ url_parse (const char *url) { int pathname_len = (int)strlen(pathname); data->pathname = pathname; - char *search = (char *) malloc(sizeof(search)); + char *search = (char *) malloc(strlen(tmp_path)+1); if (!search) { free(tmp_url); url_free(data); @@ -190,7 +190,7 @@ url_parse (const char *url) { int search_len = (int)strlen(search); free(tmp_path); - char *query = (char *) malloc(sizeof(char)); + char *query = (char *) malloc(search_len+1); if (!query) { free(tmp_url); url_free(data); @@ -199,7 +199,7 @@ url_parse (const char *url) { sscanf(search, "?%s", query); data->query = query; - char *hash = (char *) malloc(sizeof(char)); + char *hash = (char *) malloc(strlen(path)+1); if (!hash) { free(tmp_url); url_free(data); diff --git a/src/clib-build.c b/src/clib-build.c index 4977eb4b..752f2d41 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -193,7 +193,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { #endif } else { #ifdef DEBUG - package = clib_package_new_from_slug(dir, 1); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 1); #else package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif diff --git a/src/clib-configure.c b/src/clib-configure.c index 4e2ea077..5ac8058d 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -186,7 +186,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { #endif } else { #ifdef DEBUG - package = clib_package_new_from_slug(dir, 1); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 1); #else package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif diff --git a/src/clib-settings.h b/src/clib-settings.h deleted file mode 100644 index bdd17902..00000000 --- a/src/clib-settings.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef CLIB_SRC_CLIB_SETTINGS_H -#define CLIB_SRC_CLIB_SETTINGS_H - -// Shared settings -#define CLIB_PACKAGE_CACHE_TIME 30 * 24 * 60 * 60 - -#ifdef HAVE_PTHREADS -#define MAX_THREADS 12 -#endif - -const char *manifest_names[] = {"clib.json", "package.json", 0}; - -#endif//CLIB_SRC_CLIB_SETTINGS_H diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c index a6978ce3..70bf7c45 100644 --- a/src/common/clib-package-installer.c +++ b/src/common/clib-package-installer.c @@ -195,6 +195,7 @@ int clib_package_install_executable(clib_package_t *pkg, const char *dir, int ve E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); E_FORMAT(&tarball, "%s/%s", tmp, file); + // TODO, move to repository rc = http_get_file(url, tarball, NULL, 0); if (0 != rc) { @@ -517,6 +518,7 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { repository_file_free(handles[j]); } #endif + free(handles); #ifdef HAVE_PTHREADS pthread_mutex_lock(&lock.mutex); @@ -539,10 +541,9 @@ int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { goto cleanup; } - // TODO, check if we want to enable this. - //if (pkg->install) { - // rc = clib_package_install_executable(pkg, dir, verbose); - //} + if (pkg->install) { + rc = clib_package_install_executable(pkg, dir, verbose); + } if (0 == rc) { rc = clib_package_install_dependencies(pkg, dir, verbose);