diff --git a/.gitignore b/.gitignore index fa21d235..36c912cf 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ clib-build clib-update clib-upgrade clib-uninstall +*.exe test/package/package-* !test/package/package-*.c diff --git a/Building.md b/Building.md new file mode 100644 index 00000000..fc41f0c4 --- /dev/null +++ b/Building.md @@ -0,0 +1,40 @@ +# Building clib from source + +## OSx + +```sh +$ git clone https://github.com/clibs/clib.git /tmp/clib +$ cd /tmp/clib +$ make install +``` + +## Ubuntu or debian + +```sh +# install libcurl +$ sudo apt install libcurl4-gnutls-dev -qq +# clone +$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib +# build +$ make +# put on path +$ sudo make install +``` + +## Windows (crosscompiling from linux) +The docker image contains the mingw toolchain which is used to compile for windows. +Curl needs to be built from source. +```shell +# Download and compile curl +$ docker run --rm dockcross/windows-static-64-posix > dockcross-windows-x64 +$ cat dockcross-windows-x64 +$ chmod +x dockcross-windows-x64 +$ wget https://curl.haxx.se/download/curl-7.76.0.tar.gz +$ tar xzf curl-* +$ CURL_SRC=curl-* +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && ./configure --prefix="/work/deps/curl" --host=x86_64-w64-mingw32.static --with-winssl --disable-dependency-tracking --disable-pthreads --enable-threaded-resolver --disable-imap --disable-pop3 --disable-smpt --disable-ldap --disable-mqtt --disable-smb' +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make' +$ ./dockcross-windows-x64 bash -c 'cd '"$CURL_SRC"' && make install' +$ git clone https://github.com/clibs/clib.git && cd clib +$ ./dockcross-windows-x64 make all NO_PTHREADS=1 EXE=true +``` \ No newline at end of file diff --git a/Makefile b/Makefile index 35d5528c..a7a300f9 100644 --- a/Makefile +++ b/Makefile @@ -12,17 +12,16 @@ RM = rm -f MKDIR = mkdir -p SRC = $(wildcard src/*.c) -COMMON_SRC = $(wildcard src/common/*.c) -ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h test/package/*.c test/cache/*.c) +COMMON_SRC = $(wildcard src/common/*.c src/registry/*.c src/repository/*.c) +ALL_SRC = $(wildcard src/*.c src/*.h src/common/*.c src/common/*.h src/registry/*.c src/registry/*.h src/repository/*.h src/repository/*.c test/package/*.c test/cache/*.c) SDEPS = $(wildcard deps/*/*.c) ODEPS = $(SDEPS:.c=.o) DEPS = $(filter-out $(ODEPS), $(SDEPS)) OBJS = $(DEPS:.c=.o) -MAKEFILES = $(wildcard deps/*/Makefile) export CC -CFLAGS += -std=c99 -Ideps -Wall -Wno-unused-function -U__STRICT_ANSI__ +CFLAGS += -std=c99 -Ideps -Isrc/common -Isrc/repository -Isrc/registry -Wall -Werror=return-type -Werror=implicit-function-declaration -Wno-unused-function -U__STRICT_ANSI__ ifdef STATIC CFLAGS += -DCURL_STATICLIB $(shell deps/curl/bin/curl-config --cflags) @@ -48,7 +47,7 @@ all: $(BINS) build: $(BINS) -$(BINS): $(SRC) $(MAKEFILES) $(OBJS) +$(BINS): $(SRC) $(COMMON_SRC) $(OBJS) $(CC) $(CFLAGS) -o $@ $(COMMON_SRC) src/$(@:.exe=).c $(OBJS) $(LDFLAGS) $(MAKEFILES): diff --git a/Readme.md b/Readme.md index 2bbd882d..f1404303 100644 --- a/Readme.md +++ b/Readme.md @@ -7,58 +7,28 @@ ![c package manager screenshot](https://i.cloudup.com/GwqOU2hh9Y.png) -## Installation - - Expects [libcurl](http://curl.haxx.se/libcurl/) to be installed and linkable. - - With [homebrew](https://github.com/Homebrew/homebrew): - -```sh -$ brew install clib -``` - - Or [MacPorts](https://www.macports.org): - -```sh -$ sudo port selfupdate -$ sudo port install clib -``` - - With git: - -```sh -$ git clone https://github.com/clibs/clib.git /tmp/clib -$ cd /tmp/clib -$ make install -``` +## About - Ubuntu: +Basically the lazy-man's copy/paste promoting smaller C utilities, also +serving as a nice way to discover these sort of libraries. From my experience +C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide +stand-alone "micro" C libraries for developers to quickly install without coupling +to large frameworks. -```sh -# install libcurl -$ sudo apt-get install libcurl4-gnutls-dev -qq -# clone -$ git clone https://github.com/clibs/clib.git /tmp/clib && cd /tmp/clib -# build -$ make -# put on path -$ sudo make install -``` +You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction. -## About +The [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry". The registry is used by `clib(1)` when searching for packages. - Basically the lazy-man's copy/paste promoting smaller C utilities, also - serving as a nice way to discover these sort of libraries. From my experience - C libraries are scattered all over the web and discovery is relatively poor. The footprint of these libraries is usually quite large and unfocused. The goal of `clibs` is to provide - stand-alone "micro" C libraries for developers to quickly install without coupling - to large frameworks. +## Installation and building +Binaries for `clib(1)` releases can be found at [releases](https://github.com/clibs/clib/releases/). +For OSx and linux [libcurl](http://curl.haxx.se/libcurl/) should be installed and linkable. +The windows binaries do not require any libraries to be installed. - You should use `clib(1)` to fetch these files for you and check them into your repository, the end-user and contributors should not require having `clib(1)` installed. This allows `clib(1)` to fit into any new or existing C workflow without friction. +See [Building](Building.md) for instructions on how to build clib. - The wiki [listing of packages](https://github.com/clibs/clib/wiki/Packages) acts as the "registry" and populates the `clib-search(1)` results. ## Usage - +More detailed information on how to use `clib` can be found in [Usage](Usage.md). ``` clib [options] @@ -78,13 +48,9 @@ $ sudo make install search [query] Search for packages help Display help for cmd ``` +More information about the Command Line Interface can be found [here](https://github.com/clibs/clib/wiki/Command-Line-Interface). -More about the Command Line Interface [here](https://github.com/clibs/clib/wiki/Command-Line-Interface). - -## Examples - - More examples and best practices at [BEST_PRACTICE.md](https://github.com/clibs/clib/blob/master/BEST_PRACTICE.md). - +### Example usage Install a few dependencies to `./deps`: ```sh @@ -109,10 +75,26 @@ $ clib install ms file hash $ clib install visionmedia/mon visionmedia/every visionmedia/watch ``` -## clib.json - - Example of a clib.json explicitly listing the source: +## Clib.json +Information about a clib project or a package is stored in `clib.json`. +In a project that uses `clib` to install dependencies a typical `clib.json` will only contain the required dependencies. +It may look something like: +```json +{ + "name": "Copy and Paste-Inator", + "version": "0.4.2", + "description": "Creates copies of yourself to do all of his waiting in lines for you.", + "dependencies": { + "clibs/buffer": "0.0.1", + "clibs/term": "0.0.1", + "jwerle/throw.h": "0.0.0" + } +} +``` +Packages that can be installed by `clib` should also provide a `clib.json`. +It contains the files that should be installed by `clib` in `"src"` +An example of a clib.json for a package may look like: ```json { "name": "term", @@ -125,7 +107,7 @@ $ clib install visionmedia/mon visionmedia/every visionmedia/watch } ``` - Example of a clib.json for an executable: +Example of a clib.json for an executable: ```json { diff --git a/BEST_PRACTICE.md b/Usage.md similarity index 71% rename from BEST_PRACTICE.md rename to Usage.md index c2064a60..2bf29c21 100644 --- a/BEST_PRACTICE.md +++ b/Usage.md @@ -3,13 +3,12 @@ This page will cover: - [How to use libraries](#how-to-use-installed-libraries-for-your-project). - - [Example Makefile](#example-makefile). - - [Example `clib.json` for executables](#example-clibjson-for-executable-project). + - [Example Makefile](#example-makefile). + - [Example `clib.json` for executables](#example-packagejson-for-executable-project). - [Making your own library package](#making-your-own-libraries). - - [Example `clib.json` for libraries](#example-clibjson-for-libraries). + - [Example `clib.json` for libraries](#example-packagejson-for-libraries). - [How to install/uninstall executables](#install-and-uninstall-executables-packages). - -For instructions on installation, check out the [README](https://github.com/clibs/clib#installation). +- [How to fetch packages from other sources](#how-to-fetch-packages-from-other-sources). ## How to use installed libraries for your project @@ -116,7 +115,7 @@ at a example `clib.json` file for your project: (executable package) "dependencies": { "stephenmathieson/trim.c": "0.0.2", "clibs/commander": "1.3.2", - "clibs/logger": "0.0.1", + "clibs/logger": "0.0.1" }, "install": "make install", "uninstall": "make uninstall" @@ -129,9 +128,9 @@ _**NOTE:** Make sure you have a release as the same version in your `clib.json` ## Making your own libraries -Now that you know how to use libraries, heres how to make your own: +Now that you know how to use libraries, here is how to make your own: -Like before, heres a typical project directory tree: +Like before, a typical project directory tree: ``` your-library-c/ @@ -230,3 +229,41 @@ $ sudo clib-uninstall visionmedia/mon ```
+ +## How to fetch packages from other sources +By default `clib` uses the [listing of packages](https://github.com/clibs/clib/wiki/Packages) as the place to look for packages, the registry, and [github.com](https://github.com) for downloading packages. +You can specify additional registries and download from other repositories than github. +This might be useful when using `clib` to install a mix of private and public packages. + +### Adding additional registries +Additional registries can be provided in the `clib.json` of a project. +Currently github wiki's and gitlab — both [gitlab.com](https://www.gitlab.com) and self hosted — are supported. +For gitlab the format is a bit complicated as it has to conform to the gitlab api. +You should use the same format as the default registry but in a file in a repository instead of in a wiki. +```json +{ + // other definitions + "registries": [ + "https://gitlab.com/api/v4/projects/25447829/repository/files/README.md/raw?ref=master", + "https://github.com/awesome-org/clib/wiki/Packages" + ] +} +``` + +_**CAUTION:** For gitlab, the url should be of the form `/project/` and not `/group/repo` check [my-gitlab-registry](https://gitlab.com/nouwaarom/my-clib-registry) for an example._ + +### Downloading from gitlab or private sources +To download from some sources, authentication might be required. +To facilitate this `clib_secrets.json` is used to store the credentials. + +```json +{ + "github.com": "GITHUB_API_TOKEN", + "github.example.com": "GITLAB_USER_TOKEN" +} +``` + +Gitlab always requires a secret in order to use the API. +The secret can be obtained by clicking your profile and then (Preferences -> Access Tokens) and create a token with only `read_repository` rights. + +_**TIP:** To prevent accidentally commiting your secrets add `clib_secrets.json` to `.gitignore` and use `clib_secrets.json.dist` to specify for which domains a secret is required._ diff --git a/clib.json b/clib.json index f25c6722..eb0db306 100644 --- a/clib.json +++ b/clib.json @@ -18,11 +18,10 @@ "which": "0.1.3", "stephenmathieson/str-flatten.c": "0.0.4", "commander": "1.3.2", - "stephenmathieson/wiki-registry.c": "0.0.4", "stephenmathieson/case.c": "0.1.3", "jwerle/fs.c": "0.2.0", "stephenmathieson/str-replace.c": "0.0.6", - "strdup": "*", + "strdup": "0.1.*", "Constellation/console-colors.c": "1.0.1", "littlstar/asprintf.c": "0.0.3", "logger": "0.0.1", @@ -37,9 +36,16 @@ "stephenmathieson/debug.c": "0.0.0", "stephenmathieson/tempdir.c": "0.0.2", "isty001/copy": "0.0.0", - "stephenmathieson/rimraf.c": "0.1.0" + "stephenmathieson/rimraf.c": "0.1.0", + "thlorenz/gumbo-parser.c": "*", + "stephenmathieson/http-get.c": "*", + "stephenmathieson/gumbo-text-content.c": "*", + "stephenmathieson/gumbo-get-element-by-id.c": "*", + "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", + "clibs/list": "*", + "jwerle/url.h": "0.2.*" }, "development": { "stephenmathieson/describe.h": "2.0.1" } -} +} \ No newline at end of file diff --git a/deps/http-get/http-get.c b/deps/http-get/http-get.c index 0aeaaffa..98f1accf 100644 --- a/deps/http-get/http-get.c +++ b/deps/http-get/http-get.c @@ -40,7 +40,7 @@ static size_t http_get_cb(void *contents, size_t size, size_t nmemb, void *userp return realsize; } -http_get_response_t *http_get_shared(const char *url, CURLSH *share) { +http_get_response_t *http_get_shared(const char *url, CURLSH *share, const char** const headers, int header_count) { CURL *req = curl_easy_init(); http_get_response_t *res = malloc(sizeof(http_get_response_t)); @@ -50,6 +50,15 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) { curl_easy_setopt(req, CURLOPT_SHARE, share); } + if (header_count > 0) { + struct curl_slist *chunk = NULL; + for (int i = 0; i < header_count; i++) { + chunk = curl_slist_append(chunk, headers[i]); + } + /* set our custom set of headers */ + curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk); + } + curl_easy_setopt(req, CURLOPT_URL, url); curl_easy_setopt(req, CURLOPT_HTTPGET, 1); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1); @@ -70,8 +79,8 @@ http_get_response_t *http_get_shared(const char *url, CURLSH *share) { * Perform an HTTP(S) GET on `url` */ -http_get_response_t *http_get(const char *url) { - return http_get_shared(url, NULL); +http_get_response_t *http_get(const char *url, const char** const headers, int header_count) { + return http_get_shared(url, NULL, headers, header_count); } /** @@ -88,7 +97,7 @@ static size_t http_get_file_cb(void *ptr, size_t size, size_t nmemb, void *strea * Request `url` and save to `file` */ -int http_get_file_shared(const char *url, const char *file, CURLSH *share) { +int http_get_file_shared(const char *url, const char *file, CURLSH *share, const char** const headers, int header_count) { CURL *req = curl_easy_init(); if (!req) return -1; @@ -99,6 +108,15 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) { curl_easy_setopt(req, CURLOPT_SHARE, share); } + if (header_count > 0) { + struct curl_slist *chunk = NULL; + for (int i = 0; i < header_count; i++) { + chunk = curl_slist_append(chunk, headers[i]); + } + /* set our custom set of headers */ + curl_easy_setopt(req, CURLOPT_HTTPHEADER, chunk); + } + curl_easy_setopt(req, CURLOPT_URL, url); curl_easy_setopt(req, CURLOPT_HTTPGET, 1); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1); @@ -115,8 +133,8 @@ int http_get_file_shared(const char *url, const char *file, CURLSH *share) { return (200 == status && CURLE_ABORTED_BY_CALLBACK != res) ? 0 : -1; } -int http_get_file(const char *url, const char *file) { - return http_get_file_shared(url, file, NULL); +int http_get_file(const char *url, const char *file, const char** const headers, int header_count) { + return http_get_file_shared(url, file, NULL, NULL, 0); } /** diff --git a/deps/http-get/http-get.h b/deps/http-get/http-get.h index d584086d..16dd3b4f 100644 --- a/deps/http-get/http-get.h +++ b/deps/http-get/http-get.h @@ -21,11 +21,11 @@ typedef struct { int ok; } http_get_response_t; -http_get_response_t *http_get(const char *); -http_get_response_t *http_get_shared(const char *, void *); +http_get_response_t *http_get(const char *url, const char** headers, int header_count); +http_get_response_t *http_get_shared(const char *url, void *, const char** headers, int header_count); -int http_get_file(const char *, const char *); -int http_get_file_shared(const char *, const char *, void *); +int http_get_file(const char *, const char *, const char** headers, int header_count); +int http_get_file_shared(const char *, const char *, void *, const char** headers, int header_count); void http_get_free(http_get_response_t *); diff --git a/deps/strdup/strdup.c b/deps/strdup/strdup.c index f34bd5d9..4abda28e 100644 --- a/deps/strdup/strdup.c +++ b/deps/strdup/strdup.c @@ -8,9 +8,9 @@ #ifndef HAVE_STRDUP +#include "strdup.h" #include #include -#include "strdup.h" #ifndef strdup @@ -30,6 +30,19 @@ strdup(const char *str) { return buf; } +char* strndup(const char *str, size_t len) { + if (NULL == (char *) str) { + return NULL; + } + + char *buf = malloc(len+1); + + if (buf) { + memset(buf, 0, len+1); + memcpy(buf, str, len); + } + return buf; +} #endif #endif /* HAVE_STRDUP */ diff --git a/deps/strdup/strdup.h b/deps/strdup/strdup.h index cd75dd94..6a3bae38 100644 --- a/deps/strdup/strdup.h +++ b/deps/strdup/strdup.h @@ -17,10 +17,10 @@ * Returns a pointer to the newly allocated * copy of `str`, or `NULL` on failure. */ - +#include #ifndef strdup -char * -strdup(const char *str); +char * strdup(const char *str); +char * strndup(const char *str, size_t len); #endif #endif /* HAVE_STRDUP */ diff --git a/deps/url/clib.json b/deps/url/clib.json new file mode 100644 index 00000000..bcdb8de1 --- /dev/null +++ b/deps/url/clib.json @@ -0,0 +1,10 @@ +{ + "name": "url", + "version": "0.2.0", + "repo": "jwerle/url.h", + "description": "Parse URLs much like Node's url module", + "keywords": ["url", "parse"], + "license": "MIT", + "makefile": "Makefile", + "src": ["url.h", "url.c"] +} \ No newline at end of file diff --git a/deps/url/url.c b/deps/url/url.c new file mode 100644 index 00000000..c6262464 --- /dev/null +++ b/deps/url/url.c @@ -0,0 +1,457 @@ +#include +#include "url.h" +#include + +/** + * URI Schemes + * http://en.wikipedia.org/wiki/URI_scheme + */ + +static const char *URL_SCHEMES[] = { + // official IANA registered schemes + "aaa", "aaas", "about", "acap", "acct", "adiumxtra", "afp", "afs", "aim", "apt", "attachment", "aw", + "beshare", "bitcoin", "bolo", "callto", "cap", "chrome", "crome-extension", "com-evenbrite-attendee", + "cid", "coap", "coaps","content", "crid", "cvs", "data", "dav", "dict", "lna-playsingle", "dln-playcontainer", + "dns", "dtn", "dvb", "ed2k", "facetime", "fax", "feed", "file", "finger", "fish","ftp", "geo", "gg","git", + "gizmoproject", "go", "gopher", "gtalk", "h323", "hcp", "http", "https", "iax", "icap", "icon","im", + "imap", "info", "ipn", "ipp", "irc", "irc6", "ircs", "iris", "iris.beep", "iris.xpc", "iris.xpcs","iris.lws", + "itms", "jabber", "jar", "jms", "keyparc", "lastfm", "ldap", "ldaps", "magnet", "mailserver","mailto", + "maps", "market", "message", "mid", "mms", "modem", "ms-help", "mssettings-power", "msnim", "msrp", + "msrps", "mtqp", "mumble", "mupdate", "mvn", "news", "nfs", "ni", "nih", "nntp", "notes","oid", + "paquelocktoken", "pack", "palm", "paparazzi", "pkcs11", "platform", "pop", "pres", "prospero", "proxy", + "psyc","query", "reload", "res", "resource", "rmi", "rsync", "rtmp","rtsp", "secondlife", "service","session", + "sftp", "sgn", "shttp", "sieve", "sip", "sips", "skype", "smb", "sms", "snews", "snmp", "soap.beep","soap.beeps", + "soldat", "spotify", "ssh", "steam", "svn", "tag", "teamspeak", "tel", "telnet", "tftp", "things","thismessage", + "tn3270", "tip", "tv", "udp", "unreal", "urn", "ut2004", "vemmi","ventrilo", "videotex", "view-source", "wais","webcal", + "ws", "wss", "wtai", "wyciwyg", "xcon", "xcon-userid", "xfire","xmlrpc.beep", "xmlrpc.beeps", "xmpp", "xri","ymsgr", + + // unofficial schemes + "javascript", "jdbc", "doi" +}; + + +static char * +strff (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + (void) *ptr++; + } + + return strdup(ptr); +} + +static char * +strrwd (char *ptr, int n) { + for (int i = 0; i < n; ++i) { + (void) *ptr--; + } + + return strdup(ptr); +} + +static char * +get_part (char *url, const char *format, int l) { + bool has = false; + char *tmp = strdup(url); + char *tmp_url = strdup(url); + char *fmt_url = strdup(url); + char *ret; + + if (!tmp || !tmp_url || !fmt_url) + return NULL; + + strcpy(tmp, ""); + strcpy(fmt_url, ""); + + // move pointer exactly the amount + // of characters in the `prototcol` char + // plus 3 characters that represent the `://` + // part of the url + char* fmt_url_new = strff(fmt_url, l); + free(fmt_url); + fmt_url = fmt_url_new; + + sscanf(fmt_url, format, tmp); + + if (0 != strcmp(tmp, tmp_url)) { + has = true; + ret = strdup(tmp); + } + + free(tmp); + free(tmp_url); + free(fmt_url); + + return has? ret : NULL; +} + +url_data_t * +url_parse (const char *url) { + url_data_t *data = (url_data_t *) malloc(sizeof(url_data_t)); + if (!data) return NULL; + + data->href = strdup(url); + char *tmp_url = strdup(url); + bool is_ssh = false; + + char *protocol = url_get_protocol(tmp_url); + if (!protocol) { + free(tmp_url); + return NULL; + } + // length of protocol plus :// + int protocol_len = (int) strlen(protocol) + 3; + data->protocol = protocol; + + is_ssh = url_is_ssh(protocol); + + char *auth = (char *) malloc(sizeof(char)); + int auth_len = 0; + if (strstr(tmp_url, "@")) { + auth = get_part(tmp_url, "%[^@]", protocol_len); + auth_len = (int)strlen(auth); + if (auth) auth_len++; + } + + data->auth = auth; + + char *hostname; + + hostname = (is_ssh) + ? get_part(tmp_url, "%[^:]", protocol_len + auth_len) + : get_part(tmp_url, "%[^/]", protocol_len + auth_len); + + if (!hostname) { + free(tmp_url); + url_free(data); + return NULL; + } + int hostname_len = (int) strlen(hostname); + char *tmp_hostname = strdup(hostname); + data->hostname = hostname; + + char *host = (char *) malloc((strlen(tmp_hostname)+1) * sizeof(char)); + sscanf(tmp_hostname, "%[^:]", host); + free(tmp_hostname); + if (!host) { + free(tmp_url); + url_free(data); + return NULL; + } + data->host = host; + + int host_len = (int)strlen(host); + if (hostname_len > host_len) { + data->port = strff(hostname, host_len + 1); // +1 for ':' char; + } else { + data->port = NULL; + } + + char *tmp_path; + + tmp_path = (is_ssh) + ? get_part(tmp_url, ":%s", protocol_len + auth_len + hostname_len) + : get_part(tmp_url, "/%s", protocol_len + auth_len + hostname_len); + + char *path = (char *) malloc((strlen(tmp_path) + 2) * sizeof(char)); + if (!path) { + free(tmp_url); + url_free(data); + return NULL; + } + const char *fmt = (is_ssh)? "%s" : "/%s"; + sprintf(path, fmt, tmp_path); + data->path = path; + + char *pathname = (char *) malloc((strlen(tmp_path) + 2) * sizeof(char)); + free(tmp_path); + if (!pathname) { + free(tmp_url); + url_free(data); + return NULL; + } + strcat(pathname, ""); + tmp_path = strdup(path); + sscanf(tmp_path, "%[^? | ^#]", pathname); + int pathname_len = (int)strlen(pathname); + data->pathname = pathname; + + char *search = (char *) malloc(strlen(tmp_path)+1); + if (!search) { + free(tmp_url); + url_free(data); + return NULL; + } + char* tmp_path_new = strff(tmp_path, pathname_len); + free(tmp_path); + tmp_path = tmp_path_new; + strcpy(search, ""); + sscanf(tmp_path, "%[^#]", search); + data->search = search; + int search_len = (int)strlen(search); + free(tmp_path); + + char *query = (char *) malloc(search_len+1); + if (!query) { + free(tmp_url); + url_free(data); + return NULL; + } + sscanf(search, "?%s", query); + data->query = query; + + char *hash = (char *) malloc(strlen(path)+1); + if (!hash) { + free(tmp_url); + url_free(data); + return NULL; + } + tmp_path = strff(path, pathname_len + search_len); + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + data->hash = hash; + free(tmp_path); + free(tmp_url); + + return data; +} + +bool +url_is_protocol (char *str) { + int count = sizeof(URL_SCHEMES) / sizeof(URL_SCHEMES[0]); + + for (int i = 0; i < count; ++i) { + if (0 == strcmp(URL_SCHEMES[i], str)) { + return true; + } + } + + return false; +} + +bool +url_is_ssh (char *str) { + str = strdup(str); + if (0 == strcmp(str, "ssh") || 0 == strcmp(str, "git")) { + free(str); + return true; + + } + free(str); + + return false; +} + +char * +url_get_protocol (char *url) { + char *protocol = (char *) malloc(URL_PROTOCOL_MAX_LENGTH * sizeof(char)); + if (!protocol) return NULL; + sscanf(url, "%[^://]", protocol); + if (url_is_protocol(protocol)) return protocol; + return NULL; +} + + +char * +url_get_auth (char *url) { + char *protocol = url_get_protocol(url); + if (!protocol) return NULL; + int l = (int) strlen(protocol) + 3; + return get_part(url, "%[^@]", l); +} + +char * +url_get_hostname (char *url) { + int l = 3; + char *protocol = url_get_protocol(url); + char *tmp_protocol = strdup(protocol); + char *auth = url_get_auth(url); + + if (!protocol) return NULL; + if (auth) l += (int)strlen(auth) + 1; // add one @ symbol + if (auth) free(auth); + + l += (int) strlen(protocol); + + free(protocol); + + char * hostname = url_is_ssh(tmp_protocol) + ? get_part(url, "%[^:]", l) + : get_part(url, "%[^/]", l); + free(tmp_protocol); + return hostname; +} + +char * +url_get_host (char *url) { + char *host = (char *) malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + + if (!host || !hostname) return NULL; + + sscanf(hostname, "%[^:]", host); + + free(hostname); + + return host; +} + +char * +url_get_pathname (char *url) { + char *path = url_get_path(url); + char *pathname = (char *) malloc(sizeof(char)); + + if (!path || !pathname) return NULL; + + strcat(pathname, ""); + sscanf(path, "%[^?]", pathname); + + free(path); + + return pathname; +} + +char * +url_get_path (char *url) { + int l = 3; + char *tmp_path; + char *protocol = url_get_protocol(url); + char *auth = url_get_auth(url); + char *hostname = url_get_hostname(url); + + + if (!protocol || !hostname) + return NULL; + + bool is_ssh = url_is_ssh(protocol); + + l += (int) strlen(protocol) + (int) strlen(hostname); + + if (auth) l+= (int) strlen(auth) +1; // @ symbol + + tmp_path = (is_ssh) + ? get_part(url, ":%s", l) + : get_part(url, "/%s", l); + + const char *fmt = (is_ssh)? "%s" : "/%s"; + char *path = (char *) malloc(strlen(tmp_path) * sizeof(char)); + sprintf(path, fmt, tmp_path); + + if (auth) free(auth); + free(protocol); + free(hostname); + free(tmp_path); + + return path; + +} + +char * +url_get_search (char *url) { + char *path = url_get_path(url); + char *pathname = url_get_pathname(url); + char *search = (char *) malloc(sizeof(char)); + + if (!path || !search) return NULL; + + char *tmp_path = strff(path, (int)strlen(pathname)); + strcat(search, ""); + sscanf(tmp_path, "%[^#]", search); + + tmp_path = strrwd(tmp_path, (int)strlen(pathname)); + + free(path); + free(pathname); + + return search; +} + +char * +url_get_query (char *url) { + char *search = url_get_search(url); + char *query = (char *) malloc(sizeof(char)); + if (!search) return NULL; + sscanf(search, "?%s", query); + free(search); + return query; +} + +char * +url_get_hash (char *url) { + char *hash = (char *) malloc(sizeof(char)); + if (!hash) return NULL; + + char *path = url_get_path(url); + if (!path) return NULL; + + char *pathname = url_get_pathname(url); + if (!pathname) return NULL; + char *search = url_get_search(url); + + int pathname_len = (int) strlen(pathname); + int search_len = (int) strlen(search); + char *tmp_path = strff(path, pathname_len + search_len); + + strcat(hash, ""); + sscanf(tmp_path, "%s", hash); + tmp_path = strrwd(tmp_path, pathname_len + search_len); + free(tmp_path); + free(pathname); + free(path); + if (search) free(search); + + return hash; +} + +char * +url_get_port (char *url) { + char *port = (char *) malloc(sizeof(char)); + char *hostname = url_get_hostname(url); + char *host = url_get_host(url); + if (!port || !hostname) return NULL; + + char *tmp_hostname = strff(hostname, strlen(host) +1); + sscanf(tmp_hostname, "%s", port); + + free(hostname); + free(tmp_hostname); + return port; +} + +void +url_inspect (char *url) { + url_data_inspect(url_parse(url)); +} + + +void +url_data_inspect (url_data_t *data) { + printf("#url =>\n"); + printf(" .href: \"%s\"\n", data->href); + printf(" .protocol: \"%s\"\n", data->protocol); + printf(" .host: \"%s\"\n", data->host); + printf(" .auth: \"%s\"\n", data->auth); + printf(" .hostname: \"%s\"\n", data->hostname); + printf(" .pathname: \"%s\"\n", data->pathname); + printf(" .search: \"%s\"\n", data->search); + printf(" .path: \"%s\"\n", data->path); + printf(" .hash: \"%s\"\n", data->hash); + printf(" .query: \"%s\"\n", data->query); + printf(" .port: \"%s\"\n", data->port); +} + +void +url_free (url_data_t *data) { + if (!data) return; + if (data->href) free(data->href); + if (data->auth) free(data->auth); + if (data->protocol) free(data->protocol); + if (data->hostname) free(data->hostname); + if (data->host) free(data->host); + if (data->pathname) free(data->pathname); + if (data->path) free(data->path); + if (data->hash) free(data->hash); + if (data->port) free(data->port); + if (data->search) free(data->search); + if (data->query) free(data->query); + free(data); +} \ No newline at end of file diff --git a/deps/url/url.h b/deps/url/url.h new file mode 100644 index 00000000..a6b572eb --- /dev/null +++ b/deps/url/url.h @@ -0,0 +1,123 @@ +#ifndef URL_H +#define URL_H 1 + +/** + * Dependencies + */ + +#include +#include +#include + +/** + * url.h version + */ + +#define URL_VERSION 0.2.0 + + +/** + * Max length of a url protocol scheme + */ + +#define URL_PROTOCOL_MAX_LENGTH 16 + + +/** + * Max length of a url host part + */ + +#define URL_HOSTNAME_MAX_LENGTH 128 + + +/** + * Max length of a url tld part + */ + +#define URL_TLD_MAX_LENGTH 16 + + +/** + * Max length of a url auth part + */ + +#define URL_AUTH_MAX_LENGTH 32 + + +/** + * `url_data` struct that defines parts + * of a parsed URL such as host and protocol + */ + +typedef struct url_data { + char *href; + char *protocol; + char *host; + char *auth; + char *hostname; + char *pathname; + char *search; + char *path; + char *hash; + char *query; + char *port; +} url_data_t; + + +// prototype + +/** + * Parses a url into parts and returns + * a `url_data_t *` pointer + */ + +url_data_t * +url_parse (const char *url); + +char * +url_get_protocol (char *url); + +char * +url_get_auth (char *url); + +char * +url_get_hostname (char *url); + +char * +url_get_host (char *url); + +char * +url_get_pathname (char *url); + +char * +url_get_path (char *url); + +char * +url_get_search (char *url); + +char * +url_get_query (char *url); + +char * +url_get_hash (char *url); + +char * +url_get_port (char *url); + +void +url_free (url_data_t *data); + +bool +url_is_protocol (char *str); + +bool +url_is_ssh (char *str); + +void +url_inspect (char *url); + +void +url_data_inspect (url_data_t *data); + + +#endif \ No newline at end of file diff --git a/deps/wiki-registry/package.json b/deps/wiki-registry/package.json deleted file mode 100644 index 6ab5362e..00000000 --- a/deps/wiki-registry/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "wiki-registry", - "version": "0.0.4", - "repo": "stephenmathieson/wiki-registry.c", - "description": "Turn a GitHub Wiki page into a package registry", - "keywords": [ "registry", "github", "wiki" ], - "license": "MIT", - "src": [ - "src/wiki-registry.c", - "src/wiki-registry.h" - ], - "dependencies": { - "thlorenz/gumbo-parser.c": "*", - "stephenmathieson/substr.c": "*", - "strdup": "0.0.0", - "stephenmathieson/http-get.c": "*", - "stephenmathieson/case.c": "*", - "stephenmathieson/trim.c": "*", - "stephenmathieson/gumbo-text-content.c": "*", - "stephenmathieson/gumbo-get-element-by-id.c": "*", - "stephenmathieson/gumbo-get-elements-by-tag-name.c": "*", - "clibs/list": "*" - } -} diff --git a/deps/wiki-registry/wiki-registry.h b/deps/wiki-registry/wiki-registry.h deleted file mode 100644 index 21c219cd..00000000 --- a/deps/wiki-registry/wiki-registry.h +++ /dev/null @@ -1,31 +0,0 @@ - -// -// wiki-registry.h -// -// Copyright (c) 2013 Stephen Mathieson -// MIT licensed -// - - -#ifndef WIKI_REGISTRY_H -#define WIKI_REGISTRY_H 1 - -#include "list/list.h" - -typedef struct { - char *repo; - char *href; - char *description; - char *category; -} wiki_package_t; - -list_t * -wiki_registry(const char *); - -list_t * -wiki_registry_parse(const char *); - -void -wiki_package_free(wiki_package_t *); - -#endif diff --git a/src/clib-build.c b/src/clib-build.c index 8f3bceda..752f2d41 100755 --- a/src/clib-build.c +++ b/src/clib-build.c @@ -13,10 +13,6 @@ #include #include -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #ifdef HAVE_PTHREADS #include #endif @@ -41,7 +37,7 @@ #include #include #include - +#include "clib-settings.h" #include "version.h" #define PROGRAM_NAME "clib-build" @@ -79,7 +75,7 @@ struct options { #endif }; -clib_package_opts_t package_opts = {0}; +clib_package_opts_t build_package_opts = {0}; clib_package_t *root_package = 0; command_t program = {0}; @@ -197,9 +193,9 @@ int build_package_with_manifest_name(const char *dir, const char *file) { #endif } else { #ifdef DEBUG - package = clib_package_new_from_slug(dir, 1); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 1); #else - package = clib_package_new_from_slug(dir, 0); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif } @@ -231,8 +227,8 @@ int build_package_with_manifest_name(const char *dir, const char *file) { } if (root_package && root_package->prefix) { - package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); + build_package_opts.prefix = root_package->prefix; + clib_package_set_opts(build_package_opts); setenv("PREFIX", package_opts.prefix, 1); } else if (opts.prefix) { setenv("PREFIX", opts.prefix, 1); @@ -330,7 +326,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { char *dep_dir = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); if (opts.dir && dependency && dependency->name) { dep_dir = path_join(opts.dir, dependency->name); } @@ -401,7 +397,7 @@ int build_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); @@ -671,12 +667,12 @@ int main(int argc, char **argv) { clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; + build_package_opts.skip_cache = opts.skip_cache; + build_package_opts.prefix = opts.prefix; + build_package_opts.global = opts.global; + build_package_opts.force = opts.force; - clib_package_set_opts(package_opts); + clib_package_set_opts(build_package_opts); if (0 == program.argc || (argc == rest_offset + rest_argc)) { rc = build_package(CWD); diff --git a/src/clib-configure.c b/src/clib-configure.c index b2dffcad..5ac8058d 100755 --- a/src/clib-configure.c +++ b/src/clib-configure.c @@ -12,10 +12,6 @@ #include #include -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #ifdef HAVE_PTHREADS #include #endif @@ -41,7 +37,7 @@ #include #include #include - +#include "clib-settings.h" #include "version.h" #define PROGRAM_NAME "clib-configure" @@ -70,7 +66,7 @@ struct options { #endif }; -clib_package_opts_t package_opts = {0}; +clib_package_opts_t configure_package_opts = {0}; clib_package_t *root_package = 0; hash_t *configured = 0; @@ -190,9 +186,9 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { #endif } else { #ifdef DEBUG - package = clib_package_new_from_slug(dir, 1); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 1); #else - package = clib_package_new_from_slug(dir, 0); + package = clib_package_new_from_slug_and_url(dir, "FIXME", 0); #endif } @@ -288,7 +284,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); @@ -359,7 +355,7 @@ int configure_package_with_manifest_name(const char *dir, const char *file) { char *slug = 0; asprintf(&slug, "%s/%s@%s", dep->author, dep->name, dep->version); - clib_package_t *dependency = clib_package_new_from_slug(slug, 0); + clib_package_t *dependency = clib_package_new_from_slug_and_url(slug, "FIXME", 0); char *dep_dir = path_join(opts.dir, dependency->name); free(slug); @@ -603,12 +599,12 @@ int main(int argc, char **argv) { clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; + configure_package_opts.skip_cache = package_opts.skip_cache; + configure_package_opts.prefix = package_opts.prefix; + configure_package_opts.global = package_opts.global; + configure_package_opts.force = package_opts.force; - clib_package_set_opts(package_opts); + clib_package_set_opts(configure_package_opts); if (0 == program.argc || (argc == rest_offset + rest_argc)) { rc = configure_package(CWD); diff --git a/src/clib-init.c b/src/clib-init.c index fd4178b2..d067f803 100755 --- a/src/clib-init.c +++ b/src/clib-init.c @@ -7,9 +7,7 @@ #include "asprintf/asprintf.h" #include "commander/commander.h" -#include "common/clib-package.h" #include "debug/debug.h" -#include "fs/fs.h" #include "logger/logger.h" #include "parson/parson.h" #include "version.h" @@ -17,6 +15,7 @@ #include #include #include +#include #if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ defined(__MINGW64__) diff --git a/src/clib-install.c b/src/clib-install.c index 3e1538aa..ca362e39 100755 --- a/src/clib-install.c +++ b/src/clib-install.c @@ -12,17 +12,19 @@ #include "common/clib-validate.h" #include "debug/debug.h" #include "fs/fs.h" -#include "http-get/http-get.h" #include "logger/logger.h" #include "parson/parson.h" -#include "str-replace/str-replace.h" #include "version.h" +#include #include -#include #include +#include +#include #include #include #include +#include "clib-package-installer.h" +#include "strdup/strdup.h" #define SX(s) #s #define S(s) SX(s) @@ -55,8 +57,8 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; -static clib_package_t *root_package = NULL; +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; /** * Option setters. @@ -121,59 +123,25 @@ static void setopt_skip_cache(command_t *self) { debug(&debugger, "set skip cache flag"); } -static int install_local_packages_with_package_name(const char *file) { - if (0 != clib_validate(file)) { - return 1; - } - - debug(&debugger, "reading local clib.json or package.json"); - char *json = fs_read(file); - if (NULL == json) - return 1; - - clib_package_t *pkg = clib_package_new(json, opts.verbose); - if (NULL == pkg) - goto e1; - - if (pkg->prefix) { - setenv("PREFIX", pkg->prefix, 1); +/** + * Install dependency packages at `pwd`. + */ +static int install_local_packages(clib_package_t* root_package) { + if (root_package && root_package->prefix) { + setenv("PREFIX", root_package->prefix, 1); } - int rc = clib_package_install_dependencies(pkg, opts.dir, opts.verbose); + int rc = clib_package_install_dependencies(root_package, opts.dir, opts.verbose); if (-1 == rc) - goto e2; + return 1; if (opts.dev) { - rc = clib_package_install_development(pkg, opts.dir, opts.verbose); + rc = clib_package_install_development(root_package, opts.dir, opts.verbose); if (-1 == rc) - goto e2; + return 1; } - free(json); - clib_package_free(pkg); return 0; - -e2: - clib_package_free(pkg); -e1: - free(json); - return 1; -} - -/** - * Install dependency packages at `pwd`. - */ -static int install_local_packages() { - const char *name = NULL; - unsigned int i = 0; - int rc = 0; - - do { - name = manifest_names[i]; - rc = install_local_packages_with_package_name(name); - } while (NULL != manifest_names[++i] && 0 != rc); - - return rc; } static int write_dependency_with_package_name(clib_package_t *pkg, char *prefix, @@ -238,8 +206,7 @@ static int save_dev_dependency(clib_package_t *pkg) { /** * Create and install a package from `slug`. */ - -static int install_package(const char *slug) { +static int install_package(clib_package_t* root_package, const char *slug) { clib_package_t *pkg = NULL; int rc; @@ -251,27 +218,12 @@ static int install_package(const char *slug) { long path_max = 4096; #endif - if (!root_package) { - const char *name = NULL; - char *json = NULL; - unsigned int i = 0; - - do { - name = manifest_names[i]; - json = fs_read(name); - } while (NULL != manifest_names[++i] && !json); - - if (json) { - root_package = clib_package_new(json, opts.verbose); - } - } - if ('.' == slug[0]) { if (1 == strlen(slug) || ('/' == slug[1] && 2 == strlen(slug))) { char dir[path_max]; realpath(slug, dir); slug = dir; - return install_local_packages(); + return install_local_packages(root_package); } } @@ -283,7 +235,7 @@ static int install_package(const char *slug) { #endif )) { free(stats); - return install_local_packages_with_package_name(slug); + return install_local_packages(root_package); } if (stats) { @@ -291,16 +243,21 @@ static int install_package(const char *slug) { } } - if (!pkg) { - pkg = clib_package_new_from_slug(slug, opts.verbose); + char* author = clib_package_parse_author(slug); + char* name = clib_package_parse_name(slug); + char* package_id = clib_package_get_id(author, name); + registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); + if (!package_info) { + debug(&debugger, "Package %s not found in any registry.", slug); + return -1; } + pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), opts.verbose); if (NULL == pkg) return -1; if (root_package && root_package->prefix) { package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); } rc = clib_package_install(pkg, opts.dir, opts.verbose); @@ -315,10 +272,6 @@ static int install_package(const char *slug) { } } - if (0 == pkg->repo || 0 != strcmp(slug, pkg->repo)) { - pkg->repo = strdup(slug); - } - if (opts.save) save_dependency(pkg); if (opts.savedev) @@ -333,10 +286,10 @@ static int install_package(const char *slug) { * Install the given `pkgs`. */ -static int install_packages(int n, char *pkgs[]) { +static int install_packages(clib_package_t* root_package, int n, char *pkgs[]) { for (int i = 0; i < n; i++) { debug(&debugger, "install %s (%d)", pkgs[i], i); - if (-1 == install_package(pkgs[i])) { + if (-1 == install_package(root_package, pkgs[i])) { logger_error("error", "Unable to install package %s", pkgs[i]); return 1; } @@ -418,30 +371,41 @@ int main(int argc, char *argv[]) { memset(prefix, 0, path_max); realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = opts.skip_cache; - package_opts.prefix = opts.prefix; - package_opts.global = opts.global; - package_opts.force = opts.force; - package_opts.token = opts.token; + clib_package_opts_t install_package_opts = {0}; + install_package_opts.skip_cache = opts.skip_cache; + install_package_opts.prefix = opts.prefix; + install_package_opts.global = opts.global; + install_package_opts.force = opts.force; + install_package_opts.token = opts.token; #ifdef HAVE_PTHREADS - package_opts.concurrency = opts.concurrency; + install_package_opts.concurrency = opts.concurrency; #endif - clib_package_set_opts(package_opts); + clib_package_set_opts(install_package_opts); + + // Read local config files. + secrets = clib_secrets_load_from_file("clib_secrets.json"); + clib_package_t* root_package = clib_package_load_local_manifest(0); + + repository_init(secrets); // The repository requires the secrets for authentication. + registries = registry_manager_init_registries(root_package ? root_package->registries : NULL, secrets); + registry_manager_fetch_registries(registries); + + clib_package_installer_init(registries, secrets); - int code = 0 == program.argc ? install_local_packages() - : install_packages(program.argc, program.argv); + // TODO, move argument parsing here. + int code = 0 == program.argc ? install_local_packages(root_package) + : install_packages(root_package, program.argc, program.argv); curl_global_cleanup(); clib_package_cleanup(); + clib_package_free(root_package); command_free(&program); return code; diff --git a/src/clib-search.c b/src/clib-search.c index 6e0edf30..0351f95c 100755 --- a/src/clib-search.c +++ b/src/clib-search.c @@ -17,14 +17,17 @@ #include "http-get/http-get.h" #include "logger/logger.h" #include "parson/parson.h" +#include "registry-manager.h" +#include "registry/registry.h" #include "strdup/strdup.h" #include "tempdir/tempdir.h" #include "version.h" -#include "wiki-registry/wiki-registry.h" +#include #include #include #include #include +#include #define CLIB_WIKI_URL "https://github.com/clibs/clib/wiki/Packages" @@ -46,22 +49,22 @@ static void setopt_nocache(command_t *self) { opt_cache = 0; } static void setopt_json(command_t *self) { opt_json = 1; } -#define COMPARE(v) \ - { \ - if (NULL == v) { \ - rc = 0; \ - goto cleanup; \ - } \ - case_lower(v); \ - for (int i = 0; i < count; i++) { \ - if (strstr(v, args[i])) { \ - rc = 1; \ - break; \ - } \ - } \ +#define COMPARE(v) \ + { \ + if (NULL == v) { \ + rc = 0; \ + goto cleanup; \ + } \ + case_lower(v); \ + for (int i = 0; i < count; i++) { \ + if (strstr(v, args[i])) { \ + rc = 1; \ + break; \ + } \ + } \ } -static int matches(int count, char *args[], wiki_package_t *pkg) { +static int matches(int count, char *args[], registry_package_ptr_t pkg) { // Display all packages if there's no query if (0 == count) return 1; @@ -72,16 +75,16 @@ static int matches(int count, char *args[], wiki_package_t *pkg) { char *href = NULL; int rc = 0; - name = clib_package_parse_name(pkg->repo); + name = clib_package_parse_name(registry_package_get_id(pkg)); COMPARE(name); - description = strdup(pkg->description); + description = strdup(registry_package_get_description(pkg)); COMPARE(description); - repo = strdup(pkg->repo); + repo = strdup(registry_package_get_id(pkg)); COMPARE(repo); - href = strdup(pkg->href); + href = strdup(registry_package_get_href(pkg)); COMPARE(href); cleanup: @@ -92,57 +95,54 @@ static int matches(int count, char *args[], wiki_package_t *pkg) { return rc; } +/* static char *wiki_html_cache() { + if (clib_cache_has_search() && opt_cache) { + char *data = clib_cache_read_search(); - if (clib_cache_has_search() && opt_cache) { - char *data = clib_cache_read_search(); - - if (data) { - return data; + if (data) { + return data; + } } - goto set_cache; - } - -set_cache: + debug(&debugger, "setting cache from %s", CLIB_WIKI_URL); + http_get_response_t *res = http_get(CLIB_WIKI_URL); + if (!res->ok) + return NULL; - debug(&debugger, "setting cache from %s", CLIB_WIKI_URL); - http_get_response_t *res = http_get(CLIB_WIKI_URL); - if (!res->ok) - return NULL; + char *html = strdup(res->data); + if (NULL == html) + return NULL; + http_get_free(res); - char *html = strdup(res->data); - if (NULL == html) - return NULL; - http_get_free(res); - - if (NULL == html) + if (NULL == html) + return html; + clib_cache_save_search(html); + debug(&debugger, "wrote cache"); return html; - clib_cache_save_search(html); - debug(&debugger, "wrote cach"); - return html; } +*/ -static void display_package(const wiki_package_t *pkg, +static void display_package(const registry_package_ptr_t pkg, cc_color_t fg_color_highlight, cc_color_t fg_color_text) { - cc_fprintf(fg_color_highlight, stdout, " %s\n", pkg->repo); + cc_fprintf(fg_color_highlight, stdout, " %s\n", registry_package_get_id(pkg)); printf(" url: "); - cc_fprintf(fg_color_text, stdout, "%s\n", pkg->href); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_href(pkg)); printf(" desc: "); - cc_fprintf(fg_color_text, stdout, "%s\n", pkg->description); + cc_fprintf(fg_color_text, stdout, "%s\n", registry_package_get_description(pkg)); printf("\n"); } -static void add_package_to_json(const wiki_package_t *pkg, +static void add_package_to_json(const registry_package_ptr_t pkg, JSON_Array *json_list) { JSON_Value *json_pkg_root = json_value_init_object(); JSON_Object *json_pkg = json_value_get_object(json_pkg_root); - json_object_set_string(json_pkg, "repo", pkg->repo); - json_object_set_string(json_pkg, "href", pkg->href); - json_object_set_string(json_pkg, "description", pkg->description); - json_object_set_string(json_pkg, "category", pkg->category); + json_object_set_string(json_pkg, "repo", registry_package_get_id(pkg)); + json_object_set_string(json_pkg, "href", registry_package_get_href(pkg)); + json_object_set_string(json_pkg, "description", registry_package_get_description(pkg)); + json_object_set_string(json_pkg, "category", registry_package_get_category(pkg)); json_array_append_value(json_list, json_pkg_root); } @@ -177,56 +177,66 @@ int main(int argc, char *argv[]) { cc_color_t fg_color_highlight = opt_color ? CC_FG_DARK_CYAN : CC_FG_NONE; cc_color_t fg_color_text = opt_color ? CC_FG_DARK_GRAY : CC_FG_NONE; - char *html = wiki_html_cache(); - if (NULL == html) { - command_free(&program); - logger_error("error", "failed to fetch wiki HTML"); - return 1; - } - - list_t *pkgs = wiki_registry_parse(html); - free(html); - - debug(&debugger, "found %zu packages", pkgs->len); - - list_node_t *node; - list_iterator_t *it = list_iterator_new(pkgs, LIST_HEAD); + // We search the local manifest for extra registries. + // It is important to give the extra registries preference over the default registry. + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + + clib_package_t *package = clib_package_load_local_manifest(0); + registries_t registries = registry_manager_init_registries(package ? package->registries : NULL, secrets); + registry_manager_fetch_registries(registries); + + // TODO, implement caching for the new registries. + /* + char *html = wiki_html_cache(); + if (NULL == html) { + command_free(&program); + logger_error("error", "failed to fetch wiki HTML"); + return 1; + } + list_t *pkgs = wiki_registry_parse(html); + free(html); + debug(&debugger, "found %zu packages", pkgs->len); + */ + + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t registry = NULL; + while ((registry = registry_iterator_next(it))) { + printf("SEARCH: packages from %s\n", registry_get_url(registry)); + registry_package_ptr_t pkg; + registry_package_iterator_t it = registry_package_iterator_new(registry); + + JSON_Array *json_list = NULL; + JSON_Value *json_list_root = NULL; + + if (opt_json) { + json_list_root = json_value_init_array(); + json_list = json_value_get_array(json_list_root); + } - JSON_Array *json_list = NULL; - JSON_Value *json_list_root = NULL; + printf("\n"); - if (opt_json) { - json_list_root = json_value_init_array(); - json_list = json_value_get_array(json_list_root); - } - - printf("\n"); - - while ((node = list_iterator_next(it))) { - wiki_package_t *pkg = (wiki_package_t *)node->val; - if (matches(program.argc, program.argv, pkg)) { - if (opt_json) { - add_package_to_json(pkg, json_list); + while ((pkg = registry_package_iterator_next(it))) { + if (matches(program.argc, program.argv, pkg)) { + if (opt_json) { + add_package_to_json(pkg, json_list); + } else { + display_package(pkg, fg_color_highlight, fg_color_text); + } } else { - display_package(pkg, fg_color_highlight, fg_color_text); + debug(&debugger, "skipped package %s", registry_package_get_id(pkg)); } - } else { - debug(&debugger, "skipped package %s", pkg->repo); } - wiki_package_free(pkg); - } + if (opt_json) { + char *serialized = json_serialize_to_string_pretty(json_list_root); + puts(serialized); - if (opt_json) { - char *serialized = json_serialize_to_string_pretty(json_list_root); - puts(serialized); + json_free_serialized_string(serialized); + json_value_free(json_list_root); + } - json_free_serialized_string(serialized); - json_value_free(json_list_root); + registry_package_iterator_destroy(it); } - - list_iterator_destroy(it); - list_destroy(pkgs); command_free(&program); return 0; } diff --git a/src/clib-uninstall.c b/src/clib-uninstall.c index 7dc37296..2d3a934c 100755 --- a/src/clib-uninstall.c +++ b/src/clib-uninstall.c @@ -19,6 +19,7 @@ #include "version.h" #include #include +#include "clib-settings.h" #define CLIB_UNINSTALL_DEFAULT_TARGET "make uninstall" @@ -160,7 +161,7 @@ static int clib_uninstall(const char *owner, const char *name, goto done; logger_info("fetch", tarball); - if (-1 == http_get_file(tarball, tarpath)) { + if (-1 == http_get_file(tarball, tarpath, NULL, 0)) { logger_error("error", "failed to fetch tarball"); goto done; } diff --git a/src/clib-update.c b/src/clib-update.c index 3c5fe4a4..f78db21c 100755 --- a/src/clib-update.c +++ b/src/clib-update.c @@ -16,10 +16,13 @@ #include "logger/logger.h" #include "parson/parson.h" #include "str-replace/str-replace.h" +#include "strdup/strdup.h" #include "version.h" +#include #include #include #include +#include #include #include #include @@ -50,8 +53,9 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; static clib_package_t *root_package = NULL; +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; /** * Option setters. @@ -230,7 +234,7 @@ static int install_package(const char *slug) { } if (!pkg) { - pkg = clib_package_new_from_slug(slug, opts.verbose); + pkg = clib_package_new_from_slug_and_url(slug, "FIXME", opts.verbose); } if (NULL == pkg) @@ -331,14 +335,12 @@ int main(int argc, char *argv[]) { logger_error("error", "Failed to initialize cURL"); } - if (opts.prefix) { + if (package_opts.prefix) { char prefix[path_max]; memset(prefix, 0, path_max); - realpath(opts.prefix, prefix); + realpath(package_opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + package_opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); @@ -355,6 +357,16 @@ int main(int argc, char *argv[]) { clib_package_set_opts(package_opts); + // Read local config files. + secrets = clib_secrets_load_from_file("clib_secrets.json"); + root_package = clib_package_load_local_manifest(0); + + repository_init(secrets); // The repository requires the secrets for authentication. + registries = registry_manager_init_registries(root_package->registries, secrets); + registry_manager_fetch_registries(registries); + + clib_package_installer_init(registries, secrets); + int code = 0 == program.argc ? install_local_packages() : install_packages(program.argc, program.argv); diff --git a/src/clib-upgrade.c b/src/clib-upgrade.c index ecf8bb57..cb3ccb68 100755 --- a/src/clib-upgrade.c +++ b/src/clib-upgrade.c @@ -16,9 +16,11 @@ #include "logger/logger.h" #include "parson/parson.h" #include "str-replace/str-replace.h" +#include "strdup/strdup.h" #include "tempdir/tempdir.h" #include "version.h" #include +#include #include #include #include @@ -54,7 +56,7 @@ struct options { static struct options opts = {0}; -static clib_package_opts_t package_opts = {0}; +static clib_package_opts_t upgrade_package_opts = {0}; static clib_package_t *root_package = NULL; /** @@ -136,7 +138,7 @@ static int install_package(const char *slug) { logger_info("info", "Upgrading to %s", extended_slug); - pkg = clib_package_new_from_slug(extended_slug, opts.verbose); + pkg = clib_package_new_from_slug_and_url(extended_slug, "FIXME", opts.verbose); if (NULL == pkg) { logger_error("error", @@ -148,8 +150,8 @@ static int install_package(const char *slug) { } if (root_package && root_package->prefix) { - package_opts.prefix = root_package->prefix; - clib_package_set_opts(package_opts); + upgrade_package_opts.prefix = root_package->prefix; + clib_package_set_opts(upgrade_package_opts); } char *tmp = gettempdir(); @@ -231,24 +233,22 @@ int main(int argc, char *argv[]) { memset(prefix, 0, path_max); realpath(opts.prefix, prefix); unsigned long int size = strlen(prefix) + 1; - opts.prefix = malloc(size); - memset((void *)opts.prefix, 0, size); - memcpy((void *)opts.prefix, prefix, size); + opts.prefix = strndup(prefix, size); } clib_cache_init(CLIB_PACKAGE_CACHE_TIME); - package_opts.skip_cache = 1; - package_opts.prefix = opts.prefix; - package_opts.global = 1; - package_opts.force = opts.force; - package_opts.token = opts.token; + upgrade_package_opts.skip_cache = 1; + upgrade_package_opts.prefix = opts.prefix; + upgrade_package_opts.global = 1; + upgrade_package_opts.force = opts.force; + upgrade_package_opts.token = opts.token; #ifdef HAVE_PTHREADS - package_opts.concurrency = opts.concurrency; + upgrade_package_opts.concurrency = opts.concurrency; #endif - clib_package_set_opts(package_opts); + clib_package_set_opts(upgrade_package_opts); char *slug = 0; diff --git a/src/common/clib-package-installer.c b/src/common/clib-package-installer.c new file mode 100644 index 00000000..70bf7c45 --- /dev/null +++ b/src/common/clib-package-installer.c @@ -0,0 +1,588 @@ +// +// clib-package-installer.c +// +// Copyright (c) 2021 clib authors +// MIT licensed +// + +#include "clib-package-installer.h" +#include "asprintf/asprintf.h" +#include "clib-cache.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ + defined(__MINGW64__) || defined(__CYGWIN__) +#define setenv(k, v, _) _putenv_s(k, v) +#define realpath(a, b) _fullpath(a, b, strlen(a)) +#endif + +CURLSH *clib_package_curl_share; +//TODO, cleanup somewhere curl_share_cleanup(clib_package_curl_share); + +static debug_t _debugger; + +#define _debug(...) \ + ({ \ + if (!(_debugger.name)) \ + debug_init(&_debugger, "package-installer"); \ + debug(&_debugger, __VA_ARGS__); \ + }) + +#define E_FORMAT(...) \ + ({ \ + rc = asprintf(__VA_ARGS__); \ + if (-1 == rc) \ + goto cleanup; \ + }); + +static hash_t *visited_packages = 0; + +#ifdef HAVE_PTHREADS +typedef struct clib_package_lock clib_package_lock_t; +struct clib_package_lock { + pthread_mutex_t mutex; +}; + +static clib_package_lock_t lock = {PTHREAD_MUTEX_INITIALIZER}; + +#endif + +static clib_secrets_t secrets = NULL; +static registries_t registries = NULL; + +void clib_package_installer_init(registries_t _registries, clib_secrets_t _secrets) { + secrets = _secrets; + registries = _registries; +} + +static inline int install_packages(list_t *list, const char *dir, int verbose) { + list_node_t *node = NULL; + list_iterator_t *iterator = NULL; + int rc = -1; + list_t *freelist = NULL; + + if (!list || !dir) + goto cleanup; + + iterator = list_iterator_new(list, LIST_HEAD); + if (NULL == iterator) + goto cleanup; + + freelist = list_new(); + + while ((node = list_iterator_next(iterator))) { + clib_package_dependency_t *dep = NULL; + char *slug = NULL; + clib_package_t *pkg = NULL; + int error = 1; + + dep = (clib_package_dependency_t *) node->val; + char *package_id = clib_package_get_id(dep->author, dep->name); + slug = clib_package_slug(dep->author, dep->name, dep->version); + if (NULL == slug) + goto loop_cleanup; + + registry_package_ptr_t package_info = registry_manager_find_package(registries, package_id); + if (!package_info) { + logger_error("package-installer", "Package %s not found in any registry.", package_id); + goto loop_cleanup; + } + + pkg = clib_package_new_from_slug_and_url(slug, registry_package_get_href(package_info), verbose); + if (NULL == pkg) + goto loop_cleanup; + + if (-1 == clib_package_install(pkg, dir, verbose)) + goto loop_cleanup; + + list_rpush(freelist, list_node_new(pkg)); + error = 0; + + loop_cleanup: + free(package_id); + if (slug) + free(slug); + if (error) { + list_iterator_destroy(iterator); + iterator = NULL; + rc = -1; + goto cleanup; + } + } + + rc = 0; + +cleanup: + if (iterator) + list_iterator_destroy(iterator); + + if (freelist) { + iterator = list_iterator_new(freelist, LIST_HEAD); + while ((node = list_iterator_next(iterator))) { + clib_package_t *pkg = node->val; + if (pkg) + clib_package_free(pkg); + } + list_iterator_destroy(iterator); + list_destroy(freelist); + } + return rc; +} + +int clib_package_install_executable(clib_package_t *pkg, const char *dir, int verbose) { +#ifdef PATH_MAX + long path_max = PATH_MAX; +#elif defined(_PC_PATH_MAX) + long path_max = pathconf(dir, _PC_PATH_MAX); +#else + long path_max = 4096; +#endif + + int rc; + char *url = NULL; + char *file = NULL; + char *tarball = NULL; + char *command = NULL; + char *unpack_dir = NULL; + char *deps = NULL; + char *tmp = NULL; + char *reponame = NULL; + char dir_path[path_max]; + + _debug("install executable %s", pkg->repo); + + tmp = gettempdir(); + + if (NULL == tmp) { + if (verbose) { + logger_error("error", "gettempdir() out of memory"); + } + return -1; + } + + if (!pkg->repo) { + if (verbose) { + logger_error("error", "repo field required to install executable"); + } + return -1; + } + + reponame = strrchr(pkg->repo, '/'); + if (reponame && *reponame != '\0') + reponame++; + else { + if (verbose) { + logger_error("error", + "malformed repo field, must be in the form user/pkg"); + } + return -1; + } + + E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, pkg->version); + E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); + E_FORMAT(&tarball, "%s/%s", tmp, file); + + // TODO, move to repository + rc = http_get_file(url, tarball, NULL, 0); + + if (0 != rc) { + if (verbose) { + logger_error("error", "download failed for '%s@%s' - HTTP GET '%s'", + pkg->repo, pkg->version, url); + } + + goto cleanup; + } + + E_FORMAT(&command, "cd %s && gzip -dc %s | tar x", tmp, file); + + _debug("download url: %s", url); + _debug("file: %s", file); + _debug("tarball: %s", tarball); + _debug("command(extract): %s", command); + + // cheap untar + rc = system(command); + if (0 != rc) + goto cleanup; + + free(command); + command = NULL; + + clib_package_set_prefix(pkg, path_max); + + const char *configure = pkg->configure; + + if (0 == configure) { + configure = ":"; + } + + memset(dir_path, 0, path_max); + realpath(dir, dir_path); + + char *version = pkg->version; + if ('v' == version[0]) { + (void) version++; + } + + E_FORMAT(&unpack_dir, "%s/%s-%s", tmp, reponame, version); + + _debug("dir: %s", unpack_dir); + + if (pkg->dependencies) { + E_FORMAT(&deps, "%s/deps", unpack_dir); + _debug("deps: %s", deps); + rc = clib_package_install_dependencies(pkg, deps, verbose); + if (-1 == rc) + goto cleanup; + } + + if (!package_opts.global && pkg->makefile) { + E_FORMAT(&command, "cp -fr %s/%s/%s %s", dir_path, pkg->name, + basename(pkg->makefile), unpack_dir); + + rc = system(command); + if (0 != rc) { + goto cleanup; + } + + free(command); + } + + if (pkg->flags) { + char *flags = NULL; +#ifdef _GNU_SOURCE + char *cflags = secure_getenv("CFLAGS"); +#else + char *cflags = getenv("CFLAGS"); +#endif + + if (cflags) { + asprintf(&flags, "%s %s", cflags, pkg->flags); + } else { + asprintf(&flags, "%s", pkg->flags); + } + + setenv("CFLAGS", cflags, 1); + } + + E_FORMAT(&command, "cd %s && %s", unpack_dir, pkg->install); + + _debug("command(install): %s", command); + rc = system(command); + +cleanup: + free(tmp); + free(command); + free(tarball); + free(file); + free(url); + return rc; +} + +/** + * Install the given `pkg` in `dir` + */ + +int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { + list_iterator_t *iterator = NULL; + char *package_json = NULL; + char *pkg_dir = NULL; + char *command = NULL; + int rc = 0; + int thread_index = 0; + +#ifdef PATH_MAX + long path_max = PATH_MAX; +#elif defined(_PC_PATH_MAX) + long path_max = pathconf(dir, _PC_PATH_MAX); +#else + long path_max = 4096; +#endif + +#ifdef HAVE_PTHREADS + int max = package_opts.concurrency; +#endif + + if (0 == package_opts.prefix) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif +#ifdef _GNU_SOURCE + char *prefix = secure_getenv("PREFIX"); +#else + char *prefix = getenv("PREFIX"); +#endif + + if (prefix) { + package_opts.prefix = prefix; + } +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (0 == visited_packages) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + visited_packages = hash_new(); + // initial write because sometimes `hash_set()` crashes + hash_set(visited_packages, strdup(""), ""); + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (0 == package_opts.force && pkg && pkg->name) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + if (hash_has(visited_packages, pkg->name)) { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + return 0; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + if (!pkg || !dir) { + rc = -1; + goto cleanup; + } + + clib_package_set_prefix(pkg, path_max); + + if (!(pkg_dir = path_join(dir, pkg->name))) { + rc = -1; + goto cleanup; + } + + if (!package_opts.global) { + _debug("mkdir -p %s", pkg_dir); + // create directory for pkg + if (-1 == mkdirp(pkg_dir, 0777)) { + logger_error("error", "Could not create directory %s", pkg_dir); + rc = -1; + goto cleanup; + } + } + + /* + if (NULL == pkg->url) { + pkg->url = clib_package_url(pkg->author, pkg->repo_name, pkg->version); + + if (NULL == pkg->url) { + rc = -1; + goto cleanup; + } + } + */ + + // write clib.json or package.json + if (!(package_json = path_join(pkg_dir, pkg->filename))) { + rc = -1; + goto cleanup; + } + + if (!package_opts.global && NULL != pkg->src) { + _debug("write: %s", package_json); + if (-1 == fs_write(package_json, pkg->json)) { + if (verbose) { + logger_error("error", "Failed to write %s", package_json); + } + + rc = -1; + goto cleanup; + } + } + + if (pkg->name) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + if (!hash_has(visited_packages, pkg->name)) { + hash_set(visited_packages, strdup(pkg->name), "t"); + } +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + } + + // fetch makefile + if (!package_opts.global && pkg->makefile) { + _debug("fetch: %s/%s", pkg->repo, pkg->makefile); + repository_file_handle_t handle = repository_download_package_file(pkg->url, clib_package_get_id(pkg->author, pkg->repo_name), pkg->version, pkg->makefile, pkg_dir); + if (handle == NULL) { + goto cleanup; + } + +#ifdef HAVE_PTHREADS + repository_file_finish_download(handle); + repository_file_free(handle); +#endif + } + + // if no sources are listed, just install + if (package_opts.global || NULL == pkg->src) + goto install; + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + + if (clib_cache_has_package(pkg->author, pkg->name, pkg->version)) { + if (package_opts.skip_cache) { + clib_cache_delete_package(pkg->author, pkg->name, pkg->version); +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + goto download; + } + + if (0 != clib_cache_load_package(pkg->author, pkg->name, pkg->version, pkg_dir)) { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + goto download; + } + + if (verbose) { + logger_info("cache", pkg->repo); + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + + goto install; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + +download: + + iterator = list_iterator_new(pkg->src, LIST_HEAD); + list_node_t *source; + repository_file_handle_t *handles = malloc(max * sizeof(repository_file_handle_t)); + char* package_id = clib_package_get_id(pkg->author, pkg->repo_name); + // TODO, refactor this. + while ((source = list_iterator_next(iterator))) { + handles[thread_index] = repository_download_package_file(pkg->url, package_id, pkg->version, source->val, pkg_dir); + if (handles[thread_index] == NULL) { + list_iterator_destroy(iterator); + iterator = NULL; + rc = -1; + goto cleanup; + } + +#ifdef HAVE_PTHREADS + if (thread_index < (max-1)) { + thread_index++; + } else { + for (int j = 0; j <= thread_index; j++) { + repository_file_finish_download(handles[j]); + repository_file_free(handles[j]); + } + thread_index = 0; + } +#endif + } + +#ifdef HAVE_PTHREADS + // Here thread_index is one higher than the actual thread index. + for (int j = 0; j < thread_index; j++) { + repository_file_finish_download(handles[j]); + repository_file_free(handles[j]); + } +#endif + free(handles); + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&lock.mutex); +#endif + if (!package_opts.skip_cache) { + clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); + } +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&lock.mutex); +#endif + +install: + if (pkg->configure) { + E_FORMAT(&command, "cd %s/%s && %s", dir, pkg->name, pkg->configure); + + _debug("command(configure): %s", command); + + rc = system(command); + if (0 != rc) + goto cleanup; + } + + if (pkg->install) { + rc = clib_package_install_executable(pkg, dir, verbose); + } + + if (0 == rc) { + rc = clib_package_install_dependencies(pkg, dir, verbose); + } + +cleanup: + if (pkg_dir) + free(pkg_dir); + if (package_json) + free(package_json); + if (iterator) + list_iterator_destroy(iterator); + if (command) + free(command); + return rc; +} + +/** + * Install the given `pkg`'s dependencies in `dir` + */ +int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, + int verbose) { + if (!pkg || !dir) + return -1; + if (NULL == pkg->dependencies) + return 0; + + return install_packages(pkg->dependencies, dir, verbose); +} + +/** + * Install the given `pkg`'s development dependencies in `dir` + */ +int clib_package_install_development(clib_package_t *pkg, const char *dir, + int verbose) { + if (!pkg || !dir) + return -1; + if (NULL == pkg->development) + return 0; + + return install_packages(pkg->development, dir, verbose); +} diff --git a/src/common/clib-package-installer.h b/src/common/clib-package-installer.h new file mode 100644 index 00000000..65722e13 --- /dev/null +++ b/src/common/clib-package-installer.h @@ -0,0 +1,23 @@ +// +// clib-package-installer.h +// +// Copyright (c) 2021 Clib authors +// MIT licensed +// +#include "clib-package.h" +#include + +#ifndef CLIB_SRC_COMMON_CLIB_PACKAGE_INSTALLER_H +#define CLIB_SRC_COMMON_CLIB_PACKAGE_INSTALLER_H + +void clib_package_installer_init(registries_t registries, clib_secrets_t secrets); + +int clib_package_install(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_executable(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, int verbose); + +int clib_package_install_development(clib_package_t *pkg, const char *dir, int verbose); + +#endif diff --git a/src/common/clib-package.c b/src/common/clib-package.c index c01639ca..ed87d9a9 100755 --- a/src/common/clib-package.c +++ b/src/common/clib-package.c @@ -5,10 +5,6 @@ // MIT license // -#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) -#include -#endif - #include "asprintf/asprintf.h" #include "clib-cache.h" #include "clib-package.h" @@ -21,20 +17,14 @@ #include "mkdirp/mkdirp.h" #include "parse-repo/parse-repo.h" #include "parson/parson.h" -#include "path-join/path-join.h" #include "strdup/strdup.h" -#include "substr/substr.h" -#include "tempdir/tempdir.h" -#include -#include -#include -#include #include #include #include #ifdef HAVE_PTHREADS #include +#include #endif #ifndef DEFAULT_REPO_VERSION @@ -45,9 +35,6 @@ #define DEFAULT_REPO_OWNER "clibs" #endif -#define GITHUB_CONTENT_URL "https://raw.githubusercontent.com/" -#define GITHUB_CONTENT_URL_WITH_TOKEN "https://%s@raw.githubusercontent.com/" - #if defined(_WIN32) || defined(WIN32) || defined(__MINGW32__) || \ defined(__MINGW64__) #define setenv(k, v, _) _putenv_s(k, v) @@ -57,17 +44,6 @@ static hash_t *visited_packages = 0; #ifdef HAVE_PTHREADS -typedef struct fetch_package_file_thread_data fetch_package_file_thread_data_t; -struct fetch_package_file_thread_data { - clib_package_t *pkg; - const char *dir; - char *file; - int verbose; - pthread_t thread; - pthread_attr_t attr; - void *data; -}; - typedef struct clib_package_lock clib_package_lock_t; struct clib_package_lock { pthread_mutex_t mutex; @@ -77,8 +53,7 @@ static clib_package_lock_t lock = {PTHREAD_MUTEX_INITIALIZER}; #endif -CURLSH *clib_package_curl_share; -debug_t _debugger; +static debug_t _debugger; #define _debug(...) \ ({ \ @@ -94,7 +69,7 @@ debug_t _debugger; goto cleanup; \ }); -static clib_package_opts_t opts = { +clib_package_opts_t package_opts = { #ifdef HAVE_PTHREADS .concurrency = MAX_THREADS, #endif @@ -113,59 +88,54 @@ static inline char *json_object_get_string_safe(JSON_Object *, const char *); static inline char *json_array_get_string_safe(JSON_Array *, int); -static inline char *clib_package_file_url(const char *, const char *); - -static inline char *clib_package_slug(const char *, const char *, const char *); - -static inline char *clib_package_repo(const char *, const char *); static inline list_t *parse_package_deps(JSON_Object *); static inline int install_packages(list_t *, const char *, int); void clib_package_set_opts(clib_package_opts_t o) { - if (1 == opts.skip_cache && 0 == o.skip_cache) { - opts.skip_cache = 0; - } else if (0 == opts.skip_cache && 1 == o.skip_cache) { - opts.skip_cache = 1; + if (1 == package_opts.skip_cache && 0 == o.skip_cache) { + package_opts.skip_cache = 0; + } else if (0 == package_opts.skip_cache && 1 == o.skip_cache) { + package_opts.skip_cache = 1; } - if (1 == opts.global && 0 == o.global) { - opts.global = 0; - } else if (0 == opts.global && 1 == o.global) { - opts.global = 1; + if (1 == package_opts.global && 0 == o.global) { + package_opts.global = 0; + } else if (0 == package_opts.global && 1 == o.global) { + package_opts.global = 1; } - if (1 == opts.force && 0 == o.force) { - opts.force = 0; - } else if (0 == opts.force && 1 == o.force) { - opts.force = 1; + if (1 == package_opts.force && 0 == o.force) { + package_opts.force = 0; + } else if (0 == package_opts.force && 1 == o.force) { + package_opts.force = 1; } if (0 != o.prefix) { if (0 == strlen(o.prefix)) { - opts.prefix = 0; + package_opts.prefix = 0; } else { - opts.prefix = o.prefix; + package_opts.prefix = o.prefix; } } if (0 != o.token) { if (0 == strlen(o.token)) { - opts.token = 0; + package_opts.token = 0; } else { - opts.token = o.token; + package_opts.token = o.token; } } if (o.concurrency) { - opts.concurrency = o.concurrency; + package_opts.concurrency = o.concurrency; } else if (o.concurrency < 0) { - opts.concurrency = 0; + package_opts.concurrency = 0; } - if (opts.concurrency < 0) { - opts.concurrency = 0; + if (package_opts.concurrency < 0) { + package_opts.concurrency = 0; } } @@ -196,31 +166,11 @@ static inline char *json_array_get_string_safe(JSON_Array *array, int index) { return strdup(val); } -/** - * Build a URL for `file` of the package belonging to `url` - */ - -static inline char *clib_package_file_url(const char *url, const char *file) { - if (!url || !file) - return NULL; - - int size = strlen(url) + 1 // / - + strlen(file) + 1 // \0 - ; - - char *res = malloc(size); - if (res) { - memset(res, 0, size); - sprintf(res, "%s/%s", url, file); - } - return res; -} - /** * Build a slug */ -static inline char *clib_package_slug(const char *author, const char *name, +char *clib_package_slug(const char *author, const char *name, const char *version) { int size = strlen(author) + 1 // / + strlen(name) + 1 // @ @@ -244,11 +194,11 @@ clib_package_t *clib_package_load_from_manifest(const char *manifest, clib_package_t *pkg = NULL; if (-1 == fs_exists(manifest)) { - logger_error("error", "Missing %s", manifest); + _debug("clib-package", "%s not found", manifest); return NULL; } - logger_info("info", "reading local %s", manifest); + _debug("clib-package", "reading local %s", manifest); char *json = fs_read(manifest); if (NULL == json) @@ -283,7 +233,7 @@ clib_package_t *clib_package_load_local_manifest(int verbose) { * Build a repo */ -static inline char *clib_package_repo(const char *author, const char *name) { +char *clib_package_get_id(const char *author, const char *name) { int size = strlen(author) + 1 // / + strlen(name) + 1 // \0 ; @@ -340,100 +290,6 @@ static inline list_t *parse_package_deps(JSON_Object *obj) { return list; } -static inline int install_packages(list_t *list, const char *dir, int verbose) { - list_node_t *node = NULL; - list_iterator_t *iterator = NULL; - int rc = -1; - list_t *freelist = NULL; - - if (!list || !dir) - goto cleanup; - - iterator = list_iterator_new(list, LIST_HEAD); - if (NULL == iterator) - goto cleanup; - - freelist = list_new(); - - while ((node = list_iterator_next(iterator))) { - clib_package_dependency_t *dep = NULL; - char *slug = NULL; - clib_package_t *pkg = NULL; - int error = 1; - - dep = (clib_package_dependency_t *)node->val; - slug = clib_package_slug(dep->author, dep->name, dep->version); - if (NULL == slug) - goto loop_cleanup; - - pkg = clib_package_new_from_slug(slug, verbose); - if (NULL == pkg) - goto loop_cleanup; - - if (-1 == clib_package_install(pkg, dir, verbose)) - goto loop_cleanup; - - list_rpush(freelist, list_node_new(pkg)); - error = 0; - - loop_cleanup: - if (slug) - free(slug); - if (error) { - list_iterator_destroy(iterator); - iterator = NULL; - rc = -1; - goto cleanup; - } - } - - rc = 0; - -cleanup: - if (iterator) - list_iterator_destroy(iterator); - - if (freelist) { - iterator = list_iterator_new(freelist, LIST_HEAD); - while ((node = list_iterator_next(iterator))) { - clib_package_t *pkg = node->val; - if (pkg) - clib_package_free(pkg); - } - list_iterator_destroy(iterator); - list_destroy(freelist); - } - return rc; -} - -#ifdef HAVE_PTHREADS -static void curl_lock_callback(CURL *handle, curl_lock_data data, - curl_lock_access access, void *userptr) { - pthread_mutex_lock(&lock.mutex); -} - -static void curl_unlock_callback(CURL *handle, curl_lock_data data, - curl_lock_access access, void *userptr) { - pthread_mutex_unlock(&lock.mutex); -} - -static void init_curl_share() { - if (0 == clib_package_curl_share) { - pthread_mutex_lock(&lock.mutex); - clib_package_curl_share = curl_share_init(); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_SHARE, - CURL_LOCK_DATA_CONNECT); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_LOCKFUNC, - curl_lock_callback); - curl_share_setopt(clib_package_curl_share, CURLSHOPT_UNLOCKFUNC, - curl_unlock_callback); - curl_share_setopt(clib_package_curl_share, CURLOPT_NETRC, - CURL_NETRC_OPTIONAL); - pthread_mutex_unlock(&lock.mutex); - } -} -#endif - /** * Create a new clib package from the given `json` */ @@ -563,6 +419,24 @@ clib_package_t *clib_package_new(const char *json, int verbose) { pkg->src = NULL; } + if (!(pkg->registries = list_new())) { + goto cleanup; + } + JSON_Array* registries = json_object_get_array(json_object, "registries"); + if (registries) { + pkg->registries->free = free; + for (unsigned int i = 0; i < json_array_get_count(registries); i++) { + char *file = json_array_get_string_safe(registries, i); + _debug("file: %s", file); + if (!file) + goto cleanup; + if (!(list_rpush(pkg->registries, list_node_new(file)))) + goto cleanup; + } + } else { + _debug("no extra registries listed in clib.json or package.json file"); + } + if ((deps = json_object_get_object(json_object, "dependencies"))) { if (!(pkg->dependencies = parse_package_deps(deps))) { goto cleanup; @@ -594,15 +468,11 @@ clib_package_t *clib_package_new(const char *json, int verbose) { return pkg; } -static clib_package_t * -clib_package_new_from_slug_with_package_name(const char *slug, int verbose, - const char *file) { +static clib_package_t * clib_package_new_from_slug_with_package_name(const char *slug, const char* url, int verbose, const char *manifest_file) { char *author = NULL; char *name = NULL; char *version = NULL; - char *url = NULL; char *json_url = NULL; - char *repo = NULL; char *json = NULL; char *log = NULL; http_get_response_t *res = NULL; @@ -619,10 +489,6 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, goto error; if (!(version = parse_repo_version(slug, DEFAULT_REPO_VERSION))) goto error; - if (!(url = clib_package_url(author, name, version))) - goto error; - if (!(json_url = clib_package_file_url(url, file))) - goto error; _debug("author: %s", author); _debug("name: %s", name); @@ -633,7 +499,7 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, #endif // fetch json if (clib_cache_has_json(author, name, version)) { - if (opts.skip_cache) { + if (package_opts.skip_cache) { clib_cache_delete_json(author, name, version); goto download; } @@ -656,91 +522,69 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, if (retries-- <= 0) { goto error; } else { -#ifdef HAVE_PTHREADS - init_curl_share(); - _debug("GET %s", json_url); + _debug("Fetching package manifest for %s", slug); // clean up when retrying - http_get_free(res); - res = http_get_shared(json_url, clib_package_curl_share); -#else - res = http_get(json_url); -#endif - json = res->data; + res = repository_fetch_package_manifest(url, clib_package_get_id(author, name), version, manifest_file); + _debug("status: %d", res->status); - if (!res || !res->ok) { + if (!res || (res->status != 200 && res->status != 404)) { goto download; + } else if (res->status == 404) { + goto error; } + json = res->data; log = "fetch"; } } if (verbose) { - logger_info(log, "%s/%s:%s", author, name, file); + logger_info(log, "%s/%s:%s", author, name, manifest_file); } free(json_url); json_url = NULL; - free(name); - name = NULL; - if (json) { - // build package - pkg = clib_package_new(json, verbose); - } + // build package + pkg = clib_package_new(json, verbose); if (!pkg) goto error; - // force version number + // Set the url so that we can download the other files. + pkg->url = strdup(url); + + // force the supplied version number if the registry returned a different version from what we expected. if (pkg->version) { - if (version) { - if (0 != strcmp(version, DEFAULT_REPO_VERSION)) { + if (0 != strcmp(version, DEFAULT_REPO_VERSION) && 0 != strcmp(pkg->version, version)) { _debug("forcing version number: %s (%s)", version, pkg->version); free(pkg->version); pkg->version = version; } else { free(version); + version = NULL; } - } } else { pkg->version = version; } - // force package author (don't know how this could fail) - if (author && pkg->author) { - if (0 != strcmp(author, pkg->author)) { - free(pkg->author); - pkg->author = author; - } else { - free(author); - } - } else { + if (!pkg->author) { pkg->author = strdup(author); } - - if (!(repo = clib_package_repo(pkg->author, pkg->name))) { - goto error; + if (!pkg->repo_name) { + pkg->repo_name = strdup(name); } + free(name); + name = NULL; - if (pkg->repo) { - if (0 != strcmp(repo, pkg->repo)) { - free(url); - if (!(url = clib_package_url_from_repo(pkg->repo, pkg->version))) - goto error; - } - free(repo); - repo = NULL; - } else { - pkg->repo = repo; + if (!(pkg->repo = clib_package_get_id(pkg->author, pkg->repo_name))) { + goto error; } - pkg->url = url; - #ifdef HAVE_PTHREADS pthread_mutex_lock(&lock.mutex); #endif // cache json - if (pkg && pkg->author && pkg->name && pkg->version) { + if (pkg->author && pkg->name && pkg->version) { if (-1 == clib_cache_save_json(pkg->author, pkg->name, pkg->version, json)) { _debug("failed to cache JSON for: %s/%s@%s", pkg->author, pkg->name, @@ -766,17 +610,17 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, error: if (0 == retries) { - if (verbose && author && name && file) { - logger_warn("warning", "unable to fetch %s/%s:%s", author, name, file); + if (verbose && author && name && manifest_file) { + logger_warn("warning", "unable to fetch %s/%s:%s", author, name, manifest_file); } } free(author); free(name); - free(version); - free(url); + if (version != NULL) { + free(version); + } free(json_url); - free(repo); if (!res && json) free(json); if (res) @@ -789,80 +633,21 @@ clib_package_new_from_slug_with_package_name(const char *slug, int verbose, /** * Create a package from the given repo `slug` */ - -clib_package_t *clib_package_new_from_slug(const char *slug, int verbose) { +clib_package_t *clib_package_new_from_slug_and_url(const char *slug, const char* url, int verbose) { clib_package_t *package = NULL; - const char *name = NULL; unsigned int i = 0; do { - name = manifest_names[i]; - package = clib_package_new_from_slug_with_package_name(slug, verbose, name); + const char *manifest_name = manifest_names[i]; + package = clib_package_new_from_slug_with_package_name(slug, url, verbose, manifest_name); if (NULL != package) { - package->filename = (char *)name; + package->filename = (char *) manifest_name; } } while (NULL != manifest_names[++i] && NULL == package); return package; } -/** - * Get a slug for the package `author/name@version` - */ - -char *clib_package_url(const char *author, const char *name, - const char *version) { - if (!author || !name || !version) - return NULL; - int size = strlen(GITHUB_CONTENT_URL) + strlen(author) + 1 // / - + strlen(name) + 1 // / - + strlen(version) + 1 // \0 - ; - - if (0 != opts.token) { - size += strlen(opts.token); - size += 1; // @ - } - - char *slug = malloc(size); - if (slug) { - memset(slug, '\0', size); - if (0 != opts.token) { - sprintf(slug, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s/%s", opts.token, - author, name, version); - } else { - sprintf(slug, GITHUB_CONTENT_URL "%s/%s/%s", author, name, version); - } - } - - return slug; -} - -char *clib_package_url_from_repo(const char *repo, const char *version) { - if (!repo || !version) - return NULL; - int size = strlen(GITHUB_CONTENT_URL) + strlen(repo) + 1 // / - + strlen(version) + 1 // \0 - ; - - if (0 != opts.token) { - size += strlen(opts.token); - size += 1; // @ - } - - char *slug = malloc(size); - if (slug) { - memset(slug, '\0', size); - if (0 != opts.token) { - sprintf(slug, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s", opts.token, repo, - version); - } else { - sprintf(slug, GITHUB_CONTENT_URL "%s/%s", repo, version); - } - } - return slug; -} - /** * Parse the package author from the given `slug` */ @@ -910,170 +695,14 @@ clib_package_dependency_t *clib_package_dependency_new(const char *repo, return dep; } -static int fetch_package_file_work(clib_package_t *pkg, const char *dir, - char *file, int verbose) { - char *url = NULL; - char *path = NULL; - int saved = 0; - int rc = 0; - - _debug("fetch file: %s/%s", pkg->repo, file); - - if (NULL == pkg) { - return 1; - } - - if (NULL == pkg->url) { - return 1; - } - if (0 == strncmp(file, "http", 4)) { - url = strdup(file); - } else if (!(url = clib_package_file_url(pkg->url, file))) { - return 1; - } - - _debug("file URL: %s", url); - - if (!(path = path_join(dir, basename(file)))) { - rc = 1; - goto cleanup; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (1 == opts.force || -1 == fs_exists(path)) { - if (verbose) { - logger_info("fetch", "%s:%s", pkg->repo, file); - fflush(stdout); - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - - rc = http_get_file_shared(url, path, clib_package_curl_share); - saved = 1; - } else { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (-1 == rc) { - if (verbose) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - logger_error("error", "unable to fetch %s:%s", pkg->repo, file); - fflush(stderr); - rc = 1; -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto cleanup; - } - } - - if (saved) { - if (verbose) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - logger_info("save", path); - fflush(stdout); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - } - -cleanup: - - free(url); - free(path); - return rc; -} - -#ifdef HAVE_PTHREADS -static void *fetch_package_file_thread(void *arg) { - fetch_package_file_thread_data_t *data = arg; - int *status = malloc(sizeof(int)); - int rc = - fetch_package_file_work(data->pkg, data->dir, data->file, data->verbose); - *status = rc; - (void)data->pkg->refs--; - pthread_exit((void *)status); - return (void *)(intptr_t)rc; -} -#endif - -/** - * Fetch a file associated with the given `pkg`. - * - * Returns 0 on success. - */ - -static int fetch_package_file(clib_package_t *pkg, const char *dir, char *file, - int verbose, void **data) { -#ifndef HAVE_PTHREADS - return fetch_package_file_work(pkg, dir, file, verbose); -#else - fetch_package_file_thread_data_t *fetch = malloc(sizeof(*fetch)); - int rc = 0; - - if (0 == fetch) { - return -1; - } - - *data = 0; - - memset(fetch, 0, sizeof(*fetch)); - - fetch->pkg = pkg; - fetch->dir = dir; - fetch->file = file; - fetch->verbose = verbose; - - rc = pthread_attr_init(&fetch->attr); - - if (0 != rc) { - free(fetch); - return rc; - } - - (void)pkg->refs++; - rc = pthread_create(&fetch->thread, NULL, fetch_package_file_thread, fetch); - - if (0 != rc) { - pthread_attr_destroy(&fetch->attr); - free(fetch); - return rc; - } - - rc = pthread_attr_destroy(&fetch->attr); - - if (0 != rc) { - pthread_cancel(fetch->thread); - free(fetch); - return rc; - } - - *data = fetch; - - return rc; -#endif -} - -static void set_prefix(clib_package_t *pkg, long path_max) { - if (NULL != opts.prefix || NULL != pkg->prefix) { +void clib_package_set_prefix(clib_package_t *pkg, long path_max) { + if (NULL != package_opts.prefix || NULL != pkg->prefix) { char path[path_max]; memset(path, 0, path_max); - if (opts.prefix) { - realpath(opts.prefix, path); + if (package_opts.prefix) { + realpath(package_opts.prefix, path); } else { realpath(pkg->prefix, path); } @@ -1084,543 +713,6 @@ static void set_prefix(clib_package_t *pkg, long path_max) { } } -int clib_package_install_executable(clib_package_t *pkg, const char *dir, - int verbose) { -#ifdef PATH_MAX - long path_max = PATH_MAX; -#elif defined(_PC_PATH_MAX) - long path_max = pathconf(dir, _PC_PATH_MAX); -#else - long path_max = 4096; -#endif - - int rc; - char *url = NULL; - char *file = NULL; - char *tarball = NULL; - char *command = NULL; - char *unpack_dir = NULL; - char *deps = NULL; - char *tmp = NULL; - char *reponame = NULL; - char dir_path[path_max]; - - _debug("install executable %s", pkg->repo); - - tmp = gettempdir(); - - if (NULL == tmp) { - if (verbose) { - logger_error("error", "gettempdir() out of memory"); - } - return -1; - } - - if (!pkg->repo) { - if (verbose) { - logger_error("error", "repo field required to install executable"); - } - return -1; - } - - reponame = strrchr(pkg->repo, '/'); - if (reponame && *reponame != '\0') - reponame++; - else { - if (verbose) { - logger_error("error", - "malformed repo field, must be in the form user/pkg"); - } - return -1; - } - - E_FORMAT(&url, "https://github.com/%s/archive/%s.tar.gz", pkg->repo, - pkg->version); - - E_FORMAT(&file, "%s-%s.tar.gz", reponame, pkg->version); - - E_FORMAT(&tarball, "%s/%s", tmp, file); - - rc = http_get_file_shared(url, tarball, clib_package_curl_share); - - if (0 != rc) { - if (verbose) { - logger_error("error", "download failed for '%s@%s' - HTTP GET '%s'", - pkg->repo, pkg->version, url); - } - - goto cleanup; - } - - E_FORMAT(&command, "cd %s && gzip -dc %s | tar x", tmp, file); - - _debug("download url: %s", url); - _debug("file: %s", file); - _debug("tarball: %s", tarball); - _debug("command(extract): %s", command); - - // cheap untar - rc = system(command); - if (0 != rc) - goto cleanup; - - free(command); - command = NULL; - - set_prefix(pkg, path_max); - - const char *configure = pkg->configure; - - if (0 == configure) { - configure = ":"; - } - - memset(dir_path, 0, path_max); - realpath(dir, dir_path); - - char *version = pkg->version; - if ('v' == version[0]) { - (void)version++; - } - - E_FORMAT(&unpack_dir, "%s/%s-%s", tmp, reponame, version); - - _debug("dir: %s", unpack_dir); - - if (pkg->dependencies) { - E_FORMAT(&deps, "%s/deps", unpack_dir); - _debug("deps: %s", deps); - rc = clib_package_install_dependencies(pkg, deps, verbose); - if (-1 == rc) - goto cleanup; - } - - if (!opts.global && pkg->makefile) { - E_FORMAT(&command, "cp -fr %s/%s/%s %s", dir_path, pkg->name, - basename(pkg->makefile), unpack_dir); - - rc = system(command); - if (0 != rc) { - goto cleanup; - } - - free(command); - } - - if (pkg->flags) { - char *flags = NULL; -#ifdef _GNU_SOURCE - char *cflags = secure_getenv("CFLAGS"); -#else - char *cflags = getenv("CFLAGS"); -#endif - - if (cflags) { - asprintf(&flags, "%s %s", cflags, pkg->flags); - } else { - asprintf(&flags, "%s", pkg->flags); - } - - setenv("CFLAGS", cflags, 1); - } - - E_FORMAT(&command, "cd %s && %s", unpack_dir, pkg->install); - - _debug("command(install): %s", command); - rc = system(command); - -cleanup: - free(tmp); - free(command); - free(tarball); - free(file); - free(url); - return rc; -} - -/** - * Install the given `pkg` in `dir` - */ - -int clib_package_install(clib_package_t *pkg, const char *dir, int verbose) { - list_iterator_t *iterator = NULL; - char *package_json = NULL; - char *pkg_dir = NULL; - char *command = NULL; - int pending = 0; - int rc = 0; - int i = 0; - -#ifdef PATH_MAX - long path_max = PATH_MAX; -#elif defined(_PC_PATH_MAX) - long path_max = pathconf(dir, _PC_PATH_MAX); -#else - long path_max = 4096; -#endif - -#ifdef HAVE_PTHREADS - int max = opts.concurrency; -#endif - -#ifdef CLIB_PACKAGE_PREFIX - if (0 == opts.prefix) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - opts.prefix = CLIB_PACKAGE_PREFIX; -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } -#endif - - if (0 == opts.prefix) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif -#ifdef _GNU_SOURCE - char *prefix = secure_getenv("PREFIX"); -#else - char *prefix = getenv("PREFIX"); -#endif - - if (prefix) { - opts.prefix = prefix; - } -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (0 == visited_packages) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - visited_packages = hash_new(); - // initial write because sometimes `hash_set()` crashes - hash_set(visited_packages, strdup(""), ""); - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - if (0 == opts.force && pkg && pkg->name) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (hash_has(visited_packages, pkg->name)) { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - return 0; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - -#ifdef HAVE_PTHREADS - fetch_package_file_thread_data_t **fetchs = 0; - if (NULL != pkg && NULL != pkg->src) { - if (pkg->src->len > 0) { - fetchs = malloc(pkg->src->len * sizeof(fetch_package_file_thread_data_t)); - } - } - - if (fetchs) { - memset(fetchs, 0, pkg->src->len * sizeof(fetch_package_file_thread_data_t)); - } - -#endif - - if (!pkg || !dir) { - rc = -1; - goto cleanup; - } - - set_prefix(pkg, path_max); - - if (!(pkg_dir = path_join(dir, pkg->name))) { - rc = -1; - goto cleanup; - } - - if (!opts.global) { - _debug("mkdir -p %s", pkg_dir); - // create directory for pkg - if (-1 == mkdirp(pkg_dir, 0777)) { - rc = -1; - goto cleanup; - } - } - - if (NULL == pkg->url) { - pkg->url = clib_package_url(pkg->author, pkg->repo_name, pkg->version); - - if (NULL == pkg->url) { - rc = -1; - goto cleanup; - } - } - - // write clib.json or package.json - if (!(package_json = path_join(pkg_dir, pkg->filename))) { - rc = -1; - goto cleanup; - } - - if (!opts.global && NULL != pkg->src) { - _debug("write: %s", package_json); - if (-1 == fs_write(package_json, pkg->json)) { - if (verbose) { - logger_error("error", "Failed to write %s", package_json); - } - - rc = -1; - goto cleanup; - } - } - - if (pkg->name) { -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - if (!hash_has(visited_packages, pkg->name)) { - hash_set(visited_packages, strdup(pkg->name), "t"); - } -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - } - - // fetch makefile - if (!opts.global && pkg->makefile) { - _debug("fetch: %s/%s", pkg->repo, pkg->makefile); - void *fetch = 0; - rc = fetch_package_file(pkg, pkg_dir, pkg->makefile, verbose, &fetch); - if (0 != rc) { - goto cleanup; - } - -#ifdef HAVE_PTHREADS - if (0 != fetch) { - fetch_package_file_thread_data_t *data = fetch; - int *status; - pthread_join(data->thread, (void **)&status); - if (NULL != status) { - rc = *status; - free(status); - status = 0; - if (0 != rc) { - rc = 0; - logger_warn("warning", "unable to fetch Makefile (%s) for '%s'", - pkg->makefile, pkg->name); - } - } - } -#endif - } - - // if no sources are listed, just install - if (opts.global || NULL == pkg->src) - goto install; - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - - if (clib_cache_has_package(pkg->author, pkg->name, pkg->version)) { - if (opts.skip_cache) { - clib_cache_delete_package(pkg->author, pkg->name, pkg->version); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto download; - } - - if (0 != clib_cache_load_package(pkg->author, pkg->name, pkg->version, - pkg_dir)) { -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - goto download; - } - - if (verbose) { - logger_info("cache", pkg->repo); - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - - goto install; - } - -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - -download: - - iterator = list_iterator_new(pkg->src, LIST_HEAD); - list_node_t *source; - - while ((source = list_iterator_next(iterator))) { - void *fetch = NULL; - rc = fetch_package_file(pkg, pkg_dir, source->val, verbose, &fetch); - - if (0 != rc) { - list_iterator_destroy(iterator); - iterator = NULL; - rc = -1; - goto cleanup; - } - -#ifdef HAVE_PTHREADS - if (i < 0) { - i = 0; - } - - fetchs[i] = fetch; - - (void)pending++; - - if (i < (max - 1)) { - (void)i++; - } else { - for (int j = 0; j <= i; j++) { - fetch_package_file_thread_data_t *data = fetchs[j]; - int *status; - pthread_join(data->thread, (void **)&status); - free(data); - fetchs[j] = NULL; - - (void)pending--; - - if (NULL != status) { - rc = *status; - free(status); - status = 0; - } - - if (0 != rc) { - rc = -1; - goto cleanup; - } - } - i = 0; - } -#endif - } - -#ifdef HAVE_PTHREADS - // Here there are i-1 threads running. - for (int j = 0; j < i; j++) { - fetch_package_file_thread_data_t *data = fetchs[j]; - int *status; - - pthread_join(data->thread, (void **)&status); - - (void)pending--; - free(data); - fetchs[j] = NULL; - - if (NULL != status) { - rc = *status; - free(status); - status = 0; - } - - if (0 != rc) { - rc = -1; - goto cleanup; - } - } -#endif - -#ifdef HAVE_PTHREADS - pthread_mutex_lock(&lock.mutex); -#endif - clib_cache_save_package(pkg->author, pkg->name, pkg->version, pkg_dir); -#ifdef HAVE_PTHREADS - pthread_mutex_unlock(&lock.mutex); -#endif - -install: - if (pkg->configure) { - E_FORMAT(&command, "cd %s/%s && %s", dir, pkg->name, pkg->configure); - - _debug("command(configure): %s", command); - - rc = system(command); - if (0 != rc) - goto cleanup; - } - - if (0 == rc && pkg->install) { - rc = clib_package_install_executable(pkg, dir, verbose); - } - - if (0 == rc) { - rc = clib_package_install_dependencies(pkg, dir, verbose); - } - -cleanup: - if (pkg_dir) - free(pkg_dir); - if (package_json) - free(package_json); - if (iterator) - list_iterator_destroy(iterator); - if (command) - free(command); -#ifdef HAVE_PTHREADS - if (NULL != pkg && NULL != pkg->src) { - if (pkg->src->len > 0) { - if (fetchs) { - free(fetchs); - } - } - } - fetchs = NULL; -#endif - return rc; -} - -/** - * Install the given `pkg`'s dependencies in `dir` - */ - -int clib_package_install_dependencies(clib_package_t *pkg, const char *dir, - int verbose) { - if (!pkg || !dir) - return -1; - if (NULL == pkg->dependencies) - return 0; - - return install_packages(pkg->dependencies, dir, verbose); -} - -/** - * Install the given `pkg`'s development dependencies in `dir` - */ - -int clib_package_install_development(clib_package_t *pkg, const char *dir, - int verbose) { - if (!pkg || !dir) - return -1; - if (NULL == pkg->development) - return 0; - - return install_packages(pkg->development, dir, verbose); -} - /** * Free a clib package */ @@ -1688,6 +780,4 @@ void clib_package_cleanup() { hash_free(visited_packages); visited_packages = 0; } - - curl_share_cleanup(clib_package_curl_share); } diff --git a/src/common/clib-package.h b/src/common/clib-package.h index 3007ec83..c6e4ce9d 100644 --- a/src/common/clib-package.h +++ b/src/common/clib-package.h @@ -37,6 +37,7 @@ typedef struct { list_t *dependencies; list_t *development; list_t *src; + list_t *registries; void *data; // user data unsigned int refs; } clib_package_t; @@ -51,21 +52,19 @@ typedef struct { } clib_package_opts_t; extern CURLSH *clib_package_curl_share; +// TODO, move to a separate file. +extern clib_package_opts_t package_opts; void clib_package_set_opts(clib_package_opts_t opts); clib_package_t *clib_package_new(const char *, int); -clib_package_t *clib_package_new_from_slug(const char *, int); +clib_package_t *clib_package_new_from_slug_and_url(const char* slug, const char* url, int); clib_package_t *clib_package_load_from_manifest(const char *, int); clib_package_t *clib_package_load_local_manifest(int); -char *clib_package_url(const char *, const char *, const char *); - -char *clib_package_url_from_repo(const char *repo, const char *version); - char *clib_package_parse_version(const char *); char *clib_package_parse_author(const char *); @@ -75,14 +74,11 @@ char *clib_package_parse_name(const char *); clib_package_dependency_t *clib_package_dependency_new(const char *, const char *); -int clib_package_install_executable(clib_package_t *pkg, const char *dir, - int verbose); - -int clib_package_install(clib_package_t *, const char *, int); +char *clib_package_get_id(const char *, const char *); -int clib_package_install_dependencies(clib_package_t *, const char *, int); +char *clib_package_slug(const char *author, const char *name, const char* version); -int clib_package_install_development(clib_package_t *, const char *, int); +void clib_package_set_prefix(clib_package_t *pkg, long path_max); void clib_package_free(clib_package_t *); diff --git a/src/common/clib-release-info.c b/src/common/clib-release-info.c index c64a2dbf..3c75005c 100644 --- a/src/common/clib-release-info.c +++ b/src/common/clib-release-info.c @@ -18,7 +18,7 @@ static debug_t debugger; const char *clib_release_get_latest_tag(void) { debug_init(&debugger, "clib-release-info"); - http_get_response_t *res = http_get(LATEST_RELEASE_ENDPOINT); + http_get_response_t *res = http_get(LATEST_RELEASE_ENDPOINT, NULL, 0); JSON_Value *root_json = NULL; JSON_Object *json_object = NULL; diff --git a/src/common/clib-secrets.c b/src/common/clib-secrets.c new file mode 100644 index 00000000..faef7f9d --- /dev/null +++ b/src/common/clib-secrets.c @@ -0,0 +1,85 @@ +#include "clib-secrets.h" +#include "fs/fs.h" +#include "list/list.h" +#include "logger/logger.h" +#include "parson/parson.h" +#include +#include +#include +#include + +struct clib_secret { + char *hostname; + char *secret; +}; + +struct clib_secret_handle { + list_t *secrets; +}; + +clib_secrets_t clib_secrets_load_from_file(const char *file) { + if (-1 == fs_exists(file)) { + //logger_warn("warning", "Secrets file %s does not exist.", file); + return NULL; + } + + //logger_info("info", "Reading secrets from %s.", file); + + char* json = NULL; + json = fs_read(file); + if (NULL == json) { + return NULL; + } + + JSON_Value *root = json_parse_string(json); + if (root == NULL) { + logger_error("error", "unable to parse secrets JSON"); + return NULL; + } + + JSON_Object *json_object = NULL; + if (!(json_object = json_value_get_object(root))) { + logger_error("error", "Invalid json file, root is not an object."); + return NULL; + } + + clib_secrets_t handle = malloc(sizeof(struct clib_secret_handle)); + + if (!(handle->secrets = list_new())) { + free(json); + free(handle); + + return NULL; + } + + for (unsigned int i = 0; i < json_object_get_count(json_object); i++) { + const char *domain = json_object_get_name(json_object, i); + const char *secret = json_object_get_string(json_object, domain); + + struct clib_secret *secret_struct = malloc(sizeof(struct clib_secret)); + secret_struct->hostname = strdup(domain); + secret_struct->secret = strdup(secret); + + list_rpush(handle->secrets, list_node_new(secret_struct)); + } + + return handle; +} + +char *clib_secret_find_for_hostname(clib_secrets_t secrets, const char *hostname) { + if (secrets == NULL) { + return NULL; + } + list_iterator_t *iterator = list_iterator_new(secrets->secrets, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(iterator))) { + struct clib_secret *secret = node->val; + if (strcmp(hostname, secret->hostname) == 0) { + list_iterator_destroy(iterator); + return secret->secret; + } + } + + list_iterator_destroy(iterator); + return NULL; +} diff --git a/src/common/clib-secrets.h b/src/common/clib-secrets.h new file mode 100644 index 00000000..c788302e --- /dev/null +++ b/src/common/clib-secrets.h @@ -0,0 +1,10 @@ +#ifndef CLIB_SRC_COMMON_CLIB_SECRETS_H +#define CLIB_SRC_COMMON_CLIB_SECRETS_H + +typedef struct clib_secret_handle* clib_secrets_t; + +clib_secrets_t clib_secrets_load_from_file(const char* file); + +char* clib_secret_find_for_hostname(clib_secrets_t secrets, const char* hostname); + +#endif//CLIB_SRC_COMMON_CLIB_SECRETS_H diff --git a/deps/wiki-registry/wiki-registry.c b/src/registry/github-registry.c similarity index 63% rename from deps/wiki-registry/wiki-registry.c rename to src/registry/github-registry.c index e06e8c5a..a19b42d0 100644 --- a/deps/wiki-registry/wiki-registry.c +++ b/src/registry/github-registry.c @@ -1,84 +1,58 @@ - // -// wiki-registry.c +// github-registry.c // -// Copyright (c) 2014 Stephen Mathieson +// Copyright (c) 2021 Elbert van de Put +// Based on work by Stephen Mathieson // MIT licensed // - -#include -#include -#include -#include "gumbo-parser/gumbo.h" -#include "gumbo-text-content/gumbo-text-content.h" +#include "github-registry.h" +#include "case/case.h" #include "gumbo-get-element-by-id/get-element-by-id.h" #include "gumbo-get-elements-by-tag-name/get-elements-by-tag-name.h" +#include "gumbo-text-content/gumbo-text-content.h" #include "http-get/http-get.h" -#include "list/list.h" -#include "substr/substr.h" +#include "registry-internal.h" #include "strdup/strdup.h" -#include "case/case.h" +#include "substr/substr.h" #include "trim/trim.h" -#include "wiki-registry.h" - -// -// TODO find dox on gumbo so the node iteration isn't so ugly -// - -/** - * Create a new wiki package. - */ - -static wiki_package_t * -wiki_package_new() { - wiki_package_t *pkg = malloc(sizeof(wiki_package_t)); - if (pkg) { - pkg->repo = NULL; - pkg->href = NULL; - pkg->description = NULL; - pkg->category = NULL; - } - return pkg; -} - -/** - * Add `href` to the given `package`. - */ +#include +#include +#include -static void -add_package_href(wiki_package_t *self) { - size_t len = strlen(self->repo) + 20; // https://github.com/ \0 - self->href = malloc(len); - if (self->href) - sprintf(self->href, "https://github.com/%s", self->repo); -} +#define GITHUB_BASE_URL "https://github.com/" /** * Parse the given wiki `li` into a package. */ - -static wiki_package_t * -parse_li(GumboNode *li) { - wiki_package_t *self = wiki_package_new(); +static registry_package_ptr_t parse_li(GumboNode *li) { + registry_package_ptr_t self = registry_package_new(); char *text = NULL; - if (!self) goto cleanup; + if (!self) + goto cleanup; text = gumbo_text_content(li); - if (!text) goto cleanup; + if (!text) + goto cleanup; // TODO support unicode dashes char *tok = strstr(text, " - "); - if (!tok) goto cleanup; + if (!tok) + goto cleanup; int pos = tok - text; - self->repo = substr(text, 0, pos); + self->id = substr(text, 0, pos); self->description = substr(text, pos + 3, -1); - if (!self->repo || !self->description) goto cleanup; + if (!self->id || !self->description) + goto cleanup; trim(self->description); - trim(self->repo); + trim(self->id); - add_package_href(self); + size_t len = strlen(self->id) + 20;// https://github.com/ \0 + self->href = malloc(len); + if (!self->href) + goto cleanup; + sprintf(self->href, GITHUB_BASE_URL "%s", self->id); cleanup: free(text); @@ -88,9 +62,7 @@ parse_li(GumboNode *li) { /** * Parse a list of packages from the given `html` */ - -list_t * -wiki_registry_parse(const char *html) { +list_t *wiki_registry_parse(const char *html) { GumboOutput *output = gumbo_parse(html); list_t *pkgs = list_new(); @@ -105,7 +77,8 @@ wiki_registry_parse(const char *html) { char *category = gumbo_text_content(heading); // die if we failed to parse a category, as it's // almost certinaly a malloc error - if (!category) break; + if (!category) + break; trim(case_lower(category)); GumboVector *siblings = &heading->parent->v.element.children; size_t pos = heading->index_within_parent; @@ -125,13 +98,15 @@ wiki_registry_parse(const char *html) { list_iterator_t *li_iterator = list_iterator_new(lis, LIST_HEAD); list_node_t *li_node; while ((li_node = list_iterator_next(li_iterator))) { - wiki_package_t *package = parse_li(li_node->val); + registry_package_ptr_t package = parse_li(li_node->val); if (package && package->description) { package->category = strdup(category); list_rpush(pkgs, list_node_new(package)); } else { // failed to parse package - if (package) wiki_package_free(package); + if (package) + logger_error("error", "Github registry could not parse entry:", li_node->val); + registry_package_free(package); } } list_iterator_destroy(li_iterator); @@ -149,26 +124,12 @@ wiki_registry_parse(const char *html) { /** * Get a list of packages from the given GitHub wiki `url`. */ - -list_t * -wiki_registry(const char *url) { - http_get_response_t *res = http_get(url); - if (!res->ok) return NULL; +list_t *github_registry_fetch(const char *url) { + http_get_response_t *res = http_get(url, NULL, 0); + if (!res->ok) + return NULL; list_t *list = wiki_registry_parse(res->data); http_get_free(res); return list; } - -/** - * Free a wiki_package_t. - */ - -void -wiki_package_free(wiki_package_t *pkg) { - free(pkg->repo); - free(pkg->href); - free(pkg->description); - free(pkg->category); - free(pkg); -} diff --git a/src/registry/github-registry.h b/src/registry/github-registry.h new file mode 100644 index 00000000..82d9ddce --- /dev/null +++ b/src/registry/github-registry.h @@ -0,0 +1,8 @@ +#ifndef CLIB_GITHUB_REGISTRY_H +#define CLIB_GITHUB_REGISTRY_H + +#include "list/list.h" + +list_t* github_registry_fetch(const char *url); + +#endif //CLIB_GITHUB_REGISTRY_H diff --git a/src/registry/gitlab-registry.c b/src/registry/gitlab-registry.c new file mode 100644 index 00000000..4e23850b --- /dev/null +++ b/src/registry/gitlab-registry.c @@ -0,0 +1,85 @@ +// +// gitlab-registry.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "gitlab-registry.h" +#include "http-get/http-get.h" +#include "registry-internal.h" +#include +#include +#include + +static char *string_split(char *in, char sep) { + char *next_sep = strchr(in, sep); + if (next_sep == NULL) { + return next_sep; + } + *next_sep = '\0'; + return next_sep + sizeof(char); +} + +/** + * Parse a list of packages from the given `html` + */ +static list_t *gitlab_registry_parse(const char *hostname, const char *html) { + list_t *pkgs = list_new(); + + // Try to parse the markdown file. + char *input = strdup(html); + char *line = input; + char *category = NULL; + while ((line = string_split(line, '\n'))) { + char *dash_position = strstr(line, "-"); + // The line starts with a dash, so we expect a package. + if (dash_position != NULL && dash_position - line < 4) { + + char *link_name_start = strstr(line, "[") + 1; + char *link_name_end = strstr(link_name_start, "]"); + char *link_value_start = strstr(link_name_end, "(") + 1; + char *link_value_end = strstr(link_value_start, ")"); + char *description_position = strstr(link_value_end, "-") + 1; + + registry_package_ptr_t package = registry_package_new(); + package->href = strndup(link_value_start, link_value_end - link_value_start); + package->id = strndup(link_name_start, link_name_end - link_name_start); + package->description = strdup(description_position); + package->category = strdup(category != NULL ? category : "unknown"); + list_rpush(pkgs, list_node_new(package)); + } + + char *header_position = strstr(line, "##"); + // The category starts with a ##. + if (header_position != NULL && header_position - line < 4) { + category = header_position + 2; + } + } + + free(input); + + return pkgs; +} + +/** + * Get a list of packages from the given gitlab file `url`. + */ +list_t *gitlab_registry_fetch(const char *url, const char *hostname, const char *secret) { + http_get_response_t *res; + if (secret == NULL) { + res = http_get(url, NULL, 0); + } else { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + res = http_get(url, (const char **) &authentication_header, 1); + } + if (!res->ok) { + return NULL; + } + + list_t *list = gitlab_registry_parse(hostname, res->data); + http_get_free(res); + return list; +} diff --git a/src/registry/gitlab-registry.h b/src/registry/gitlab-registry.h new file mode 100644 index 00000000..9075736e --- /dev/null +++ b/src/registry/gitlab-registry.h @@ -0,0 +1,8 @@ +#ifndef CLIB_GITLAB_REGISTRY_H +#define CLIB_GITLAB_REGISTRY_H + +#include "list/list.h" + +list_t* gitlab_registry_fetch(const char* url, const char* hostname, const char* secret); + +#endif //CLIB_GITLAB_REGISTRY_H diff --git a/src/registry/registry-internal.h b/src/registry/registry-internal.h new file mode 100644 index 00000000..770fba06 --- /dev/null +++ b/src/registry/registry-internal.h @@ -0,0 +1,23 @@ +// +// registry-internal.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +// DO NOT INCLUDE. THIS HEADER IS INTERNAL ONLY +#ifndef REGISTRY_INTERNAL_H +#define REGISTRY_INTERNAL_H +#include "registry.h" + +struct registry_package_t { + char *id; + char *href; + char *description; + char *category; +}; + +registry_package_ptr_t registry_package_new(); + +void registry_package_free(registry_package_ptr_t pkg); + +#endif diff --git a/src/registry/registry-manager.c b/src/registry/registry-manager.c new file mode 100644 index 00000000..ed78d92d --- /dev/null +++ b/src/registry/registry-manager.c @@ -0,0 +1,81 @@ +// +// registry-manager.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "registry-manager.h" +#include +#include "url/url.h" + +#define CLIB_WIKI_URL "https://github.com/clibs/clib/wiki/Packages" + +registries_t registry_manager_init_registries(list_t* registry_urls, clib_secrets_t secrets) { + list_t* registries = list_new(); + + if (registry_urls != NULL) { + // Add all the registries that were provided. + list_iterator_t *registry_iterator = list_iterator_new(registry_urls, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(registry_iterator))) { + char *url = node->val; + url_data_t *parsed = url_parse(url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + registry_ptr_t registry = registry_create(url, secret); + if (registry != NULL) { + list_rpush(registries, list_node_new(registry)); + } + } + list_iterator_destroy(registry_iterator); + } + + // And add the default registry. + registry_ptr_t registry = registry_create(CLIB_WIKI_URL, NULL); + list_rpush(registries, list_node_new(registry)); + + return registries; +} + +void registry_manager_fetch_registries(registries_t registries) { + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t reg; + while ((reg = registry_iterator_next(it))) { + if (!registry_fetch(reg)) { + printf("REGISTRY: could not list packages from. %s secret is: %s\n", registry_get_url(reg), registry_get_secret(reg)); + } + } + registry_iterator_destroy(it); +} + +registry_package_ptr_t registry_manager_find_package(registries_t registries, const char* package_id) { + registry_iterator_t it = registry_iterator_new(registries); + registry_ptr_t reg; + while ((reg = registry_iterator_next(it))) { + registry_package_ptr_t package = registry_find_package(reg, package_id); + if (package != NULL) { + registry_iterator_destroy(it); + return package; + } + } + registry_iterator_destroy(it); + + return NULL; +} + +registry_iterator_t registry_iterator_new(registries_t registries) { + return list_iterator_new(registries, LIST_HEAD); +} + +registry_ptr_t registry_iterator_next(registry_iterator_t iterator) { + list_node_t *node = list_iterator_next(iterator); + if (node == NULL) { + return NULL; + } + + return (registry_ptr_t) node->val; +} + +void registry_iterator_destroy(registry_iterator_t iterator) { + list_iterator_destroy(iterator); +} diff --git a/src/registry/registry-manager.h b/src/registry/registry-manager.h new file mode 100644 index 00000000..4eaa8962 --- /dev/null +++ b/src/registry/registry-manager.h @@ -0,0 +1,44 @@ +// +// registry-manager.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H +#define CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H + +#include "clib-secrets.h" +#include "list/list.h" +#include "registry.h" + +// Contains an abstraction for handling multiple registries + +typedef list_t* registries_t; +typedef list_iterator_t* registry_iterator_t; + +/** + * Initializes all registies specified by the urls + * @param registry_urls + * @param secrets + * @return + */ +registries_t registry_manager_init_registries(list_t* registry_urls, clib_secrets_t secrets); + +void registry_manager_fetch_registries(registries_t registries); + +/** + * An iterator through the registries. + */ +registry_iterator_t registry_iterator_new(registries_t registry); +registry_ptr_t registry_iterator_next(registry_iterator_t iterator); +void registry_iterator_destroy(registry_iterator_t iterator); + +/** + * Search the registry for a package + * @param registry a registry handle + * @param package_id the identifier of the package "/" + * @return a pointer to the package if it could be found or NULL + */ +registry_package_ptr_t registry_manager_find_package(registries_t registries, const char* package_id); + +#endif//CLIB_SRC_REGISTRY_REGISTRY_MANAGER_H diff --git a/src/registry/registry.c b/src/registry/registry.c new file mode 100644 index 00000000..77084fca --- /dev/null +++ b/src/registry/registry.c @@ -0,0 +1,172 @@ +// +// registry.c +// +// Copyright (c) 2020 clib authors +// MIT licensed +// + +#include "github-registry.h" +#include "gitlab-registry.h" +#include "gumbo-parser/gumbo.h" +#include "list/list.h" +#include "registry-internal.h" +#include "url/url.h" +#include +#include +#include +#include + +enum registry_type_t { + REGISTRY_TYPE_GITHUB, + REGISTRY_TYPE_GITLAB, +}; + +struct registry_t { + enum registry_type_t type; + char *url; + char *hostname; + char *secret; + list_t *packages; +}; + +/** + * Create a new registry package. + */ +registry_package_ptr_t registry_package_new() { + registry_package_ptr_t pkg = malloc(sizeof(struct registry_package_t)); + if (pkg) { + pkg->id = NULL; + pkg->href = NULL; + pkg->description = NULL; + pkg->category = NULL; + } + return pkg; +} + +/** + * Release the memory held by the package. + */ +void registry_package_free(registry_package_ptr_t pkg) { + free(pkg->id); + free(pkg->href); + free(pkg->description); + free(pkg->category); + free(pkg); +} + +registry_ptr_t registry_create(const char *url, const char *secret) { + registry_ptr_t registry = malloc(sizeof(struct registry_t)); + registry->url = strdup(url); + registry->secret = strdup(secret); + registry->packages = NULL; + url_data_t *parsed = url_parse(url); + registry->hostname = strdup(parsed->hostname); + url_free(parsed); + + if (strstr(registry->hostname, "github.com") != NULL) { + registry->type = REGISTRY_TYPE_GITHUB; + } else if (strstr(registry->hostname, "gitlab") != NULL) { + registry->type = REGISTRY_TYPE_GITLAB; + } else { + logger_error("error", "Registry type (%s) not supported, currently github.com, gitlab.com and self-hosted gitlab are supported.", registry->url); + registry_free(registry); + + return NULL; + } + + return registry; +} + +void registry_free(registry_ptr_t registry) { + free(registry->url); + free(registry->hostname); + if (registry->packages != NULL) { + list_iterator_t *it = list_iterator_new(registry->packages, LIST_HEAD); + list_node_t *node; + while ((node = list_iterator_next(it))) { + registry_package_free(node->val); + } + list_iterator_destroy(it); + list_destroy(registry->packages); + } + free(registry); +} + +const char *registry_get_url(registry_ptr_t registry) { + return registry->url; +} + +const char* registry_get_secret(registry_ptr_t registry) { + return registry->secret; +} + +bool registry_fetch(registry_ptr_t registry) { + switch (registry->type) { + case REGISTRY_TYPE_GITLAB: + registry->packages = gitlab_registry_fetch(registry->url, registry->hostname, registry->secret); + if (registry->packages != NULL) { + return true; + } + break; + case REGISTRY_TYPE_GITHUB: + registry->packages = github_registry_fetch(registry->url); + if (registry->packages != NULL) { + return true; + } + break; + default: + registry->packages = list_new(); + return false; + } + + logger_error("error", "Fetching package list from (%s) failed.", registry->url); + registry->packages = list_new(); + return false; +} + +registry_package_iterator_t registry_package_iterator_new(registry_ptr_t registry) { + return list_iterator_new(registry->packages, LIST_HEAD); +} + +registry_package_ptr_t registry_package_iterator_next(registry_package_iterator_t iterator) { + list_node_t *node = list_iterator_next(iterator); + if (node == NULL) { + return NULL; + } + + return (registry_package_ptr_t) node->val; +} + +void registry_package_iterator_destroy(registry_package_iterator_t iterator) { + list_iterator_destroy(iterator); +} + +registry_package_ptr_t registry_find_package(registry_ptr_t registry, const char *package_id) { + registry_package_iterator_t it = registry_package_iterator_new(registry); + registry_package_ptr_t pack; + while ((pack = registry_package_iterator_next(it))) { + if (0 == strcmp(package_id, pack->id)) { + registry_package_iterator_destroy(it); + return pack; + } + } + registry_package_iterator_destroy(it); + + return NULL; +} + +char *registry_package_get_id(registry_package_ptr_t package) { + return package->id; +} + +char *registry_package_get_href(registry_package_ptr_t package) { + return package->href; +} + +char *registry_package_get_description(registry_package_ptr_t package) { + return package->description; +} + +char *registry_package_get_category(registry_package_ptr_t package) { + return package->category; +} \ No newline at end of file diff --git a/src/registry/registry.h b/src/registry/registry.h new file mode 100644 index 00000000..237e850a --- /dev/null +++ b/src/registry/registry.h @@ -0,0 +1,71 @@ +// +// registry.h +// +// Copyright (c) 2020 clib authors +// MIT licensed +// + +#ifndef REGISTRY_H +#define REGISTRY_H 1 + +#include + +typedef struct registry_package_t * registry_package_ptr_t; +typedef struct registry_t * registry_ptr_t; +typedef list_iterator_t* registry_package_iterator_t; + +/** + * Create a new registry for the given url. + * @param url the url of the registry. + * @param secret the secret to authenticate with this registry of NULL if no secret is required. + * @return a handle to the registry + */ +registry_ptr_t registry_create(const char* url, const char* secret); + +/** + * Free the memory held by the registry. + * @param registry + */ +void registry_free(registry_ptr_t registry); + +/** + * Fetch the list of packages from the registry. + * @param registry + */ +bool registry_fetch(registry_ptr_t registry); + +/** + * Get the url for the registry + * @param registry + * @return the url + */ +const char* registry_get_url(registry_ptr_t registry); + +/** + * Get the secret for this registry + * @param registry + * @return the secret or NULL if there is no secret. + */ +const char* registry_get_secret(registry_ptr_t registry); + +/** + * An iterator through the packages in the registry. + */ +registry_package_iterator_t registry_package_iterator_new(registry_ptr_t registry); +registry_package_ptr_t registry_package_iterator_next(registry_package_iterator_t iterator); +void registry_package_iterator_destroy(registry_package_iterator_t iterator); + +/** + * Search the registry for a package + * @param registry a registry handle + * @param package_id the identifier of the package "/" + * @return a pointer to the package if it could be found or NULL + */ +registry_package_ptr_t registry_find_package(registry_ptr_t registry, const char* package_id); + +char* registry_package_get_id(registry_package_ptr_t package); +char* registry_package_get_href(registry_package_ptr_t package); +char* registry_package_get_description(registry_package_ptr_t package); +char* registry_package_get_category(registry_package_ptr_t package); + +#endif \ No newline at end of file diff --git a/src/repository/github-repository.c b/src/repository/github-repository.c new file mode 100644 index 00000000..99a578d0 --- /dev/null +++ b/src/repository/github-repository.c @@ -0,0 +1,37 @@ +// +// github-repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "github-repository.h" +#include +#include +#include + +#define GITHUB_CONTENT_URL "https://raw.githubusercontent.com/" +#define GITHUB_CONTENT_URL_WITH_TOKEN "https://%s@raw.githubusercontent.com/" + +char *github_repository_get_url_for_file(const char *hostname, const char *package_id, const char *version, const char *file_path, const char *secret) { + + int size = strlen(GITHUB_CONTENT_URL) + strlen(package_id) + 1// / + + strlen(version) + 1 + strlen(file_path) + 1 // \0 + ; + + if (secret != NULL) { + size += strlen(secret); + size += 1;// @ + } + + char *url = malloc(size); + if (url) { + memset(url, '\0', size); + if (secret != NULL) { + sprintf(url, GITHUB_CONTENT_URL_WITH_TOKEN "%s/%s/%s", secret, package_id, version, file_path); + } else { + sprintf(url, GITHUB_CONTENT_URL "%s/%s/%s", package_id, version, file_path); + } + } + + return url; +} diff --git a/src/repository/github-repository.h b/src/repository/github-repository.h new file mode 100644 index 00000000..93579775 --- /dev/null +++ b/src/repository/github-repository.h @@ -0,0 +1,12 @@ +// +// github-repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H + +char* github_repository_get_url_for_file(const char* hostname, const char*package_id, const char* version, const char *file, const char* secret); + +#endif//CLIB_SRC_REPOSITORY_GITHUB_REPOSITORY_H diff --git a/src/repository/gitlab-repository.c b/src/repository/gitlab-repository.c new file mode 100644 index 00000000..666282c9 --- /dev/null +++ b/src/repository/gitlab-repository.c @@ -0,0 +1,31 @@ +// +// gitlab-repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "gitlab-repository.h" +#include +#include +#include +#include +#include "str-replace/str-replace.h" + +#define GITLAB_API_V4_URL "https://%s/api/v4%s/repository/files/%s/raw?ref=%s" + +// GET :hostname/api/v4/projects/:id/repository/files/:file_path/raw +char *gitlab_repository_get_url_for_file(const char *package_url, const char *slug, const char *version, const char *file, const char *secret) { + url_data_t *parsed = url_parse(package_url); + + char* encoded_filename = str_replace(file, "/", "%2F"); + + size_t size = strlen(parsed->hostname) + strlen(parsed->pathname) + strlen(encoded_filename) + strlen(GITLAB_API_V4_URL) + strlen(version) + 1; + char *url = malloc(size); + if (url) { + snprintf(url, size, GITLAB_API_V4_URL, parsed->hostname, parsed->pathname, encoded_filename, version); + } + + url_free(parsed); + + return url; +} diff --git a/src/repository/gitlab-repository.h b/src/repository/gitlab-repository.h new file mode 100644 index 00000000..2adaaae3 --- /dev/null +++ b/src/repository/gitlab-repository.h @@ -0,0 +1,12 @@ +// +// gitlab-repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H + +char* gitlab_repository_get_url_for_file(const char*package_url, const char* slug, const char* version, const char *file, const char* secret); + +#endif//CLIB_SRC_REPOSITORY_GITLAB_REPOSITORY_H diff --git a/src/repository/repository.c b/src/repository/repository.c new file mode 100644 index 00000000..03341a39 --- /dev/null +++ b/src/repository/repository.c @@ -0,0 +1,264 @@ +// +// repository.c +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#include "repository.h" +#include + +#include "debug/debug.h" +#include "github-repository.h" +#include "gitlab-repository.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static debug_t _debugger; +#define _debug(...) \ + ({ \ + if (!(_debugger.name)) \ + debug_init(&_debugger, "clib-repository"); \ + debug(&_debugger, __VA_ARGS__); \ + }) + +struct repository_file_t { + char *url; + const char *dir; + const char *file; + const char *secret; + pthread_t thread; + pthread_attr_t attr; + void *data; +}; + +static pthread_mutex_t mutex; + +static clib_secrets_t secrets; + +static int fetch_package_file(const char *url, const char *dir, const char *file, const char *secret, repository_file_handle_t *thread_data_ptr); + +static void curl_lock_callback(CURL *handle, curl_lock_data data, curl_lock_access access, void *userptr) { + pthread_mutex_lock(&mutex); +} + +static void curl_unlock_callback(CURL *handle, curl_lock_data data, curl_lock_access access, void *userptr) { + pthread_mutex_unlock(&mutex); +} + +static void init_curl_share() { + if (0 == clib_package_curl_share) { + pthread_mutex_lock(&mutex); + clib_package_curl_share = curl_share_init(); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_LOCKFUNC, curl_lock_callback); + curl_share_setopt(clib_package_curl_share, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback); + curl_share_setopt(clib_package_curl_share, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + pthread_mutex_unlock(&mutex); + } +} + +void repository_init(clib_secrets_t _secrets) { + init_curl_share(); + secrets = _secrets; +} + +static char *repository_create_url_for_file(const char *package_url, const char *package_id, const char *version, const char *file_path, const char *secret) { + if (strstr(package_url, "github.com") != NULL) { + return github_repository_get_url_for_file(package_url, package_id, version, file_path, secret); + } else if (strstr(package_url, "gitlab") != NULL) { + return gitlab_repository_get_url_for_file(package_url, package_id, version, file_path, secret); + } else { + return NULL; + } +} + +http_get_response_t *repository_fetch_package_manifest(const char *package_url, const char *package_id, const char *version, const char* manifest_file) { + // Check if there is a secret for the requested repository. + url_data_t *parsed = url_parse(package_url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + char *manifest_url = repository_create_url_for_file(package_url, package_id, version, manifest_file, secret); + + http_get_response_t *res; + if (secret && strstr(package_url, "gitlab") != NULL) { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + + res = http_get_shared(manifest_url, clib_package_curl_share, (const char **) &authentication_header, 1); + } else { + res = http_get_shared(manifest_url, clib_package_curl_share, NULL, 0); + } + + free(manifest_url); + + return res; +} + +repository_file_handle_t repository_download_package_file(const char *package_url, const char *package_id, const char *version, const char *file_path, const char *destination_path) { + // Check if there is a secret for the requested repository. + url_data_t *parsed = url_parse(package_url); + char *secret = clib_secret_find_for_hostname(secrets, parsed->hostname); + url_free(parsed); + char *url = repository_create_url_for_file(package_url, package_id, version, file_path, secret); + + repository_file_handle_t handle; + fetch_package_file(url, destination_path, file_path, secret, &handle); + + return handle; +} + +void repository_file_finish_download(repository_file_handle_t file) { + void *rc; + pthread_join(file->thread, &rc); + free(rc); +} + +void repository_file_free(repository_file_handle_t file) { + // TODO, check what else should be freed. + free(file); +} + +static int fetch_package_file_work(const char *url, const char *dir, const char *file, const char *secret) { + char *path = NULL; + int saved = 0; + int rc = 0; + + if (NULL == url) { + return 1; + } + + char* file_copy = strdup(file); + if (!(path = path_join(dir, basename(file_copy)))) { + rc = 1; + goto cleanup; + } + +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + + if (package_opts.force || -1 == fs_exists(path)) { + _debug("fetching %s", url); + fflush(stdout); + +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + + if (secret && strstr(url, "gitlab") != NULL) { + char *key = "PRIVATE-TOKEN"; + unsigned int size = strlen(key) + strlen(secret) + 2; + char *authentication_header = malloc(size); + snprintf(authentication_header, size, "%s:%s", key, secret); + + rc = http_get_file_shared(url, path, clib_package_curl_share, (const char **) &authentication_header, 1); + } else { + rc = http_get_file_shared(url, path, clib_package_curl_share, NULL, 0); + } + saved = 1; + } else { +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + } + + if (-1 == rc) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + logger_error("error", "unable to fetch %s", url); + fflush(stderr); + rc = 1; +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + goto cleanup; + } + + if (saved) { +#ifdef HAVE_PTHREADS + pthread_mutex_lock(&mutex); +#endif + _debug("saved %s", path); + fflush(stdout); +#ifdef HAVE_PTHREADS + pthread_mutex_unlock(&mutex); +#endif + } + +cleanup: + free(path); + free(file_copy); + + return rc; +} + +#ifdef HAVE_PTHREADS +static void *fetch_package_file_thread(void *arg) { + repository_file_handle_t data = arg; + int *status = malloc(sizeof(int)); + int rc = fetch_package_file_work(data->url, data->dir, data->file, data->secret); + *status = rc; + pthread_exit((void *) status); + return status; +} +#endif + +static int fetch_package_file(const char *url, const char *dir, const char *file, const char *secret, repository_file_handle_t *thread_data_ptr) { +#ifndef HAVE_PTHREADS + return fetch_package_file_work(pkg, dir, file, secret); +#else + repository_file_handle_t fetch = malloc(sizeof(*fetch)); + int rc = 0; + + if (0 == fetch) { + return -1; + } + + *thread_data_ptr = 0; + + memset(fetch, 0, sizeof(*fetch)); + + fetch->url = strdup(url); + fetch->dir = dir; + fetch->file = file; + fetch->secret = secret; + + rc = pthread_attr_init(&fetch->attr); + + if (0 != rc) { + free(fetch); + return rc; + } + + rc = pthread_create(&fetch->thread, NULL, fetch_package_file_thread, fetch); + + if (0 != rc) { + pthread_attr_destroy(&fetch->attr); + free(fetch); + return rc; + } + + rc = pthread_attr_destroy(&fetch->attr); + + if (0 != rc) { + pthread_cancel(fetch->thread); + free(fetch); + return rc; + } + + *thread_data_ptr = fetch; + + return rc; +#endif +} diff --git a/src/repository/repository.h b/src/repository/repository.h new file mode 100644 index 00000000..063a7227 --- /dev/null +++ b/src/repository/repository.h @@ -0,0 +1,50 @@ +// +// repository.h +// +// Copyright (c) 2021 Elbert van de Put +// MIT licensed +// +#ifndef CLIB_SRC_REPOSITORY_REPOSITORY_H +#define CLIB_SRC_REPOSITORY_REPOSITORY_H + +#include +#include + +typedef struct repository_file_t* repository_file_handle_t; + +/** + * Initialize with secrets to enable authentication. + */ +void repository_init(clib_secrets_t secrets); + +/** + * Start download of package manifest for the package with url. + * The file will be stored at destination_path. + * This function starts a thread to dowload the file, the thread can be joined with `repository_file_finish_download`. + * + * @return a handle on success and NULL on failure. + */ +http_get_response_t* repository_fetch_package_manifest(const char*package_url, const char* slug, const char* version, const char* manifest_file); + +/** + * Start download of a file for the package with url. + * The file will be stored at destination_path. + * This function starts a thread to dowload the file, the thread can be joined with `repository_file_finish_download`. + * + * @return a handle on success and NULL on failure. + */ +repository_file_handle_t repository_download_package_file(const char*package_url, const char* slug, const char* version, const char *file_path, const char* destination_path); + +/** + * Waits until the download is finished. + * @param file + */ +void repository_file_finish_download(repository_file_handle_t file); + +/** + * Free the memory held by the file. + * @param file + */ +void repository_file_free(repository_file_handle_t file); + +#endif//CLIB_SRC_REPOSITORY_REPOSITORY_H diff --git a/test.sh b/test.sh index 3e002f91..633985da 100755 --- a/test.sh +++ b/test.sh @@ -27,13 +27,4 @@ fi cd ../../ -printf "\nRunning clib cache tests\n\n" -cd test/cache - -if ! make test; then - EXIT_CODE=1 -fi - -cd ../../ - exit $EXIT_CODE diff --git a/test/cache/cache-test.c b/test/cache/cache-test.c index bb2a42aa..48332796 100644 --- a/test/cache/cache-test.c +++ b/test/cache/cache-test.c @@ -30,24 +30,23 @@ int main() { rimraf(clib_cache_dir()); - describe("clib-cache opearions") { + describe("clib-cache operations") { char *author = "author"; char *name = "pkg"; char *version = "1.2.0"; char pkg_dir[BUFSIZ]; - int expiraton = 1; + int expiration = 2; it("should initialize succesfully") { - assert_equal(0, clib_cache_init(expiraton)); + assert_equal(0, clib_cache_init(expiration)); } sprintf(pkg_dir, "%s/author_pkg_1.2.0", clib_cache_dir()); it("should manage the package cache") { - assert_equal( - 0, clib_cache_save_package(author, name, version, "../../deps/copy")); + assert_equal(0, clib_cache_save_package(author, name, version, "./deps/copy")); assert_equal(1, clib_cache_has_package(author, name, version)); assert_equal(0, clib_cache_is_expired_package(author, name, version)); @@ -97,7 +96,7 @@ int main() { assert_equal(13, clib_cache_save_search("")); assert_equal(1, clib_cache_has_search()); - sleep(expiraton + 1); + sleep(expiration + 1); assert_equal(0, clib_cache_has_search()); } diff --git a/test/help.sh b/test/help.sh index 9ceb1126..af69dca9 100755 --- a/test/help.sh +++ b/test/help.sh @@ -10,7 +10,7 @@ ACTUAL=$(clib help install) EXPECTED=$(clib install --help) [ "$ACTUAL" = "$EXPECTED" ] || { - echo >&2 "\`help install\` should ouput clib-install --help" + echo >&2 "\`help install\` should output clib-install --help" exit 1 } @@ -18,7 +18,7 @@ ACTUAL=$(clib help search) EXPECTED=$(clib search --help) [ "$ACTUAL" = "$EXPECTED" ] || { - echo >&2 "\`help search\` should ouput clib-search --help" + echo >&2 "\`help search\` should output clib-search --help" exit 1 } diff --git a/test/install-deps-from-package-json.sh b/test/install-deps-from-package-json.sh index e32e13e6..08fafab6 100755 --- a/test/install-deps-from-package-json.sh +++ b/test/install-deps-from-package-json.sh @@ -10,7 +10,8 @@ mkdir -p tmp cd tmp || exit # see https://github.com/clibs/clib/issues/45 -cat > package.json << EOF +# emtter.c does not exist, we test that clib gives an error for it. +cat > clib.json << EOF { "dependencies": { "linenoise": "*", diff --git a/test/install-from-gitlab.sh b/test/install-from-gitlab.sh new file mode 100755 index 00000000..b171850f --- /dev/null +++ b/test/install-from-gitlab.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +throw() { + echo >&2 "$1" + exit 1 +} + +rm -rf tmp +mkdir -p tmp +cd tmp || exit + +cat > clib.json << EOF +{ + "registries": [ + "https://gitlab.com/api/v4/projects/25447829/repository/files/README.md/raw?ref=master" + ], + "dependencies": { + "nouwaarom/clib-dependency": "0.0.1" + } +} +EOF + +clib install -c -o tmp > /dev/null || + throw "expecting exit code of 0"; + +{ [ -d ./tmp/clib-dependency ] && [ -f ./tmp/clib-dependency/package.json ]; } || + throw "failed to install clib-dependency" + +cd - > /dev/null || exit +rm -rf ./tmp diff --git a/test/install-multiple-clibs-libs.sh b/test/install-multiple-clibs-libs.sh index 85a66c43..471f666e 100755 --- a/test/install-multiple-clibs-libs.sh +++ b/test/install-multiple-clibs-libs.sh @@ -5,16 +5,16 @@ throw() { exit 1 } -clib install -c -o tmp ms file hash > /dev/null || +clib install -c -o tmp ms jwerle/fs.c hash > /dev/null || throw "expecting successful exit code" -[ -d ./tmp/ms ] && [ -f ./tmp/ms/package.json ] || +{ [ -d ./tmp/ms ] && [ -f ./tmp/ms/package.json ]; } || throw "failed to install ms" -[ -d ./tmp/file ] && [ -f ./tmp/file/package.json ] || - throw "failed to install file" +{ [ -d ./tmp/fs ] && [ -f ./tmp/fs/clib.json ]; } || + throw "failed to install fs" -[ -d ./tmp/hash ] && [ -f ./tmp/hash/package.json ] || +{ [ -d ./tmp/hash ] && [ -f ./tmp/hash/package.json ]; } || throw "failed to install hash" rm -rf ./tmp diff --git a/test/install-save.sh b/test/install-save.sh index 4534fe2e..97645346 100755 --- a/test/install-save.sh +++ b/test/install-save.sh @@ -3,19 +3,19 @@ mkdir -p tmp/test-save cp test/data/test-save-package.json tmp/test-save/package.json cd tmp/test-save || exit -../../clib-install -c --save stephenmathieson/tabs-to-spaces@1.0.0 >/dev/null -../../clib-install -c -S darthtrevino/str-concat@0.0.2 >/dev/null +../../clib-install -c --save clibs/buffer@0.4.0 >/dev/null +../../clib-install -c -S clibs/strdup@0.1.0 >/dev/null ../../clib-install -c --save-dev jwerle/fs.c@0.1.1 >/dev/null -../../clib-install -c -D clibs/parson@1.0.2 >/dev/null -cd - || exit +../../clib-install -c -D clibs/list@0.2.0 >/dev/null +cd - >/dev/null || exit -if ! grep --quiet "stephenmathieson/tabs-to-spaces" tmp/test-save/package.json; then - echo >&2 "Failed to find stephenmathieson/tabs-to-spaces saved in package.json" +if ! grep --quiet "clibs/buffer" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/buffer.json" exit 1 fi -if ! grep --quiet "darthtrevino/str-concat" tmp/test-save/package.json; then - echo >&2 "Failed to find darthtrevino/strconcat saved in package.json" +if ! grep --quiet "clibs/strdup" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/strdup saved in package.json" exit 1 fi @@ -24,7 +24,7 @@ if ! grep --quiet "jwerle/fs.c" tmp/test-save/package.json; then exit 1 fi -if ! grep --quiet "clibs/parson" tmp/test-save/package.json; then - echo >&2 "Failed to find clibs/parson saved in package.json" +if ! grep --quiet "clibs/list" tmp/test-save/package.json; then + echo >&2 "Failed to find clibs/list saved in package.json" exit 1 fi diff --git a/test/package/Makefile b/test/package/Makefile index da68c925..47f36ccd 100644 --- a/test/package/Makefile +++ b/test/package/Makefile @@ -3,14 +3,14 @@ CC ?= cc VALGRIND ?= valgrind TEST_RUNNER ?= -SRC = ../../src/common/clib-package.c ../../src/common/clib-cache.c ../../src/common/clib-release-info.c ../../src/common/clib-settings.c +SRC = $(wildcard ../../src/*/*.c) DEPS += $(wildcard ../../deps/*/*.c) OBJS = $(SRC:.c=.o) $(DEPS:.c=.o) TEST_SRC = $(wildcard *.c) TEST_OBJ = $(TEST_SRC:.c=.o) TEST_BIN = $(TEST_SRC:.c=) -CFLAGS += -std=c99 -Wall -I../../src/common -I../../deps -DHAVE_PTHREADS -pthread -g +CFLAGS += -std=c99 -Wall -I../../src/common -I../../src/registry -I../../src/repository -I../../deps -DHAVE_PTHREADS -pthread -g LDFLAGS = -lcurl VALGRIND_OPTS ?= --leak-check=full --error-exitcode=3 diff --git a/test/package/clib_secrets.json b/test/package/clib_secrets.json new file mode 100644 index 00000000..4b93818d --- /dev/null +++ b/test/package/clib_secrets.json @@ -0,0 +1,4 @@ +{ + "gitlab.com": "GitlabSecret", + "github.com": "GithubSecret" +} \ No newline at end of file diff --git a/test/package/package-install-dependencies.c b/test/package/package-install-dependencies.c index 0904917d..911d154c 100644 --- a/test/package/package-install-dependencies.c +++ b/test/package/package-install-dependencies.c @@ -3,6 +3,8 @@ #include "describe/describe.h" #include "fs/fs.h" #include "rimraf/rimraf.h" +#include "registry-manager.h" +#include "repository.h" int main() { curl_global_init(CURL_GLOBAL_ALL); @@ -12,6 +14,10 @@ int main() { .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + describe("clib_package_install_dependencies") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install_dependencies(NULL, "./deps", 0)); @@ -19,7 +25,7 @@ int main() { it("should install the dep in its own directory") { clib_package_t *dep = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(dep); assert(0 == clib_package_install_dependencies(dep, "./test/fixtures/", 0)); @@ -31,7 +37,7 @@ int main() { it("should install the dependency's package.json") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); @@ -42,7 +48,7 @@ int main() { it("should install the dependency's sources") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); @@ -54,7 +60,7 @@ int main() { it("should install the dependency's dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/rimraf.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/rimraf.c", "https://github.com/stephanmathieson/rimraf.c", 0); assert(pkg); assert(0 == clib_package_install_dependencies(pkg, "./test/fixtures/", 0)); diff --git a/test/package/package-install-dev-dependencies.c b/test/package/package-install-dev-dependencies.c index d23bab78..184070b8 100644 --- a/test/package/package-install-dev-dependencies.c +++ b/test/package/package-install-dev-dependencies.c @@ -3,6 +3,8 @@ #include "describe/describe.h" #include "fs/fs.h" #include "rimraf/rimraf.h" +#include "registry-manager.h" +#include "repository.h" int main() { curl_global_init(CURL_GLOBAL_ALL); @@ -12,6 +14,10 @@ int main() { .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + describe("clib_package_install_development") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install_development(NULL, "./deps", 0)); @@ -19,7 +25,7 @@ int main() { it("should return -1 when given a bad dir") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert(-1 == clib_package_install_development(pkg, NULL, 0)); clib_package_free(pkg); @@ -27,7 +33,7 @@ int main() { it("should install the package's development dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/trim.c@0.0.2", 0); + clib_package_new_from_slug_and_url("stephenmathieson/trim.c@0.0.2", "https://github.com/stephanmathieson/trim.c", 0); assert(pkg); assert(0 == clib_package_install_development(pkg, "./test/fixtures", 0)); assert(0 == fs_exists("./test/fixtures/describe")); diff --git a/test/package/package-install.c b/test/package/package-install.c index 1ef464f9..fc931111 100644 --- a/test/package/package-install.c +++ b/test/package/package-install.c @@ -4,15 +4,24 @@ #include "fs/fs.h" #include "rimraf/rimraf.h" #include +#include "registry-manager.h" +#include "repository.h" +#include "clib-package-installer.h" +#include "strdup/strdup.h" int main() { curl_global_init(CURL_GLOBAL_ALL); - clib_package_set_opts((clib_package_opts_t){ + clib_package_set_opts((clib_package_opts_t) { .skip_cache = 1, .prefix = 0, .force = 1, }); + registries_t registries = registry_manager_init_registries(NULL, NULL); + registry_manager_fetch_registries(registries); + clib_package_installer_init(registries, NULL); + repository_init(NULL); + describe("clib_package_install") { it("should return -1 when given a bad package") { assert(-1 == clib_package_install(NULL, "./deps", 0)); @@ -20,7 +29,7 @@ int main() { it("should install the pkg in its own directory") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/")); @@ -31,7 +40,7 @@ int main() { it("should install the package's clib.json or package.json") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 1)); assert(0 == fs_exists("./test/fixtures/case/package.json") || @@ -42,7 +51,7 @@ int main() { it("should install the package's sources") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephenmathison/case.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/case/case.c")); @@ -53,7 +62,7 @@ int main() { it("should install the package's dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c@master", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c@master", "https://github.com/stephenmathieson/mkdirp.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/path-normalize/")); @@ -67,7 +76,7 @@ int main() { it("should not install the package's development dependencies") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/trim.c@0.0.2", 0); + clib_package_new_from_slug_and_url("stephenmathieson/trim.c@0.0.2", "https://github.com/stephenmathieson/trim.c", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(-1 == fs_exists("./test/fixtures/describe/")); @@ -79,18 +88,19 @@ int main() { rimraf("./test/fixtures"); } + /* This test is currently not feasible because clib has too many dependencies with issues. I propose we re-enable this test at the next release. it("should install itself") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/clib-package@0.4.2", 0); + clib_package_new_from_slug_and_url("clibs/clib@2.7.0", "https://github.com/clibs/clib", 0); assert(pkg); assert(0 == clib_package_install(pkg, "./test/fixtures/", 0)); assert(0 == fs_exists("./test/fixtures/")); - assert(0 == fs_exists("./test/fixtures/clib-package/")); - assert(0 == fs_exists("./test/fixtures/clib-package/clib-package.c")); - assert(0 == fs_exists("./test/fixtures/clib-package/clib-package.h")); - assert(0 == fs_exists("./test/fixtures/clib-package/package.json") || - 0 == fs_exists("./test/fixtures/clib-package/clib.json")); + assert(0 == fs_exists("./test/fixtures/clib/")); + assert(0 == fs_exists("./test/fixtures/clib/clib-package.c")); + assert(0 == fs_exists("./test/fixtures/clib/clib-package.h")); + assert(0 == fs_exists("./test/fixtures/clib/package.json") || + 0 == fs_exists("./test/fixtures/clib/clib.json")); assert(0 == fs_exists("./test/fixtures/http-get/")); assert(0 == fs_exists("./test/fixtures/http-get/http-get.c")); @@ -144,7 +154,7 @@ int main() { clib_package_free(pkg); rimraf("./test/fixtures"); - } + } */ } curl_global_cleanup(); diff --git a/test/package/package-new-from-slug.c b/test/package/package-new-from-slug.c index 60830f45..adda0bcb 100644 --- a/test/package/package-new-from-slug.c +++ b/test/package/package-new-from-slug.c @@ -7,20 +7,20 @@ int main() { describe("clib_package_new_from_slug") { it("should return NULL when given a bad slug") { - assert(NULL == clib_package_new_from_slug(NULL, 0)); + assert(NULL == clib_package_new_from_slug_and_url(NULL, NULL, 0)); } it("should return NULL when given a slug missing a name") { - assert(NULL == clib_package_new_from_slug("author/@version", 0)); + assert(NULL == clib_package_new_from_slug_and_url("author/@version", "https://github.com/author/", 0)); } it("should return NULL when given slug which doesn't resolve") { - assert(NULL == clib_package_new_from_slug("abc11234", 0)); + assert(NULL == clib_package_new_from_slug_and_url("abc11234", "https://github.com/abc11234", 0)); } it("should build the correct package") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/case.c@0.1.0", 0); + clib_package_new_from_slug_and_url("stephenmathieson/case.c@0.1.0", "https://github.com/stephanmathieson/case.c", 0); assert(pkg); assert_str_equal("case", pkg->name); assert_str_equal("0.1.0", pkg->version); @@ -32,7 +32,7 @@ int main() { it("should force package version numbers") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c@0.0.1", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c@0.0.1", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert_str_equal("0.0.1", pkg->version); clib_package_free(pkg); @@ -40,15 +40,14 @@ int main() { it("should use package version if version not provided") { clib_package_t *pkg = - clib_package_new_from_slug("stephenmathieson/mkdirp.c", 0); + clib_package_new_from_slug_and_url("stephenmathieson/mkdirp.c", "https://github.com/stephanmathieson/mkdirp.c", 0); assert(pkg); assert_str_equal("0.1.5", pkg->version); clib_package_free(pkg); } it("should save the package's json") { - clib_package_t *pkg = clib_package_new_from_slug( - "stephenmathieson/str-replace.c@8ca90fb", 0); + clib_package_t *pkg = clib_package_new_from_slug_and_url("stephenmathieson/str-replace.c@8ca90fb", "https://github.com/stephanmathieson/str-replace.c", 0); assert(pkg); assert(pkg->json); diff --git a/test/package/package-secrets-load.c b/test/package/package-secrets-load.c new file mode 100644 index 00000000..8df8afcb --- /dev/null +++ b/test/package/package-secrets-load.c @@ -0,0 +1,22 @@ +#include "describe/describe.h" +#include +#include "clib-secrets.h" + +int main() { + describe("clib_secrets") { + it("should load secrets from a valid json.") { + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + assert(secrets != NULL); + } + + it("should provide secrets for a domain.") { + clib_secrets_t secrets = clib_secrets_load_from_file("clib_secrets.json"); + char *github_secret = clib_secret_find_for_hostname(secrets, "github.com"); + assert(strcmp(github_secret, "GithubSecret") == 0); + char *gitlab_secret = clib_secret_find_for_hostname(secrets, "gitlab.com"); + assert(strcmp(gitlab_secret, "GitlabSecret") == 0); + } + + return assert_failures(); + } +} diff --git a/test/package/package-url.c b/test/package/package-url.c deleted file mode 100644 index bed9b60c..00000000 --- a/test/package/package-url.c +++ /dev/null @@ -1,28 +0,0 @@ - -#include "clib-package.h" -#include "describe/describe.h" - -int main() { - describe("clib_package_url") { - it("should return NULL when given a bad author") { - assert(NULL == clib_package_url(NULL, "name", "version")); - } - - it("should return NULL when given a bad name") { - assert(NULL == clib_package_url("author", NULL, "version")); - } - - it("should return NULL when given a bad version") { - assert(NULL == clib_package_url("author", "name", NULL)); - } - - it("should build a GitHub url") { - char *url = clib_package_url("author", "name", "version"); - assert_str_equal("https://raw.githubusercontent.com/author/name/version", - url); - free(url); - } - } - - return assert_failures(); -}