From 5e0e9a62f5a87c0718deaa9fcd02e5a939ebaab0 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 14 Jul 2022 14:26:08 -0500 Subject: [PATCH 1/7] start some jlap experiments --- CMakeLists.txt | 8 +++++++- include/powerloader/fileio.hpp | 6 ++++++ src/cli/main.cpp | 1 - 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 626270fe..f4169fae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,7 +345,13 @@ if (BUILD_EXE) target_compile_definitions(powerloader PUBLIC WITH_ZCHUNK) endif() - list(APPEND libpowerloader_targets powerloader) + + add_executable(jlap src/cli/jlap.cpp) + set_property(TARGET jlap PROPERTY CXX_STANDARD 17) + target_link_libraries(jlap libpowerloader CLI11::CLI11 yaml-cpp curl) + + list(APPEND libpowerloader_targets powerloader jlap) + endif() # Tests diff --git a/include/powerloader/fileio.hpp b/include/powerloader/fileio.hpp index 12fa3e63..d1841e1b 100644 --- a/include/powerloader/fileio.hpp +++ b/include/powerloader/fileio.hpp @@ -163,6 +163,12 @@ namespace powerloader return ::fwrite(buffer, element_size, element_count, m_fs); } + template + std::size_t write(const std::basic_string& str) const noexcept + { + return ::fwrite(str.c_str(), sizeof(C), str.size(), m_fs); + } + std::streamoff put(int c) const noexcept { return ::fputc(c, m_fs); diff --git a/src/cli/main.cpp b/src/cli/main.cpp index e1504030..81a50656 100644 --- a/src/cli/main.cpp +++ b/src/cli/main.cpp @@ -431,7 +431,6 @@ parse_mirrors(const Context& ctx, const YAML::Node& node) return res; } - int main(int argc, char** argv) { From b710b17a0b7fb3db2009caad97105fe6a76a2315 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 14 Jul 2022 14:26:28 -0500 Subject: [PATCH 2/7] .. --- src/cli/jlap.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 src/cli/jlap.cpp diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp new file mode 100644 index 00000000..3a98d7e0 --- /dev/null +++ b/src/cli/jlap.cpp @@ -0,0 +1,59 @@ +#include +#include +#include + +using namespace powerloader; +namespace fs = std::filesystem; + +struct Patch +{ + std::string hash; + std::vector patches; +}; + +void apply_jlap() +{ + std::ifstream jlap_file("repodata.jlap"); + std::ifstream json_file("repodata.json"); + std::string line; + + std::vector patches; + std::unique_ptr cur_patch; + while (std::getline(jlap_file, line)) + { + if (line.size() && line[0] != '{') + { + if (cur_patch) patches.push_back(*cur_patch); + + cur_patch = std::make_unique(); + cur_patch->hash = line; + } + else + { + auto j = nlohmann::json::parse(line); + std::cout << j.dump(4) << std::endl; + cur_patch->patches.emplace_back(std::move(j)); + } + } +} + + +int main() +{ + powerloader::Context ctx; + CURLHandle h(ctx); + + std::error_code ec; + FileIO outfile("repodata.jlap", FileIO::append_update_binary, ec); + outfile.seek(0, SEEK_END); + std::cout << outfile.tell() << std::endl; + + h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); + h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(outfile.tell())); + auto response = h.perform(); + for (auto& [k, v] : response.headers) + std::cout << k << " .. " << v << std::endl; + + outfile.write(response.content.value()); + apply_jlap(); +} \ No newline at end of file From 9b89ed6b0509dda792edefa0c654ada1f072668d Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 14 Jul 2022 19:04:11 -0500 Subject: [PATCH 3/7] proper blake2 implementation and working initial implementation --- BLAKE2/ref/blake2-impl.h | 160 ++++++++++++++ BLAKE2/ref/blake2.h | 195 +++++++++++++++++ BLAKE2/ref/blake2b-ref.c | 379 +++++++++++++++++++++++++++++++++ CMakeLists.txt | 1 + include/powerloader/fileio.hpp | 1 + include/powerloader/utils.hpp | 1 + src/cli/jlap.cpp | 152 +++++++++---- src/utils.cpp | 26 +++ 8 files changed, 872 insertions(+), 43 deletions(-) create mode 100644 BLAKE2/ref/blake2-impl.h create mode 100644 BLAKE2/ref/blake2.h create mode 100644 BLAKE2/ref/blake2b-ref.c diff --git a/BLAKE2/ref/blake2-impl.h b/BLAKE2/ref/blake2-impl.h new file mode 100644 index 00000000..c1df82e0 --- /dev/null +++ b/BLAKE2/ref/blake2-impl.h @@ -0,0 +1,160 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return ( uint16_t )((( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8)); +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif diff --git a/BLAKE2/ref/blake2.h b/BLAKE2/ref/blake2.h new file mode 100644 index 00000000..ca390305 --- /dev/null +++ b/BLAKE2/ref/blake2.h @@ -0,0 +1,195 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum { + BLAKE2_DUMMY_1 = 1/(int)(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(int)(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/BLAKE2/ref/blake2b-ref.c b/BLAKE2/ref/blake2b-ref.c new file mode 100644 index 00000000..cd38b1ba --- /dev/null +++ b/BLAKE2/ref/blake2b-ref.c @@ -0,0 +1,379 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif diff --git a/CMakeLists.txt b/CMakeLists.txt index f4169fae..8a322177 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,6 +346,7 @@ if (BUILD_EXE) endif() + include_directories(BLAKE2/ref) add_executable(jlap src/cli/jlap.cpp) set_property(TARGET jlap PROPERTY CXX_STANDARD 17) target_link_libraries(jlap libpowerloader CLI11::CLI11 yaml-cpp curl) diff --git a/include/powerloader/fileio.hpp b/include/powerloader/fileio.hpp index d1841e1b..128ff359 100644 --- a/include/powerloader/fileio.hpp +++ b/include/powerloader/fileio.hpp @@ -163,6 +163,7 @@ namespace powerloader return ::fwrite(buffer, element_size, element_count, m_fs); } + template std::size_t write(const std::basic_string& str) const noexcept { diff --git a/include/powerloader/utils.hpp b/include/powerloader/utils.hpp index 1bb0f1c3..75b8529a 100644 --- a/include/powerloader/utils.hpp +++ b/include/powerloader/utils.hpp @@ -91,6 +91,7 @@ namespace powerloader POWERLOADER_API bool contains(const std::string_view& str, const std::string_view& sub_str); POWERLOADER_API std::string sha256sum(const fs::path& path); + // POWERLOADER_API std::string blake2sum(const fs::path& path); POWERLOADER_API std::string md5sum(const fs::path& path); POWERLOADER_API std::pair parse_header( diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp index 3a98d7e0..c5b491ea 100644 --- a/src/cli/jlap.cpp +++ b/src/cli/jlap.cpp @@ -1,59 +1,125 @@ #include #include #include +#include + +extern "C" +{ +#include "blake2.h" +} using namespace powerloader; namespace fs = std::filesystem; struct Patch { - std::string hash; - std::vector patches; + std::string hash; + std::vector patches; }; -void apply_jlap() +std::string +blake2sum(const fs::path& path) +{ + size_t sum, n; + blake2b_state S; + constexpr std::size_t BUFSIZE = 32768; + std::vector buffer(BUFSIZE); + + constexpr std::size_t outbytes = 32; + blake2b_init(&S, outbytes); + + unsigned char hash[32]; + + std::ifstream infile(path, std::ios::binary); + + while (infile) + { + infile.read(buffer.data(), BUFSIZE); + size_t count = infile.gcount(); + if (!count) + break; + blake2b_update(&S, buffer.data(), count); + } + + blake2b_final(&S, hash, outbytes); + return hex_string(hash, outbytes); +} + +void +apply_jlap() { - std::ifstream jlap_file("repodata.jlap"); - std::ifstream json_file("repodata.json"); - std::string line; - - std::vector patches; - std::unique_ptr cur_patch; - while (std::getline(jlap_file, line)) - { - if (line.size() && line[0] != '{') - { - if (cur_patch) patches.push_back(*cur_patch); - - cur_patch = std::make_unique(); - cur_patch->hash = line; - } - else - { - auto j = nlohmann::json::parse(line); - std::cout << j.dump(4) << std::endl; - cur_patch->patches.emplace_back(std::move(j)); - } - } + std::ifstream jlap_file("repodata.jlap"); + std::ifstream json_file("repodata.json"); + std::string line; + + std::cout << blake2sum("repodata.json") << std::endl; + std::cout << blake2sum("repodata.jlap") << std::endl; + std::vector patches; + std::unique_ptr cur_patch; + while (std::getline(jlap_file, line)) + { + if (line.size() && line[0] != '{') + { + if (cur_patch) + patches.push_back(*cur_patch); + + cur_patch = std::make_unique(); + cur_patch->hash = line; + } + else + { + auto j = nlohmann::json::parse(line); + // std::cout << j.dump(4) << std::endl; + cur_patch->patches.emplace_back(std::move(j)); + } + } + + nlohmann::json jrdata; + json_file >> jrdata; + + std::string repo_bsum = blake2sum("repodata.json"); + + std::size_t i = 0; + for (auto& p : patches) + { + std::cout << "Found patches # " << p.patches.size() << std::endl; + for (auto& pf : p.patches) + { + if (pf.contains("from")) + { + if (pf["from"] == repo_bsum) + { + std::cout << "Applying patch " << i++ << " from " << repo_bsum << " to " + << pf["to"] << std::endl; + repo_bsum = pf["to"]; + jrdata = jrdata.patch(pf["patch"]); + } + } + } + } + std::ofstream rpatched("final_repodata.json"); + rpatched << jrdata.dump(2) << "\n"; } -int main() +int +main() { - powerloader::Context ctx; - CURLHandle h(ctx); - - std::error_code ec; - FileIO outfile("repodata.jlap", FileIO::append_update_binary, ec); - outfile.seek(0, SEEK_END); - std::cout << outfile.tell() << std::endl; - - h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); - h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(outfile.tell())); - auto response = h.perform(); - for (auto& [k, v] : response.headers) - std::cout << k << " .. " << v << std::endl; - - outfile.write(response.content.value()); - apply_jlap(); -} \ No newline at end of file + powerloader::Context ctx; + CURLHandle h(ctx); + + std::error_code ec; + FileIO outfile("repodata.jlap", FileIO::append_update_binary, ec); + outfile.seek(0, SEEK_END); + std::cout << outfile.tell() << std::endl; + + h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); + h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(outfile.tell())); + auto response = h.perform(); + for (auto& [k, v] : response.headers) + std::cout << k << " .. " << v << std::endl; + + outfile.write(response.content.value()); + apply_jlap(); + return 0; +} diff --git a/src/utils.cpp b/src/utils.cpp index 0661b81b..a8df8aae 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -82,6 +82,32 @@ namespace powerloader return hex_string(hash, 32); } + // std::string blake2sum(const fs::path& path) + // { + // unsigned char hash[32]; + // EVP_MD_CTX* mdctx; + // mdctx = EVP_MD_CTX_create(); + // EVP_DigestInit_ex(mdctx, EVP_blake2b512(), NULL); + + // std::ifstream infile(path, std::ios::binary); + // constexpr std::size_t BUFSIZE = 32768; + // std::vector buffer(BUFSIZE); + + // while (infile) + // { + // infile.read(buffer.data(), BUFSIZE); + // size_t count = infile.gcount(); + // if (!count) + // break; + // EVP_DigestUpdate(mdctx, buffer.data(), count); + // } + + // EVP_DigestFinal_ex(mdctx, hash, nullptr); + // EVP_MD_CTX_destroy(mdctx); + + // return hex_string(hash, 32); + // } + std::string md5sum(const fs::path& path) { unsigned char hash[16]; From 08ab15221c8ef863eaa7b407d81fee99638e47de Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 15 Jul 2022 23:34:09 -0500 Subject: [PATCH 4/7] working and fast version --- src/cli/jlap.cpp | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp index c5b491ea..876b7954 100644 --- a/src/cli/jlap.cpp +++ b/src/cli/jlap.cpp @@ -2,6 +2,7 @@ #include #include #include +#include extern "C" { @@ -56,6 +57,7 @@ apply_jlap() std::cout << blake2sum("repodata.jlap") << std::endl; std::vector patches; std::unique_ptr cur_patch; + while (std::getline(jlap_file, line)) { if (line.size() && line[0] != '{') @@ -68,9 +70,16 @@ apply_jlap() } else { - auto j = nlohmann::json::parse(line); - // std::cout << j.dump(4) << std::endl; - cur_patch->patches.emplace_back(std::move(j)); + try + { + auto j = nlohmann::json::parse(line); + // std::cout << j.dump(4) << std::endl; + cur_patch->patches.emplace_back(std::move(j)); + } + catch(...) + { + std::cout << "Could not parse patch " << line << std::endl; + } } } @@ -80,6 +89,7 @@ apply_jlap() std::string repo_bsum = blake2sum("repodata.json"); std::size_t i = 0; + for (auto& p : patches) { std::cout << "Found patches # " << p.patches.size() << std::endl; @@ -89,16 +99,31 @@ apply_jlap() { if (pf["from"] == repo_bsum) { - std::cout << "Applying patch " << i++ << " from " << repo_bsum << " to " - << pf["to"] << std::endl; + auto t0 = std::chrono::high_resolution_clock::now(); + std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) << "…2 to " + << pf["to"].get().substr(0, 8) << "… "; repo_bsum = pf["to"]; - jrdata = jrdata.patch(pf["patch"]); + // nlohmann::inplace_patch(jrdata, pf["patch"]); + jrdata.patch(pf["patch"]); + auto t1 = std::chrono::high_resolution_clock::now(); + std::cout << "took " << std::chrono::duration_cast(t1-t0).count() <<" ns." << std::endl; } } } } - std::ofstream rpatched("final_repodata.json"); - rpatched << jrdata.dump(2) << "\n"; + { + std::ofstream rpatched("final_repodata.json"); + rpatched << jrdata.dump(2) << "\n"; + } + + if (blake2sum("final_repodata.json") == patches.back().patches.back()["latest"]) + { + std::cout << "All successful." << std::endl; + } + else + { + std::cout << "Error: " << blake2sum("final_repodata.json") << " not matching " << patches.back().patches.back()["latest"] << std::endl; + } } @@ -119,7 +144,8 @@ main() for (auto& [k, v] : response.headers) std::cout << k << " .. " << v << std::endl; - outfile.write(response.content.value()); + if (response.ok()) + outfile.write(response.content.value()); apply_jlap(); return 0; } From f37f2b5ae74c02e66a1cedde90bf288b16b21cef Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 19 Jul 2022 15:20:22 +0200 Subject: [PATCH 5/7] implement jlap --- BLAKE2/ref/blake2-impl.h | 166 +++++----- BLAKE2/ref/blake2.h | 327 +++++++++--------- BLAKE2/ref/blake2b-ref.c | 582 ++++++++++++++++++--------------- include/powerloader/fileio.hpp | 4 +- src/cli/jlap.cpp | 88 ++++- 5 files changed, 638 insertions(+), 529 deletions(-) diff --git a/BLAKE2/ref/blake2-impl.h b/BLAKE2/ref/blake2-impl.h index c1df82e0..23948ee1 100644 --- a/BLAKE2/ref/blake2-impl.h +++ b/BLAKE2/ref/blake2-impl.h @@ -19,142 +19,142 @@ #include #if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) - #if defined(_MSC_VER) - #define BLAKE2_INLINE __inline - #elif defined(__GNUC__) - #define BLAKE2_INLINE __inline__ - #else - #define BLAKE2_INLINE - #endif +#if defined(_MSC_VER) +#define BLAKE2_INLINE __inline +#elif defined(__GNUC__) +#define BLAKE2_INLINE __inline__ #else - #define BLAKE2_INLINE inline +#define BLAKE2_INLINE +#endif +#else +#define BLAKE2_INLINE inline #endif -static BLAKE2_INLINE uint32_t load32( const void *src ) +static BLAKE2_INLINE uint32_t +load32(const void* src) { #if defined(NATIVE_LITTLE_ENDIAN) - uint32_t w; - memcpy(&w, src, sizeof w); - return w; + uint32_t w; + memcpy(&w, src, sizeof w); + return w; #else - const uint8_t *p = ( const uint8_t * )src; - return (( uint32_t )( p[0] ) << 0) | - (( uint32_t )( p[1] ) << 8) | - (( uint32_t )( p[2] ) << 16) | - (( uint32_t )( p[3] ) << 24) ; + const uint8_t* p = (const uint8_t*) src; + return ((uint32_t) (p[0]) << 0) | ((uint32_t) (p[1]) << 8) | ((uint32_t) (p[2]) << 16) + | ((uint32_t) (p[3]) << 24); #endif } -static BLAKE2_INLINE uint64_t load64( const void *src ) +static BLAKE2_INLINE uint64_t +load64(const void* src) { #if defined(NATIVE_LITTLE_ENDIAN) - uint64_t w; - memcpy(&w, src, sizeof w); - return w; + uint64_t w; + memcpy(&w, src, sizeof w); + return w; #else - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) | - (( uint64_t )( p[6] ) << 48) | - (( uint64_t )( p[7] ) << 56) ; + const uint8_t* p = (const uint8_t*) src; + return ((uint64_t) (p[0]) << 0) | ((uint64_t) (p[1]) << 8) | ((uint64_t) (p[2]) << 16) + | ((uint64_t) (p[3]) << 24) | ((uint64_t) (p[4]) << 32) | ((uint64_t) (p[5]) << 40) + | ((uint64_t) (p[6]) << 48) | ((uint64_t) (p[7]) << 56); #endif } -static BLAKE2_INLINE uint16_t load16( const void *src ) +static BLAKE2_INLINE uint16_t +load16(const void* src) { #if defined(NATIVE_LITTLE_ENDIAN) - uint16_t w; - memcpy(&w, src, sizeof w); - return w; + uint16_t w; + memcpy(&w, src, sizeof w); + return w; #else - const uint8_t *p = ( const uint8_t * )src; - return ( uint16_t )((( uint32_t )( p[0] ) << 0) | - (( uint32_t )( p[1] ) << 8)); + const uint8_t* p = (const uint8_t*) src; + return (uint16_t) (((uint32_t) (p[0]) << 0) | ((uint32_t) (p[1]) << 8)); #endif } -static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +static BLAKE2_INLINE void +store16(void* dst, uint16_t w) { #if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); + memcpy(dst, &w, sizeof w); #else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; + uint8_t* p = (uint8_t*) dst; + *p++ = (uint8_t) w; + w >>= 8; + *p++ = (uint8_t) w; #endif } -static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +static BLAKE2_INLINE void +store32(void* dst, uint32_t w) { #if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); + memcpy(dst, &w, sizeof w); #else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); + uint8_t* p = (uint8_t*) dst; + p[0] = (uint8_t) (w >> 0); + p[1] = (uint8_t) (w >> 8); + p[2] = (uint8_t) (w >> 16); + p[3] = (uint8_t) (w >> 24); #endif } -static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +static BLAKE2_INLINE void +store64(void* dst, uint64_t w) { #if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); + memcpy(dst, &w, sizeof w); #else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); - p[6] = (uint8_t)(w >> 48); - p[7] = (uint8_t)(w >> 56); + uint8_t* p = (uint8_t*) dst; + p[0] = (uint8_t) (w >> 0); + p[1] = (uint8_t) (w >> 8); + p[2] = (uint8_t) (w >> 16); + p[3] = (uint8_t) (w >> 24); + p[4] = (uint8_t) (w >> 32); + p[5] = (uint8_t) (w >> 40); + p[6] = (uint8_t) (w >> 48); + p[7] = (uint8_t) (w >> 56); #endif } -static BLAKE2_INLINE uint64_t load48( const void *src ) +static BLAKE2_INLINE uint64_t +load48(const void* src) { - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) ; + const uint8_t* p = (const uint8_t*) src; + return ((uint64_t) (p[0]) << 0) | ((uint64_t) (p[1]) << 8) | ((uint64_t) (p[2]) << 16) + | ((uint64_t) (p[3]) << 24) | ((uint64_t) (p[4]) << 32) | ((uint64_t) (p[5]) << 40); } -static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +static BLAKE2_INLINE void +store48(void* dst, uint64_t w) { - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); + uint8_t* p = (uint8_t*) dst; + p[0] = (uint8_t) (w >> 0); + p[1] = (uint8_t) (w >> 8); + p[2] = (uint8_t) (w >> 16); + p[3] = (uint8_t) (w >> 24); + p[4] = (uint8_t) (w >> 32); + p[5] = (uint8_t) (w >> 40); } -static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +static BLAKE2_INLINE uint32_t +rotr32(const uint32_t w, const unsigned c) { - return ( w >> c ) | ( w << ( 32 - c ) ); + return (w >> c) | (w << (32 - c)); } -static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +static BLAKE2_INLINE uint64_t +rotr64(const uint64_t w, const unsigned c) { - return ( w >> c ) | ( w << ( 64 - c ) ); + return (w >> c) | (w << (64 - c)); } /* prevents compiler optimizing out memset() */ -static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +static BLAKE2_INLINE void +secure_zero_memory(void* v, size_t n) { - static void *(*const volatile memset_v)(void *, int, size_t) = &memset; - memset_v(v, 0, n); + static void* (*const volatile memset_v)(void*, int, size_t) = &memset; + memset_v(v, 0, n); } #endif diff --git a/BLAKE2/ref/blake2.h b/BLAKE2/ref/blake2.h index ca390305..fce2216a 100644 --- a/BLAKE2/ref/blake2.h +++ b/BLAKE2/ref/blake2.h @@ -25,168 +25,175 @@ #endif #if defined(__cplusplus) -extern "C" { +extern "C" +{ #endif - enum blake2s_constant - { - BLAKE2S_BLOCKBYTES = 64, - BLAKE2S_OUTBYTES = 32, - BLAKE2S_KEYBYTES = 32, - BLAKE2S_SALTBYTES = 8, - BLAKE2S_PERSONALBYTES = 8 - }; - - enum blake2b_constant - { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 - }; - - typedef struct blake2s_state__ - { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2s_state; - - typedef struct blake2b_state__ - { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2b_state; - - typedef struct blake2sp_state__ - { - blake2s_state S[8][1]; - blake2s_state R[1]; - uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2sp_state; - - typedef struct blake2bp_state__ - { - blake2b_state S[4][1]; - blake2b_state R[1]; - uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2bp_state; - - - BLAKE2_PACKED(struct blake2s_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint16_t xof_length; /* 14 */ - uint8_t node_depth; /* 15 */ - uint8_t inner_length; /* 16 */ - /* uint8_t reserved[0]; */ - uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ - uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ - }); - - typedef struct blake2s_param__ blake2s_param; - - BLAKE2_PACKED(struct blake2b_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint32_t xof_length; /* 16 */ - uint8_t node_depth; /* 17 */ - uint8_t inner_length; /* 18 */ - uint8_t reserved[14]; /* 32 */ - uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ - uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ - }); - - typedef struct blake2b_param__ blake2b_param; - - typedef struct blake2xs_state__ - { - blake2s_state S[1]; - blake2s_param P[1]; - } blake2xs_state; - - typedef struct blake2xb_state__ - { - blake2b_state S[1]; - blake2b_param P[1]; - } blake2xb_state; - - /* Padded structs result in a compile-time error */ - enum { - BLAKE2_DUMMY_1 = 1/(int)(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), - BLAKE2_DUMMY_2 = 1/(int)(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) - }; - - /* Streaming API */ - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); - int blake2s_final( blake2s_state *S, void *out, size_t outlen ); - - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); - int blake2b_final( blake2b_state *S, void *out, size_t outlen ); - - int blake2sp_init( blake2sp_state *S, size_t outlen ); - int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); - int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); - - int blake2bp_init( blake2bp_state *S, size_t outlen ); - int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); - int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); - - /* Variable output length API */ - int blake2xs_init( blake2xs_state *S, const size_t outlen ); - int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); - int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); - - int blake2xb_init( blake2xb_state *S, const size_t outlen ); - int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); - int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); - - /* Simple API */ - int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - /* This is simply an alias for blake2b */ - int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum + { + BLAKE2_DUMMY_1 = 1 / (int) (sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1 / (int) (sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init(blake2s_state* S, size_t outlen); + int blake2s_init_key(blake2s_state* S, size_t outlen, const void* key, size_t keylen); + int blake2s_init_param(blake2s_state* S, const blake2s_param* P); + int blake2s_update(blake2s_state* S, const void* in, size_t inlen); + int blake2s_final(blake2s_state* S, void* out, size_t outlen); + + int blake2b_init(blake2b_state* S, size_t outlen); + int blake2b_init_key(blake2b_state* S, size_t outlen, const void* key, size_t keylen); + int blake2b_init_param(blake2b_state* S, const blake2b_param* P); + int blake2b_update(blake2b_state* S, const void* in, size_t inlen); + int blake2b_final(blake2b_state* S, void* out, size_t outlen); + + int blake2sp_init(blake2sp_state* S, size_t outlen); + int blake2sp_init_key(blake2sp_state* S, size_t outlen, const void* key, size_t keylen); + int blake2sp_update(blake2sp_state* S, const void* in, size_t inlen); + int blake2sp_final(blake2sp_state* S, void* out, size_t outlen); + + int blake2bp_init(blake2bp_state* S, size_t outlen); + int blake2bp_init_key(blake2bp_state* S, size_t outlen, const void* key, size_t keylen); + int blake2bp_update(blake2bp_state* S, const void* in, size_t inlen); + int blake2bp_final(blake2bp_state* S, void* out, size_t outlen); + + /* Variable output length API */ + int blake2xs_init(blake2xs_state* S, const size_t outlen); + int blake2xs_init_key(blake2xs_state* S, const size_t outlen, const void* key, size_t keylen); + int blake2xs_update(blake2xs_state* S, const void* in, size_t inlen); + int blake2xs_final(blake2xs_state* S, void* out, size_t outlen); + + int blake2xb_init(blake2xb_state* S, const size_t outlen); + int blake2xb_init_key(blake2xb_state* S, const size_t outlen, const void* key, size_t keylen); + int blake2xb_update(blake2xb_state* S, const void* in, size_t inlen); + int blake2xb_final(blake2xb_state* S, void* out, size_t outlen); + + /* Simple API */ + int blake2s( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + int blake2b( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + + int blake2sp( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + int blake2bp( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + + int blake2xs( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + int blake2xb( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); + + /* This is simply an alias for blake2b */ + int blake2( + void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen); #if defined(__cplusplus) } diff --git a/BLAKE2/ref/blake2b-ref.c b/BLAKE2/ref/blake2b-ref.c index cd38b1ba..dc843411 100644 --- a/BLAKE2/ref/blake2b-ref.c +++ b/BLAKE2/ref/blake2b-ref.c @@ -20,360 +20,398 @@ #include "blake2.h" #include "blake2-impl.h" -static const uint64_t blake2b_IV[8] = +static const uint64_t blake2b_IV[8] = { 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL }; + +static const uint8_t blake2b_sigma[12][16] + = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } }; + + +static void +blake2b_set_lastnode(blake2b_state* S) { - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -static void blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = (uint64_t)-1; + S->f[1] = (uint64_t) -1; } /* Some helper functions, not necessarily useful */ -static int blake2b_is_lastblock( const blake2b_state *S ) +static int +blake2b_is_lastblock(const blake2b_state* S) { - return S->f[0] != 0; + return S->f[0] != 0; } -static void blake2b_set_lastblock( blake2b_state *S ) +static void +blake2b_set_lastblock(blake2b_state* S) { - if( S->last_node ) blake2b_set_lastnode( S ); + if (S->last_node) + blake2b_set_lastnode(S); - S->f[0] = (uint64_t)-1; + S->f[0] = (uint64_t) -1; } -static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +static void +blake2b_increment_counter(blake2b_state* S, const uint64_t inc) { - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); + S->t[0] += inc; + S->t[1] += (S->t[0] < inc); } -static void blake2b_init0( blake2b_state *S ) +static void +blake2b_init0(blake2b_state* S) { - size_t i; - memset( S, 0, sizeof( blake2b_state ) ); + size_t i; + memset(S, 0, sizeof(blake2b_state)); - for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; + for (i = 0; i < 8; ++i) + S->h[i] = blake2b_IV[i]; } /* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +int +blake2b_init_param(blake2b_state* S, const blake2b_param* P) { - const uint8_t *p = ( const uint8_t * )( P ); - size_t i; + const uint8_t* p = (const uint8_t*) (P); + size_t i; - blake2b_init0( S ); + blake2b_init0(S); - /* IV XOR ParamBlock */ - for( i = 0; i < 8; ++i ) - S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + /* IV XOR ParamBlock */ + for (i = 0; i < 8; ++i) + S->h[i] ^= load64(p + sizeof(S->h[i]) * i); - S->outlen = P->digest_length; - return 0; + S->outlen = P->digest_length; + return 0; } - -int blake2b_init( blake2b_state *S, size_t outlen ) +int +blake2b_init(blake2b_state* S, size_t outlen) { - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2b_init_param( S, P ); + blake2b_param P[1]; + + if ((!outlen) || (outlen > BLAKE2B_OUTBYTES)) + return -1; + + P->digest_length = (uint8_t) outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32(&P->leaf_length, 0); + store32(&P->node_offset, 0); + store32(&P->xof_length, 0); + P->node_depth = 0; + P->inner_length = 0; + memset(P->reserved, 0, sizeof(P->reserved)); + memset(P->salt, 0, sizeof(P->salt)); + memset(P->personal, 0, sizeof(P->personal)); + return blake2b_init_param(S, P); } -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +int +blake2b_init_key(blake2b_state* S, size_t outlen, const void* key, size_t keylen) { - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = (uint8_t)keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2b_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; + blake2b_param P[1]; + + if ((!outlen) || (outlen > BLAKE2B_OUTBYTES)) + return -1; + + if (!key || !keylen || keylen > BLAKE2B_KEYBYTES) + return -1; + + P->digest_length = (uint8_t) outlen; + P->key_length = (uint8_t) keylen; + P->fanout = 1; + P->depth = 1; + store32(&P->leaf_length, 0); + store32(&P->node_offset, 0); + store32(&P->xof_length, 0); + P->node_depth = 0; + P->inner_length = 0; + memset(P->reserved, 0, sizeof(P->reserved)); + memset(P->salt, 0, sizeof(P->salt)); + memset(P->personal, 0, sizeof(P->personal)); + + if (blake2b_init_param(S, P) < 0) + return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset(block, 0, BLAKE2B_BLOCKBYTES); + memcpy(block, key, keylen); + blake2b_update(S, block, BLAKE2B_BLOCKBYTES); + secure_zero_memory(block, BLAKE2B_BLOCKBYTES); /* Burn the key from stack */ + } + return 0; } -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while(0) - -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - -static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +#define G(r, i, a, b, c, d) \ + do \ + { \ + a = a + b + m[blake2b_sigma[r][2 * i + 0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2 * i + 1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while (0) + +#define ROUND(r) \ + do \ + { \ + G(r, 0, v[0], v[4], v[8], v[12]); \ + G(r, 1, v[1], v[5], v[9], v[13]); \ + G(r, 2, v[2], v[6], v[10], v[14]); \ + G(r, 3, v[3], v[7], v[11], v[15]); \ + G(r, 4, v[0], v[5], v[10], v[15]); \ + G(r, 5, v[1], v[6], v[11], v[12]); \ + G(r, 6, v[2], v[7], v[8], v[13]); \ + G(r, 7, v[3], v[4], v[9], v[14]); \ + } while (0) + +static void +blake2b_compress(blake2b_state* S, const uint8_t block[BLAKE2B_BLOCKBYTES]) { - uint64_t m[16]; - uint64_t v[16]; - size_t i; - - for( i = 0; i < 16; ++i ) { - m[i] = load64( block + i * sizeof( m[i] ) ); - } - - for( i = 0; i < 8; ++i ) { - v[i] = S->h[i]; - } - - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = blake2b_IV[4] ^ S->t[0]; - v[13] = blake2b_IV[5] ^ S->t[1]; - v[14] = blake2b_IV[6] ^ S->f[0]; - v[15] = blake2b_IV[7] ^ S->f[1]; - - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - - for( i = 0; i < 8; ++i ) { - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - } + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for (i = 0; i < 16; ++i) + { + m[i] = load64(block + i * sizeof(m[i])); + } + + for (i = 0; i < 8; ++i) + { + v[i] = S->h[i]; + } + + v[8] = blake2b_IV[0]; + v[9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + ROUND(10); + ROUND(11); + + for (i = 0; i < 8; ++i) + { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } } #undef G #undef ROUND -int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +int +blake2b_update(blake2b_state* S, const void* pin, size_t inlen) { - const unsigned char * in = (const unsigned char *)pin; - if( inlen > 0 ) - { - size_t left = S->buflen; - size_t fill = BLAKE2B_BLOCKBYTES - left; - if( inlen > fill ) + const unsigned char* in = (const unsigned char*) pin; + if (inlen > 0) { - S->buflen = 0; - memcpy( S->buf + left, in, fill ); /* Fill buffer */ - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); /* Compress */ - in += fill; inlen -= fill; - while(inlen > BLAKE2B_BLOCKBYTES) { - blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); - blake2b_compress( S, in ); - in += BLAKE2B_BLOCKBYTES; - inlen -= BLAKE2B_BLOCKBYTES; - } + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if (inlen > fill) + { + S->buflen = 0; + memcpy(S->buf + left, in, fill); /* Fill buffer */ + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress(S, S->buf); /* Compress */ + in += fill; + inlen -= fill; + while (inlen > BLAKE2B_BLOCKBYTES) + { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress(S, in); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy(S->buf + S->buflen, in, inlen); + S->buflen += inlen; } - memcpy( S->buf + S->buflen, in, inlen ); - S->buflen += inlen; - } - return 0; + return 0; } -int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +int +blake2b_final(blake2b_state* S, void* out, size_t outlen) { - uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; - size_t i; + uint8_t buffer[BLAKE2B_OUTBYTES] = { 0 }; + size_t i; - if( out == NULL || outlen < S->outlen ) - return -1; + if (out == NULL || outlen < S->outlen) + return -1; - if( blake2b_is_lastblock( S ) ) - return -1; + if (blake2b_is_lastblock(S)) + return -1; - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); + blake2b_increment_counter(S, S->buflen); + blake2b_set_lastblock(S); + memset(S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen); /* Padding */ + blake2b_compress(S, S->buf); - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + for (i = 0; i < 8; ++i) /* Output full hash to temp buffer */ + store64(buffer + sizeof(S->h[i]) * i, S->h[i]); - memcpy( out, buffer, S->outlen ); - secure_zero_memory(buffer, sizeof(buffer)); - return 0; + memcpy(out, buffer, S->outlen); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; } /* inlen, at least, should be uint64_t. Others can be size_t. */ -int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +int +blake2b(void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen) { - blake2b_state S[1]; + blake2b_state S[1]; - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; + /* Verify parameters */ + if (NULL == in && inlen > 0) + return -1; - if ( NULL == out ) return -1; + if (NULL == out) + return -1; - if( NULL == key && keylen > 0 ) return -1; + if (NULL == key && keylen > 0) + return -1; - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + if (!outlen || outlen > BLAKE2B_OUTBYTES) + return -1; - if( keylen > BLAKE2B_KEYBYTES ) return -1; + if (keylen > BLAKE2B_KEYBYTES) + return -1; - if( keylen > 0 ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } + if (keylen > 0) + { + if (blake2b_init_key(S, outlen, key, keylen) < 0) + return -1; + } + else + { + if (blake2b_init(S, outlen) < 0) + return -1; + } - blake2b_update( S, ( const uint8_t * )in, inlen ); - blake2b_final( S, out, outlen ); - return 0; + blake2b_update(S, (const uint8_t*) in, inlen); + blake2b_final(S, out, outlen); + return 0; } -int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { - return blake2b(out, outlen, in, inlen, key, keylen); +int +blake2(void* out, size_t outlen, const void* in, size_t inlen, const void* key, size_t keylen) +{ + return blake2b(out, outlen, in, inlen, key, keylen); } #if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +int +crypto_hash(unsigned char* out, unsigned char* in, unsigned long long inlen) { - return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); + return blake2b(out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0); } #endif #if defined(BLAKE2B_SELFTEST) #include #include "blake2-kat.h" -int main( void ) +int +main(void) { - uint8_t key[BLAKE2B_KEYBYTES]; - uint8_t buf[BLAKE2_KAT_LENGTH]; - size_t i, step; - - for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) - key[i] = ( uint8_t )i; + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - buf[i] = ( uint8_t )i; + for (i = 0; i < BLAKE2B_KEYBYTES; ++i) + key[i] = (uint8_t) i; - /* Test simple API */ - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) + buf[i] = (uint8_t) i; - if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + /* Test simple API */ + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { - goto fail; + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b(hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES); + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) + { + goto fail; + } } - } - - /* Test streaming API */ - for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { - for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b_state S; - uint8_t * p = buf; - size_t mlen = i; - int err = 0; - - if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { - goto fail; - } - - while (mlen >= step) { - if ( (err = blake2b_update(&S, p, step)) < 0 ) { - goto fail; + + /* Test streaming API */ + for (step = 1; step < BLAKE2B_BLOCKBYTES; ++step) + { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t* p = buf; + size_t mlen = i; + int err = 0; + + if ((err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0) + { + goto fail; + } + + while (mlen >= step) + { + if ((err = blake2b_update(&S, p, step)) < 0) + { + goto fail; + } + mlen -= step; + p += step; + } + if ((err = blake2b_update(&S, p, mlen)) < 0) + { + goto fail; + } + if ((err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) + { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) + { + goto fail; + } } - mlen -= step; - p += step; - } - if ( (err = blake2b_update(&S, p, mlen)) < 0) { - goto fail; - } - if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { - goto fail; - } - - if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { - goto fail; - } } - } - puts( "ok" ); - return 0; + puts("ok"); + return 0; fail: - puts("error"); - return -1; + puts("error"); + return -1; } #endif diff --git a/include/powerloader/fileio.hpp b/include/powerloader/fileio.hpp index 128ff359..4116c913 100644 --- a/include/powerloader/fileio.hpp +++ b/include/powerloader/fileio.hpp @@ -51,7 +51,7 @@ namespace powerloader if (!m_fs) { ec.assign(GetLastError(), std::generic_category()); - spdlog::error("Could not open file: {}", ec.message()); + spdlog::error("Could not open file {}: {}", file_path.string(), ec.message()); } } #else @@ -66,7 +66,7 @@ namespace powerloader else { ec.assign(errno, std::generic_category()); - spdlog::error("Could not open file: {}", ec.message()); + spdlog::error("Could not open file {}: {}", file_path.string(), ec.message()); } } #endif diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp index 876b7954..5b4e7a54 100644 --- a/src/cli/jlap.cpp +++ b/src/cli/jlap.cpp @@ -76,7 +76,7 @@ apply_jlap() // std::cout << j.dump(4) << std::endl; cur_patch->patches.emplace_back(std::move(j)); } - catch(...) + catch (...) { std::cout << "Could not parse patch " << line << std::endl; } @@ -100,13 +100,16 @@ apply_jlap() if (pf["from"] == repo_bsum) { auto t0 = std::chrono::high_resolution_clock::now(); - std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) << "…2 to " - << pf["to"].get().substr(0, 8) << "… "; + std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) + << "… to " << pf["to"].get().substr(0, 8) << "… "; repo_bsum = pf["to"]; // nlohmann::inplace_patch(jrdata, pf["patch"]); jrdata.patch(pf["patch"]); auto t1 = std::chrono::high_resolution_clock::now(); - std::cout << "took " << std::chrono::duration_cast(t1-t0).count() <<" ns." << std::endl; + std::cout + << "took " + << std::chrono::duration_cast(t1 - t0).count() + << " ns." << std::endl; } } } @@ -122,7 +125,8 @@ apply_jlap() } else { - std::cout << "Error: " << blake2sum("final_repodata.json") << " not matching " << patches.back().patches.back()["latest"] << std::endl; + std::cout << "Error: " << blake2sum("final_repodata.json") << " not matching " + << patches.back().patches.back()["latest"] << std::endl; } } @@ -130,22 +134,82 @@ apply_jlap() int main() { + // std::ifstream rdata("repodata.json"); + // std::unique_ptr j = std::make_unique(); + + // { + // auto t0 = std::chrono::high_resolution_clock::now(); + // rdata >> (*j); + // auto t1 = std::chrono::high_resolution_clock::now(); + // std::cout << "took " << + // std::chrono::duration_cast(t1-t0).count() <<" ms." << + // std::endl; + // } + + // { + // auto t0 = std::chrono::high_resolution_clock::now(); + // j.reset(); + // auto t1 = std::chrono::high_resolution_clock::now(); + // std::cout << "took " << + // std::chrono::duration_cast(t1-t0).count() <<" ms." << + // std::endl; + // } + powerloader::Context ctx; CURLHandle h(ctx); + std::string filehash; + std::streamoff total_size = 0, start_offset = 0; std::error_code ec; - FileIO outfile("repodata.jlap", FileIO::append_update_binary, ec); - outfile.seek(0, SEEK_END); - std::cout << outfile.tell() << std::endl; + if (fs::exists("repodata.jlap")) + { + FileIO infile("repodata.jlap", FileIO::read_binary, ec); + infile.seek(0, SEEK_END); + total_size = infile.tell(); + + char filehashtmp[64] = { 0 }; + if (total_size > 64) + { + infile.seek(-64, SEEK_END); + infile.read(filehashtmp, sizeof(char), 64); + std::cout << infile.tell() << " ends with " << filehashtmp << std::endl; + infile.seek(-64, SEEK_END); + start_offset = infile.tell(); + filehash = std::string(&filehashtmp[0], 64); + } + } h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); - h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(outfile.tell())); + if (start_offset != 0) + { + std::cout << "Resuming from offset: " << start_offset << std::endl; + h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(start_offset)); + } auto response = h.perform(); - for (auto& [k, v] : response.headers) - std::cout << k << " .. " << v << std::endl; + + // for (auto& [k, v] : response.headers) + // std::cout << k << " .. " << v << std::endl; if (response.ok()) + { + auto open_mode = fs::exists("repodata.jlap") ? FileIO::read_update_binary + : FileIO::write_update_binary; + std::cout << "Open mode is creating file? " << (open_mode == FileIO::write_update_binary) + << std::endl; + FileIO outfile("repodata.jlap", open_mode, ec); + if (open_mode == FileIO::read_update_binary) + outfile.seek(start_offset, SEEK_SET); + std::cout << "Response size: " << response.content.value().size() << std::endl; + std::string new_filehash = response.content.value().substr(0, 64); + if (response.content.value().size() == 64) + { + std::cout << "making sure that old filehash is equal new file hash: " << new_filehash + << " == " << filehash << std::endl; + assert(new_filehash == filehash); + } outfile.write(response.content.value()); - apply_jlap(); + outfile.flush(); + apply_jlap(); + } return 0; } From fb333d5eb413608127a78acc9100f6cf0607cda4 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 21 Jul 2022 10:13:53 +0200 Subject: [PATCH 6/7] improve jlap implementation --- include/powerloader/fileio.hpp | 12 +++ src/cli/jlap.cpp | 129 ++++++++++++++++----------------- 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/include/powerloader/fileio.hpp b/include/powerloader/fileio.hpp index 4116c913..87b17d16 100644 --- a/include/powerloader/fileio.hpp +++ b/include/powerloader/fileio.hpp @@ -137,6 +137,18 @@ namespace powerloader return ::ftell(m_fs); } + char get() const + { + int res = ::fgetc(m_fs); + if (res == EOF && ferror(m_fs) != 0) + { + // TODO handle this + clearerr(m_fs); + throw std::runtime_error("Got error from get()"); + } + return res; + } + std::streamoff seek(unsigned long long offset, int origin) const noexcept { assert(offset < LLONG_MAX); diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp index 5b4e7a54..e88d0199 100644 --- a/src/cli/jlap.cpp +++ b/src/cli/jlap.cpp @@ -12,9 +12,11 @@ extern "C" using namespace powerloader; namespace fs = std::filesystem; -struct Patch +struct Jlap { - std::string hash; + std::string start_hash; + std::string end_hash; + nlohmann::json metadata; std::vector patches; }; @@ -53,28 +55,43 @@ apply_jlap() std::ifstream json_file("repodata.json"); std::string line; - std::cout << blake2sum("repodata.json") << std::endl; - std::cout << blake2sum("repodata.jlap") << std::endl; - std::vector patches; - std::unique_ptr cur_patch; + Jlap jlap_doc; + std::vector lines; while (std::getline(jlap_file, line)) { - if (line.size() && line[0] != '{') - { - if (cur_patch) - patches.push_back(*cur_patch); + lines.push_back(line); + } + + if (lines.size() >= 3) + { + jlap_doc.start_hash = lines[0]; + jlap_doc.end_hash = lines[lines.size() - 1]; + + assert(jlap_doc.start_hash.size() == 64); + assert(jlap_doc.end_hash.size() == 64); - cur_patch = std::make_unique(); - cur_patch->hash = line; + try + { + jlap_doc.metadata = nlohmann::json::parse(lines[lines.size() - 2]); + assert(jlap_doc.patches.back().contains("latest")); + assert(jlap_doc.patches.back().contains("url")); } - else + catch (...) + { + std::cout << "Could not parse metadata " << line << std::endl; + } + + // parse patches in between + auto it = lines.begin() + 1; + for (; it != lines.end() - 2; ++it) { try { - auto j = nlohmann::json::parse(line); - // std::cout << j.dump(4) << std::endl; - cur_patch->patches.emplace_back(std::move(j)); + jlap_doc.patches.push_back(nlohmann::json::parse(*it)); + assert(jlap_doc.patches.back().contains("patch")); + assert(jlap_doc.patches.back().contains("from")); + assert(jlap_doc.patches.back().contains("to")); } catch (...) { @@ -90,28 +107,21 @@ apply_jlap() std::size_t i = 0; - for (auto& p : patches) + std::cout << "Found patches # " << jlap_doc.patches.size() << std::endl; + for (auto& pf : jlap_doc.patches) { - std::cout << "Found patches # " << p.patches.size() << std::endl; - for (auto& pf : p.patches) + if (pf["from"] == repo_bsum) { - if (pf.contains("from")) - { - if (pf["from"] == repo_bsum) - { - auto t0 = std::chrono::high_resolution_clock::now(); - std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) - << "… to " << pf["to"].get().substr(0, 8) << "… "; - repo_bsum = pf["to"]; - // nlohmann::inplace_patch(jrdata, pf["patch"]); - jrdata.patch(pf["patch"]); - auto t1 = std::chrono::high_resolution_clock::now(); - std::cout - << "took " - << std::chrono::duration_cast(t1 - t0).count() - << " ns." << std::endl; - } - } + auto t0 = std::chrono::high_resolution_clock::now(); + std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) << "… to " + << pf["to"].get().substr(0, 8) << "… "; + repo_bsum = pf["to"]; + // nlohmann::inplace_patch(jrdata, pf["patch"]); + jrdata.patch(pf["patch"]); + auto t1 = std::chrono::high_resolution_clock::now(); + std::cout << "took " + << std::chrono::duration_cast(t1 - t0).count() + << " ns." << std::endl; } } { @@ -119,14 +129,14 @@ apply_jlap() rpatched << jrdata.dump(2) << "\n"; } - if (blake2sum("final_repodata.json") == patches.back().patches.back()["latest"]) + if (blake2sum("final_repodata.json") == jlap_doc.metadata["latest"]) { std::cout << "All successful." << std::endl; } else { std::cout << "Error: " << blake2sum("final_repodata.json") << " not matching " - << patches.back().patches.back()["latest"] << std::endl; + << jlap_doc.metadata["latest"] << std::endl; } } @@ -134,27 +144,6 @@ apply_jlap() int main() { - // std::ifstream rdata("repodata.json"); - // std::unique_ptr j = std::make_unique(); - - // { - // auto t0 = std::chrono::high_resolution_clock::now(); - // rdata >> (*j); - // auto t1 = std::chrono::high_resolution_clock::now(); - // std::cout << "took " << - // std::chrono::duration_cast(t1-t0).count() <<" ms." << - // std::endl; - // } - - // { - // auto t0 = std::chrono::high_resolution_clock::now(); - // j.reset(); - // auto t1 = std::chrono::high_resolution_clock::now(); - // std::cout << "took " << - // std::chrono::duration_cast(t1-t0).count() <<" ms." << - // std::endl; - // } - powerloader::Context ctx; CURLHandle h(ctx); std::string filehash; @@ -172,33 +161,37 @@ main() { infile.seek(-64, SEEK_END); infile.read(filehashtmp, sizeof(char), 64); - std::cout << infile.tell() << " ends with " << filehashtmp << std::endl; - infile.seek(-64, SEEK_END); - start_offset = infile.tell(); filehash = std::string(&filehashtmp[0], 64); + + // find line before latest (to cut metadata as well) + infile.seek(-64 - 2, SEEK_END); + while (infile.get() != '\n') + { + infile.seek(-2, SEEK_CUR); + } + + start_offset = infile.tell(); } } h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); if (start_offset != 0) { - std::cout << "Resuming from offset: " << start_offset << std::endl; + std::cout << "Resuming from offset: " << start_offset << " (cutting " + << total_size - start_offset << " bytes)" << std::endl; h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(start_offset)); } auto response = h.perform(); - // for (auto& [k, v] : response.headers) - // std::cout << k << " .. " << v << std::endl; - if (response.ok()) { auto open_mode = fs::exists("repodata.jlap") ? FileIO::read_update_binary : FileIO::write_update_binary; - std::cout << "Open mode is creating file? " << (open_mode == FileIO::write_update_binary) - << std::endl; + FileIO outfile("repodata.jlap", open_mode, ec); if (open_mode == FileIO::read_update_binary) outfile.seek(start_offset, SEEK_SET); + std::cout << "Response size: " << response.content.value().size() << std::endl; std::string new_filehash = response.content.value().substr(0, 64); if (response.content.value().size() == 64) From 6870be3b480c2be2957875b35460356258049bf8 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 5 Oct 2022 15:53:33 +0200 Subject: [PATCH 7/7] fix jlap code --- CMakeLists.txt | 2 +- src/cli/jlap.cpp | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a322177..62e47375 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -347,7 +347,7 @@ if (BUILD_EXE) include_directories(BLAKE2/ref) - add_executable(jlap src/cli/jlap.cpp) + add_executable(jlap BLAKE2/ref/blake2b-ref.c src/cli/jlap.cpp) set_property(TARGET jlap PROPERTY CXX_STANDARD 17) target_link_libraries(jlap libpowerloader CLI11::CLI11 yaml-cpp curl) diff --git a/src/cli/jlap.cpp b/src/cli/jlap.cpp index e88d0199..52a3b596 100644 --- a/src/cli/jlap.cpp +++ b/src/cli/jlap.cpp @@ -116,14 +116,14 @@ apply_jlap() std::cout << "Applying patch " << i++ << " from " << repo_bsum.substr(0, 8) << "… to " << pf["to"].get().substr(0, 8) << "… "; repo_bsum = pf["to"]; - // nlohmann::inplace_patch(jrdata, pf["patch"]); - jrdata.patch(pf["patch"]); + jrdata.patch_inplace(pf["patch"]); auto t1 = std::chrono::high_resolution_clock::now(); std::cout << "took " << std::chrono::duration_cast(t1 - t0).count() << " ns." << std::endl; } } + { std::ofstream rpatched("final_repodata.json"); rpatched << jrdata.dump(2) << "\n"; @@ -174,7 +174,7 @@ main() } } - h.url("https://conda.anaconda.org/conda-forge/linux-64/repodata.jlap"); + h.url("https://conda.anaconda.org/conda-forge/osx-arm64/repodata.jlap"); if (start_offset != 0) { std::cout << "Resuming from offset: " << start_offset << " (cutting " @@ -182,6 +182,11 @@ main() h.setopt(CURLOPT_RESUME_FROM_LARGE, static_cast(start_offset)); } auto response = h.perform(); + if (!response.ok() && response.http_status == 416) { + // offset is wrong, overwrite the entire file + h.setopt(CURLOPT_RESUME_FROM_LARGE, 0); + response = h.perform(); + } if (response.ok()) {