Implemented unhashed codes: zlib urlsafe base64 encoded by default, human readable with -h flag.

Squashed commit of the following:
commit bbdd156
commit 4c5c903
commit 13756eb
commit 434f442
commit e99df11
commit 005be9e
commit afce593
commit f0bc228
commit bd67d66

Author: adrianomitre

codegen.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+#
+# To make it easy to use with GNU Parallel, e.g.,
+#   parallel codegen.sh ::: *.mp3
+#
+echoprint-codegen -h "$1" > "$1.json"

src/Codegen.cxx

@@ -23,6 +23,10 @@ using std::string;
 using std::vector;
 
 Codegen::Codegen(const float* pcm, unsigned int numSamples, int start_offset) {
+    for (int i = 0; i < 2; ++i) {
+        is_code_string_cached[i] = false;
+    }
+
     if (Params::AudioStreamInput::MaxSamples < (uint)numSamples)
         throw std::runtime_error("File was too big\n");
 
@@ -38,7 +42,11 @@ Codegen::Codegen(const float* pcm, unsigned int numSamples, int start_offset) {
     Fingerprint *pFingerprint = new Fingerprint(pSubbandAnalysis, start_offset);
     pFingerprint->Compute();
 
+#if defined(UNHASHED_CODES)
+    _CodeString = createCodeStringJSON(pFingerprint->getCodes());
+#else
     _CodeString = createCodeString(pFingerprint->getCodes());
+#endif
     _NumCodes = pFingerprint->getCodes().size();
 
     delete pFingerprint;
@@ -63,6 +71,27 @@ string Codegen::createCodeString(vector<FPCode> vCodes) {
     return compress(codestream.str());
 }
 
+string Codegen::createCodeStringJSON(vector<FPCode> vCodes) {
+    if (vCodes.size() < 3) {
+        return "";
+    }
+    std::ostringstream codestream;
+    codestream << "[";
+    for (uint i = 0; i < vCodes.size(); i++) {
+        int hash = vCodes[i].code;
+        // codestream << std::setw(5) << hash;
+        codestream << "[" <<  vCodes[i].frame << ", "
+            << ((hash >> 20) & 7) << ", "
+            << ((hash >> 10) & 1023) << ", "
+            << ((hash >>  0) & 1023)
+            << "]";
+        if (i < vCodes.size()-1) {
+            codestream << ", ";
+        }
+    }
+    codestream << "]";
+    return codestream.str();
+}
 
 string Codegen::compress(const string& s) {
     long max_compressed_length = s.size()*2;
@@ -89,3 +118,24 @@ string Codegen::compress(const string& s) {
     delete [] compressed;
     return encoded;
 }
+
+std::string Codegen::getCodeString(bool human_readable) {
+    const uint n = human_readable;
+    if (!is_code_string_cached[n]) {
+        is_code_string_cached[n] = true;
+        if (human_readable) {
+            if (_CodeString.size() > 0) {
+                code_string_cache[n] = _CodeString;
+            } else {
+                code_string_cache[n] = "[]";
+            }
+        } else {
+            if (_CodeString.size() > 0) {
+                code_string_cache[n] = '"' + compress(_CodeString) + '"';
+            } else {
+                code_string_cache[n] = "\"\"";
+            }
+        }
+    }
+    return code_string_cache[n];
+}

src/Codegen.h

@@ -33,16 +33,20 @@ class CODEGEN_API Codegen {
 public:
     Codegen(const float* pcm, unsigned int numSamples, int start_offset);
 
-    std::string getCodeString(){return _CodeString;}
+    std::string getCodeString(bool human_readable);
+
     int getNumCodes(){return _NumCodes;}
     static double getVersion() { return ECHOPRINT_VERSION; }
 private:
     Fingerprint* computeFingerprint(SubbandAnalysis *pSubbandAnalysis, int start_offset);
     std::string createCodeString(std::vector<FPCode> vCodes);
+    std::string createCodeStringJSON(std::vector<FPCode> vCodes);
 
     std::string compress(const std::string& s);
     std::string _CodeString;
     int _NumCodes;
+    bool is_code_string_cached[2];
+    std::string code_string_cache[2];
 };
 
 #endif

src/Fingerprint.cxx

@@ -12,6 +12,8 @@
 #include "win_funcs.h"
 #endif
 
+#define SATURATE(var, val) if ((var) > (val)) (var) = (val);
+
 unsigned int MurmurHash2 ( const void * key, int len, unsigned int seed ) {
     // MurmurHash2, by Austin Appleby http://sites.google.com/site/murmurhash/
     // m and r are constants set by austin
@@ -182,13 +184,18 @@ uint Fingerprint::quantized_time_for_frame_absolute(uint frame) {
 
 void Fingerprint::Compute() {
     uint actual_codes = 0;
+#if !defined(UNHASHED_CODES)
     unsigned char hash_material[5];
     for(uint i=0;i<5;i++) hash_material[i] = 0;
+#endif
     uint * onset_counter_for_band;
     matrix_u out;
     uint onset_count = adaptiveOnsets(345, out, onset_counter_for_band);
     _Codes.resize(onset_count*6);
 
+#if defined(UNHASHED_CODES)
+    assert(SUBBANDS <= 8);
+#endif
     for(unsigned char band=0;band<SUBBANDS;band++) {
         if (onset_counter_for_band[band]>2) {
             for(uint onset=0;onset<onset_counter_for_band[band]-2;onset++) {
@@ -226,12 +233,22 @@ void Fingerprint::Compute() {
                     // Quantize the time deltas to 23ms
                     short time_delta0 = (short)quantized_time_for_frame_delta(p[0][k]);
                     short time_delta1 = (short)quantized_time_for_frame_delta(p[1][k]);
+                    uint hashed_code;
+#if defined(UNHASHED_CODES)
+                    assert(time_delta0 <= 1023);
+                    assert(time_delta1 <= 1023);
+#if defined(NDEBUG)
+                    SATURATE(time_delta0, 1023);
+                    SATURATE(time_delta1, 1023);
+#endif
+                    hashed_code = ((band & 7) << 20) | ((time_delta0 & 1023) << 10) | (time_delta1 & 1023);
+#else
                     // Create a key from the time deltas and the band index
                     memcpy(hash_material+0, (const void*)&time_delta0, 2);
                     memcpy(hash_material+2, (const void*)&time_delta1, 2);
                     memcpy(hash_material+4, (const void*)&band, 1);
-                    uint hashed_code = MurmurHash2(&hash_material, 5, HASH_SEED) & HASH_BITMASK;
-
+                    hashed_code = MurmurHash2(&hash_material, 5, HASH_SEED) & HASH_BITMASK;
+#endif
                     // Set the code alongside the time of onset
                     _Codes[actual_codes++] = FPCode(time_for_onset_ms_quantized, hashed_code);
                     //fprintf(stderr, "whee %d,%d: [%d, %d] (%d, %d), %d = %u at %d\n", actual_codes, k, time_delta0, time_delta1, p[0][k], p[1][k], band, hashed_code, time_for_onset_ms_quantized);

src/Makefile

@@ -7,8 +7,8 @@ VERSION_COMPAT := $(word 1, $(EP_VERSION)).$(word 2, $(EP_VERSION))
 UNAME := $(shell uname -s)
 CXX=g++
 CC=gcc
-#OPTFLAGS=-g -O0
-OPTFLAGS=-O3 -DBOOST_UBLAS_NDEBUG -DNDEBUG
+#OPTFLAGS=-g -O0 -DUNHASHED_CODES
+OPTFLAGS=-O3 -DBOOST_UBLAS_NDEBUG -DNDEBUG -DUNHASHED_CODES
 BOOST_CFLAGS=-I/usr/local/include/boost-1_35
 TAGLIB_CFLAGS=`taglib-config --cflags`
 TAGLIB_LIBS=`taglib-config --libs`

src/main.cxx

@@ -172,15 +172,15 @@ void print_json_to_screen(char* output, int count, int done) {
     if(done==1 && count>1) {
         printf("[\n%s,\n", output);
     } else if(done==1 && count == 1) {
-        printf("[\n%s\n]\n", output);
+        printf("%s\n", output);
     } else if(done == count) {
         printf("%s\n]\n", output);
     } else {
         printf("%s,\n", output);
     }
 }
 
-char *make_json_string(codegen_response_t* response) {
+char *make_json_string(codegen_response_t* response, bool human_readable_code) {
 
     if (response->error != NULL) {
         return response->error;
@@ -190,12 +190,12 @@ char *make_json_string(codegen_response_t* response) {
     auto_ptr<Metadata> pMetadata(new Metadata(response->filename));
 
     // preamble + codelen
-    char* output = (char*) malloc(sizeof(char)*(16384 + strlen(response->codegen->getCodeString().c_str()) ));
+    char* output = (char*) malloc(sizeof(char)*(16384 + strlen(response->codegen->getCodeString(human_readable_code).c_str()) ));
 
     sprintf(output,"{\"metadata\":{\"artist\":\"%s\", \"release\":\"%s\", \"title\":\"%s\", \"genre\":\"%s\", \"bitrate\":%d,"
                     "\"sample_rate\":%d, \"duration\":%d, \"filename\":\"%s\", \"samples_decoded\":%d, \"given_duration\":%d,"
                     " \"start_offset\":%d, \"version\":%2.2f, \"codegen_time\":%2.6f, \"decode_time\":%2.6f}, \"code_count\":%d,"
-                    " \"code\":\"%s\", \"tag\":%d}",
+                    " \"code\":%s, \"tag\":%d}",
         escape(pMetadata->Artist()).c_str(),
         escape(pMetadata->Album()).c_str(),
         escape(pMetadata->Title()).c_str(),
@@ -211,15 +211,45 @@ char *make_json_string(codegen_response_t* response) {
         response->t2,
         response->t1,
         response->codegen->getNumCodes(),
-        response->codegen->getCodeString().c_str(),
+        response->codegen->getCodeString(human_readable_code).c_str(),
         response->tag
     );
     return output;
 }
 
+// Return true if has specified flag, false otherwise. Remove specified flags.
+bool extract_flag(int* p_argc, char*** p_argv, const char* flag) {
+    int argc = *p_argc;
+    char** const argv = *p_argv;
+    for (int i = 1; i < argc; ++i) {
+        if (strcmp(flag, argv[i]) == 0) {
+            for (int k = i+1; k < argc; ++k) {
+                *(*p_argv+k-1) = *(*p_argv+k);
+            }
+            --argc;
+        }
+    }
+    const bool changed = (*p_argc != argc);
+    *p_argc = argc;
+    return changed;
+ }
+
+bool take_human_readable_flag(int* p_argc, char*** p_argv) {
+    return extract_flag(p_argc, p_argv, "-h");
+ }
+
 int main(int argc, char** argv) {
+    const bool human_readable_code = take_human_readable_flag(&argc, &argv);
+
     if (argc < 2) {
-        fprintf(stderr, "Usage: %s [ filename | -s ] [seconds_start] [seconds_duration] [< file_list (if -s is set)]\n", argv[0]);
+        const char left_margin[] = "    ";
+        fprintf(stderr,
+            "Usage:\n"
+            "%s%s [ filename | -s ] [seconds_start] [seconds_duration] "
+            "[< file_list (if -s is set)]\n",
+            left_margin, argv[0]);
+        fputs("OPTIONS\n", stderr);
+        fprintf(stderr, "%s-h\tHuman-readable code (in contrast to the default""base64 encoded zlib compressed code)\n", left_margin);
         exit(-1);
     }
 
@@ -254,7 +284,8 @@ int main(int argc, char** argv) {
         // Threading doesn't work in windows yet.
         for(int i=0;i<count;i++) {
             codegen_response_t* response = codegen_file((char*)files[i].c_str(), start_offset, duration, i);
-            char *output = make_json_string(response);
+            char *output = make_json_string(response, human_readable_code
+                );
             print_json_to_screen(output, count, i+1);
             if (response->codegen) {
                 delete response->codegen;
@@ -303,7 +334,7 @@ int main(int argc, char** argv) {
                     parm[i]->done = 0;
                     done++;
                     codegen_response_t *response = (codegen_response_t*)parm[i]->response;
-                    char *json = make_json_string(response);
+                    char *json = make_json_string(response, human_readable_code);
                     print_json_to_screen(json, count, done);
                     if (response->codegen) {
                         delete response->codegen;