Merge pull request #52 from krai/more_info_detokenised

Detokenised log now preserves qsl_idx, seq_id and token_count fields
krai · Aug 6, 2024 · a0f7cb6 · a0f7cb6
2 parents cccfc97 + 8037235
commit a0f7cb6
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/base_llama2_loadgen_experiment/code_axs.py b/base_llama2_loadgen_experiment/code_axs.py
@@ -44,7 +44,12 @@ def detokenise(
         tokens = [
             int.from_bytes(bytes.fromhex(tok), byteorder="little") for tok in hex_tokens
         ]
-        output_log.append(tokeniser.decode(tokens))
+        output_log.append({
+            "seq_id" : item["seq_id"],
+            "qsl_idx" : item["qsl_idx"],
+            "data": tokeniser.decode(tokens),
+            "token_count" : item["token_count"]
+        })
 
     with open(output_log_path, "w") as f:
         json.dump(output_log, f, indent=2)