use uuid25 format; fix sorting (#288)

- convert uuidv7 to uuid25 for a more compact format
- centralize id generation within a util, test it
- fix importer / create not generating ids using the existing timestamp; ids should now sort by note creation time (only lightly tested)

package.json

@@ -22,6 +22,7 @@
     "better-sqlite3": "^9.2.2",
     "electron-store": "^8.0.1",
     "knex": "^2.5.0",
+    "uuid25": "^0.1.5",
     "uuidv7": "^0.6.3"
   },
   "devDependencies": {

src/preload/client/documents.ts

@@ -2,7 +2,6 @@ import { Database } from "better-sqlite3";
 import fs from "fs";
 import { Knex } from "knex";
 import path from "path";
-import { uuidv7obj } from "uuidv7";
 import yaml from "yaml";
 import { mdastToString, parseMarkdown, selectNoteLinks } from "../../markdown";
 import { parseNoteLink } from "../../views/edit/editor/features/note-linking/toMdast";
@@ -20,6 +19,7 @@ import {
   SearchResponse,
   UpdateRequest,
 } from "./types";
+import { createId } from "./util";
 
 // document as it appears in the database
 interface DocumentDb {
@@ -182,13 +182,14 @@ export class DocumentsClient {
     args: CreateRequest,
     index: boolean = true,
   ): Promise<[string, string]> => {
-    const id = args.id || uuidv7obj().toHex();
     args.frontMatter.tags = Array.from(new Set(args.frontMatter.tags));
     args.frontMatter.createdAt =
       args.frontMatter.createdAt || new Date().toISOString();
     args.frontMatter.updatedAt =
       args.frontMatter.updatedAt || new Date().toISOString();
 
+    const id = args.id || createId(Date.parse(args.frontMatter.createdAt));
+
     const content = this.prependFrontMatter(args.content, args.frontMatter);
     const docPath = await this.files.uploadDocument(
       { id, content },

src/preload/client/files.ts

@@ -2,10 +2,11 @@ import Store from "electron-store";
 
 import fs from "fs";
 import path from "path";
-import { uuidv7obj } from "uuidv7";
 import { Files } from "../files";
 const { readFile, writeFile, access, stat } = fs.promises;
 
+import { createId } from "./util";
+
 interface UploadResponse {
   filename: string;
   // filepath: string;
@@ -69,7 +70,7 @@ export class FilesClient {
     const dir = path.join(chronRoot, "_attachments");
 
     const { buffer, extension } = dataURLToBufferAndExtension(dataUrl);
-    const filename = `${uuidv7obj().toHex()}${extension}`;
+    const filename = `${createId()}${extension}`;
     const filepath = path.join(dir, filename);
 
     return new Promise<UploadResponse>((resolve, reject) => {
@@ -92,7 +93,7 @@ export class FilesClient {
     const dir = path.join(chronRoot, "_attachments");
 
     const ext = path.parse(file.name).ext;
-    const filename = `${uuidv7obj().toHex()}${ext || ".unknown"}`;
+    const filename = `${createId()}${ext || ".unknown"}`;
     const filepath = path.join(dir as string, filename);
     return new Promise<UploadResponse>((res, rej) => {
       const stream = fs.createWriteStream(filepath);

src/preload/client/importer.ts

@@ -15,7 +15,6 @@ import * as mdast from "mdast";
 
 export type IImporterClient = ImporterClient;
 
-import { uuidv7obj } from "uuidv7";
 import {
   isNoteLink,
   mdastToString,
@@ -24,6 +23,7 @@ import {
 import { FilesImportResolver } from "./importer/FilesImportResolver";
 import { SourceType } from "./importer/SourceType";
 import { parseTitleAndFrontMatterForImport } from "./importer/frontmatter";
+import { createId } from "./util";
 
 // UUID in Notion notes look like 32 character hex strings; make this somewhat more lenient
 const hexIdRegex = /\b[0-9a-f]{16,}\b/;
@@ -118,7 +118,7 @@ export class ImporterClient {
     importDir = path.resolve(importDir);
 
     await this.clearIncomplete();
-    const importerId = uuidv7obj().toHex();
+    const importerId = createId();
     const chroniclesRoot = await this.ensureRoot();
 
     // Ensure `importDir` is a directory and can be accessed
@@ -244,24 +244,22 @@ export class ImporterClient {
       // 2. Whether to use birthtime or mtime
       // 3. Which timezone to use
       // 4. Whether to use the front-matter date or the file date
+      const createdAtDate = frontMatter.createdAt
+        ? new Date(Date.parse(frontMatter.createdAt))
+        : file.stats.birthtime || file.stats.mtime || new Date();
+
       const requiredFm: FrontMatter = {
         ...frontMatter,
         tags: frontMatter.tags || [],
-        createdAt:
-          frontMatter.createdAt ||
-          file.stats.birthtime.toISOString() ||
-          file.stats.mtime.toISOString() ||
-          new Date().toISOString(),
+        createdAt: frontMatter.createdAt || createdAtDate.toISOString(),
         updatedAt:
           frontMatter.updatedAt ||
           file.stats.mtime.toISOString() ||
           new Date().toISOString(),
       };
 
-      // todo: handle additional kinds of frontMatter; just add a column for them
-      // and ensure they are not overwritten when editing existing files
-      // https://github.com/cloverich/chronicles/issues/127
-      const chroniclesId = uuidv7obj().toHex();
+      const chroniclesId = createId(createdAtDate.getTime());
+
       const stagedNote: StagedNote = {
         importerId,
         chroniclesId: chroniclesId,
@@ -289,15 +287,15 @@ export class ImporterClient {
         // that is too long, or a front-matter key that is not supported, etc, user
         // can use table logs to fix and re-run th e import
         try {
-          const noteId = uuidv7obj().toHex();
           await this.knex("import_notes").insert({
             importerId,
             sourcePath: file.path,
             content: contents,
             error: (e as any).message,
 
-            // note: these all have non-null / unique constraints:
-            chroniclesId: noteId,
+            // note: these all have non-null / unique constraints;
+            // its expected re-processing will delete / replace these values
+            chroniclesId: createId(),
             chroniclesPath: "staging_error",
             journal: "staging_error",
             frontMatter: {},
@@ -628,7 +626,7 @@ export class ImporterClient {
     } catch (err) {
       // Generate a new, ugly name; user can decide what they want to do via
       // re-naming later b/c rn its not worth the complexity of doing anything else
-      journalName = uuidv7obj().toHex();
+      journalName = createId();
 
       // too long, reserved name, non-unique, etc.
       // known cases from my own import:

src/preload/client/importer/FilesImportResolver.ts

@@ -2,10 +2,10 @@ import fs from "fs";
 import { Knex } from "knex";
 import mdast from "mdast";
 import path from "path";
-import { uuidv7obj } from "uuidv7";
 import { isNoteLink } from "../../../markdown";
 import { Files, PathStatsFile } from "../../files";
 import { IFilesClient } from "../files";
+import { createId } from "../util";
 
 const ATTACHMENTS_DIR = "_attachments";
 
@@ -125,7 +125,7 @@ export class FilesImportResolver {
         // based on Files.walk behavior
         sourcePathResolved: filestats.path,
         filename: path.basename(filestats.path, ext),
-        chroniclesId: uuidv7obj().toHex(),
+        chroniclesId: createId(filestats.stats.birthtimeMs),
         extension: ext,
       });
     } catch (err: any) {

src/preload/client/sync.ts

@@ -2,13 +2,13 @@ import { Database } from "better-sqlite3";
 import fs from "fs";
 import { Knex } from "knex";
 import path from "path";
-import { UUID } from "uuidv7";
 import { Files } from "../files";
 import { IDocumentsClient } from "./documents";
 import { IFilesClient } from "./files";
 import { IJournalsClient } from "./journals";
 import { IPreferencesClient } from "./preferences";
 import { SKIPPABLE_FILES, SKIPPABLE_PREFIXES } from "./types";
+import { checkId } from "./util";
 
 export type ISyncClient = SyncClient;
 
@@ -85,11 +85,11 @@ export class SyncClient {
 
     for await (const file of Files.walk(rootDir, 1, shouldIndex)) {
       const { name, dir } = path.parse(file.path);
-      // filename is id; ensure it is formatted as a uuidv7
+      // filename is id; ensure it is formatted correctly
       const documentId = name;
 
       try {
-        UUID.parse(documentId);
+        checkId(documentId);
       } catch (e) {
         console.error(
           "Invalid document id in sync; skipping",

src/preload/client/util.test.ts

@@ -0,0 +1,24 @@
+import { assert } from "chai";
+import { suite, test } from "mocha";
+import { createId } from "./util";
+
+suite("id generation", () => {
+  test("it generates ids in order", () => {
+    const ids = [createId(), createId(), createId()];
+
+    assert.sameOrderedMembers(ids, ids.slice().sort());
+  });
+
+  test("it generates ids in order when timestamp provided", () => {
+    const backwards = [
+      createId(Date.parse("2024-01-01")),
+      createId(Date.parse("2023-01-01")),
+      createId(Date.parse("2022-01-01")),
+    ];
+
+    assert.sameOrderedMembers(
+      backwards.slice().reverse(),
+      backwards.slice().sort(),
+    );
+  });
+});

src/preload/client/util.ts

@@ -0,0 +1,35 @@
+import { Uuid25 } from "uuid25";
+import { UUID, V7Generator, uuidv7obj } from "uuidv7";
+
+const uuidV7Generator = new V7Generator();
+
+/**
+ * Generates a time-sortable chronicles id, optionally incorporating the
+ * document / files timestamp so it sorts by creation date
+ *
+ * @param unixTsMs - e.g. from Date.parse or new Date().getTime()
+ * @returns A uuid25 formatted string
+ */
+export function createId(unixTsMs?: number): string {
+  const uuid = unixTsMs
+    ? uuidV7Generator.generateOrResetCore(unixTsMs, 10_000)
+    : uuidv7obj();
+  const id = Uuid25.fromBytes(uuid.bytes);
+  return id.value;
+}
+
+/**
+ * Convert (legacy) uuidv7 str to uuid25
+ */
+export function convertId(uuidV7Str: string): string {
+  const uuid = UUID.parse(uuidV7Str);
+  const id = Uuid25.fromBytes(uuid.bytes);
+  return id.value;
+}
+
+/**
+ * Throw if uuid string is invalid
+ */
+export function checkId(uuid25Str: string): void {
+  Uuid25.parseUuid25(uuid25Str);
+}

yarn.lock

@@ -6019,6 +6019,11 @@ util-deprecate@^1.0.1, util-deprecate@^1.0.2:
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
 
+uuid25@^0.1.5:
+  version "0.1.5"
+  resolved "https://registry.yarnpkg.com/uuid25/-/uuid25-0.1.5.tgz#81031c8de17561e9a7de0b18ad24dd6e03054b9f"
+  integrity sha512-ZckmfbOOQXhcavtkqtT9wY+spMyqeAvDHZcWcyEc0qV1R/MtQ9ZbZ+Zd/g/W6DBK5BewFeTbWcsAOMKcxhv6mA==
+
 uuidv7@^0.6.3:
   version "0.6.3"
   resolved "https://registry.yarnpkg.com/uuidv7/-/uuidv7-0.6.3.tgz#2abcfa683b4ad4a0cbbbaedffc3ef940c110cf10"