fix: properly index split infinitives

Author: adalinesimonian

Diff for: docker-compose.yml

@@ -2,6 +2,8 @@ version: '3.8'
 services:
   redis:
     image: redis/redis-stack:7.2.0-v9
+    environment:
+      - REDIS_ARGS=--loglevel verbose
     ports:
       - '${REDIS_PORT}:6379'
       - '${REDIS_INSIGHT_PORT}:8001'

Diff for: packages/common/src/providers/index.ts

@@ -1,3 +1,4 @@
 export * from './redis.service';
+export * from './sanitization.service';
 export * from './uib-api.service';
 export * from './uib-redis.service';

Diff for: packages/common/src/providers/sanitization.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class SanitizationService {
+  /**
+   * Sanitizes a value for use in a search query. Sanitizes the value by
+   * removing any characters that could be used to escape the query.
+   * Alpha-numeric characters are allowed, as well as common punctuation, and
+   * special characters and accents that are commonly used in Norwegian.
+   * @param value The value to sanitize.
+   * @returns The sanitized value.
+   */
+  sanitize(value: string): string {
+    return value.replace(/[^a-zA-Z0-9æøåÆØÅéèÉÈáàÁÀäÄöÖüÜß\- ]/g, '');
+  }
+}

Diff for: packages/common/src/providers/uib-redis.service.ts

@@ -14,6 +14,8 @@ import {
   idForArticleKey,
   DeferredIterable,
   RawArticle,
+  UibArticle,
+  addUibArticleMetadata,
 } from '../types';
 
 /*
@@ -72,7 +74,7 @@ export interface FullSearchResult extends UiBArticleIdentifier {
   /**
    * The article data.
    */
-  data: RawArticle;
+  data: UibArticle;
 }
 
 /**
@@ -107,15 +109,15 @@ export class UibRedisService {
   async getArticle(
     dictionary: UiBDictionary,
     articleId: number,
-  ): Promise<RawArticle | null>;
+  ): Promise<UibArticle | null>;
   /**
    * Fetches the article with the given ID from the given dictionary.
    * @param identifier The identifier of the article to fetch.
    * @returns The article data, or null if the article does not exist.
    */
   async getArticle(
     identifier: UiBArticleIdentifier,
-  ): Promise<RawArticle | null>;
+  ): Promise<UibArticle | null>;
   /**
    * Fetches the article with the given ID from the given dictionary.
    * @param dictionaryOrIdentifier The dictionary to fetch the article from, or
@@ -127,19 +129,19 @@ export class UibRedisService {
   async getArticle(
     dictionaryOrIdentifier: UiBDictionary | UiBArticleIdentifier,
     articleId?: number,
-  ): Promise<RawArticle | null> {
+  ): Promise<UibArticle | null> {
     if (typeof dictionaryOrIdentifier === 'string') {
       return this.redis.client.json.get(
         uibKeys.article(dictionaryOrIdentifier, articleId!),
-      ) as Promise<RawArticle>;
+      ) as unknown as Promise<UibArticle>;
     }
 
     return this.redis.client.json.get(
       uibKeys.article(
         dictionaryOrIdentifier.dictionary,
         dictionaryOrIdentifier.id,
       ),
-    ) as Promise<RawArticle>;
+    ) as unknown as Promise<UibArticle>;
   }
 
   /**
@@ -151,17 +153,17 @@ export class UibRedisService {
   async getArticlesFromDictionary(
     dictionary: UiBDictionary,
     articleIds: number[],
-  ): Promise<Map<number, RawArticle>> {
+  ): Promise<Map<number, UibArticle>> {
     const articleList = await this.redis.client.json.mGet(
       articleIds.map((id) => uibKeys.article(dictionary, id)),
       '$',
     );
 
-    const articles = new Map<number, RawArticle>();
+    const articles = new Map<number, UibArticle>();
 
     for (let i = 0; i < articleIds.length; i++) {
       if (articleList[i] !== null) {
-        articles.set(articleIds[i], articleList[i] as RawArticle);
+        articles.set(articleIds[i], articleList[i] as unknown as UibArticle);
       }
     }
 
@@ -179,14 +181,14 @@ export class UibRedisService {
     transaction: RedisMulti,
     dictionary: UiBDictionary,
     articleId: number,
-    article: RawArticle,
+    article: RawArticle | UibArticle,
   ): void {
     this.#throwIfWritesNotAllowed();
 
     transaction.json.set(
       uibKeys.article(dictionary, articleId),
       '$',
-      article as RedisJSON,
+      addUibArticleMetadata(article) as unknown as RedisJSON,
     );
   }
 
@@ -507,7 +509,7 @@ export class UibRedisService {
       total,
       results: new DeferredIterable(documents).map(({ id, value }) => ({
         ...idForArticleKey(id),
-        data: value as unknown as RawArticle,
+        data: value as unknown as UibArticle,
       })),
     };
   }

Diff for: packages/common/src/types/deferred-iterable.types.ts

@@ -1,4 +1,5 @@
 import type { InspectOptionsStylized } from 'util';
+import { UnionIterable } from './union-iterable.types';
 
 /**
  * A deferred iterable is an iterable that performs some operations on each
@@ -18,9 +19,19 @@ export class DeferredIterable<T> implements Iterable<T> {
     }
   }
 
+  /**
+   * Returns a new union iterable that is a union of this iterable and the
+   * given iterable.
+   * @param other The iterable to union with.
+   */
+  union<U>(other: Iterable<U>): UnionIterable<T | U> {
+    return new UnionIterable([this.#originalData, other] as Iterable<T | U>[]);
+  }
+
   /**
    * Defer a map operation to be performed when the iterable is iterated over.
    * Returns a new deferred iterable.
+   * @param mapFn The map function to defer.
    */
   map<TMapped>(mapFn: (value: T) => TMapped): DeferredIterable<TMapped> {
     const newIterable = new DeferredIterable(this.#originalData);

Diff for: packages/common/src/types/index.ts

@@ -2,3 +2,4 @@ export * from './deferred-iterable.types';
 export * from './redis.types';
 export * from './uib-api.types';
 export * from './uib-redis.types';
+export * from './union-iterable.types';

Diff for: packages/common/src/types/uib-api.types.ts

@@ -176,7 +176,11 @@ export type ArticleElement =
       text: string;
     }
   | {
-      type_: 'quote_inset' | 'explanation' | 'etymology_language';
+      type_:
+        | 'quote_inset'
+        | 'explanation'
+        | 'etymology_language'
+        | 'etymology_reference';
       content: string | null;
       items: ArticleElement[];
     }
@@ -247,7 +251,7 @@ export type ArticleElement =
 /**
  * The format of the raw data returned by the UiB API when fetching an article.
  */
-export type RawArticle = {
+export interface RawArticle {
   article_id: number;
   submitted?: string;
   suggest: string[];
@@ -275,4 +279,4 @@ export type RawArticle = {
     definitions: ArticleElement[];
   };
   to_index: string[];
-};
+}

Diff for: packages/common/src/types/uib-redis.types.ts

@@ -1,5 +1,5 @@
 import { DateTime } from 'luxon';
-import { RawArticleMetadata, UiBDictionary } from './uib-api.types';
+import { RawArticle, RawArticleMetadata, UiBDictionary } from './uib-api.types';
 
 /**
  * Article metadata.
@@ -57,9 +57,15 @@ export enum ArticleIndex {
   /** The index for article lemmas. */
   Lemma = 'lemma',
 
+  /** The index for exact-matching the whole lemma. */
+  LemmaExact = 'lemma_exact',
+
   /** The index for suggestion terms for a given article. */
   Suggest = 'suggest',
 
+  /** The index for exact-matching the whole suggestion term. */
+  SuggestExact = 'suggest_exact',
+
   /** The index for article etymology text. */
   Etymology = 'etymology',
 
@@ -72,6 +78,9 @@ export enum ArticleIndex {
   /** The index for inflection word forms. */
   Inflection = 'inflection',
 
+  /** The index for exact-matching the whole inflection word form. */
+  InflectionExact = 'inflection_exact',
+
   /** The index for split infinitive attributes. */
   SplitInfinitive = 'split_infinitive',
 }
@@ -139,3 +148,45 @@ export const idForArticleKey = (key: string): UiBArticleIdentifier => {
     id: Number.parseInt(articleId, 10),
   };
 };
+
+/**
+ * The format of article data as stored in Redis.
+ */
+export interface UibArticle extends RawArticle {
+  /** Additional metadata used for indexing and other purposes. */
+  __ordbokapi__: UibArticleMetadata;
+}
+
+/**
+ * Returns whether or not the given article is a processed UiB article.
+ */
+export function isUibArticle(article: RawArticle): article is UibArticle {
+  return '__ordbokapi__' in article;
+}
+
+/**
+ * Given a raw article, returns the article with additional metadata added.
+ */
+export function addUibArticleMetadata(article: RawArticle): UibArticle {
+  if (isUibArticle(article)) {
+    return article;
+  }
+
+  return {
+    ...article,
+    __ordbokapi__: {
+      hasSplitInf: article.lemmas?.some((lemma) => lemma?.split_inf),
+    },
+  };
+}
+
+/**
+ * Additional metadata used for indexing and other purposes.
+ */
+export interface UibArticleMetadata {
+  /**
+   * Whether or not any of the lemmas represent a verb with a split infinitive
+   * (kløyvd infinitiv).
+   */
+  hasSplitInf: boolean;
+}

Diff for: packages/common/src/types/union-iterable.types.ts

@@ -0,0 +1,92 @@
+import type { InspectOptionsStylized } from 'util';
+import { DeferredIterable } from './deferred-iterable.types';
+
+/**
+ * Provides a type that represents an iterable of a union of types. When
+ * iterated over, it will yield the values of the first iterable, then the
+ * values of the second iterable, and so on.
+ */
+export class UnionIterable<T = never> implements Iterable<T> {
+  #iterables: Iterable<T>[];
+
+  constructor(iterables?: Iterable<T>[]) {
+    this.#iterables = iterables ?? [];
+  }
+
+  /**
+   * Iterates over the iterables in the union and yields their values.
+   */
+  *[Symbol.iterator](): Iterator<T> {
+    for (const iterable of this.#iterables) {
+      yield* iterable;
+    }
+  }
+
+  /**
+   * Returns a new union iterable that is a union of this iterable and the
+   * given iterable.
+   * @param other The iterable to union with.
+   * @returns A new union iterable that is a union of this iterable and the
+   * given iterable.
+   */
+  union<U>(other: Iterable<U>): UnionIterable<T | U> {
+    return new UnionIterable(
+      (this.#iterables as Iterable<T | U>[]).concat(other),
+    );
+  }
+
+  /**
+   * Concatenates the given iterables to this union iterable. This mutates the
+   * union iterable.
+   * @param iterables The iterables to concatenate.
+   * @returns This union iterable.
+   */
+  concat(...iterables: Iterable<T>[]): this {
+    this.#iterables.push(...iterables);
+
+    return this;
+  }
+
+  /**
+   * The size of the union iterable. This is the number of iterables in the
+   * union, not the total number of elements in all the iterables.
+   */
+  get size(): number {
+    return this.#iterables.length;
+  }
+
+  /**
+   * Defer a map operation to be performed when the iterable is iterated over.
+   * Returns a new deferred iterable.
+   * @param mapFn The map function to defer.
+   */
+  map<TMapped>(mapFn: (value: T) => TMapped): DeferredIterable<TMapped> {
+    return new DeferredIterable(this).map(mapFn);
+  }
+
+  /**
+   * Custom inspect function for Node.js `util.inspect`. Allows for a more
+   * readable output in the console.
+   */
+  [Symbol.for('nodejs.util.inspect.custom')](
+    depth: number,
+    options: InspectOptionsStylized,
+    inspect: (value: unknown, options: InspectOptionsStylized) => string,
+  ): string {
+    if (depth < 0) {
+      return options.stylize('[UnionIterable]', 'special');
+    }
+
+    const newOptions = Object.assign({}, options, {
+      depth:
+        options.depth === null || options.depth === undefined
+          ? null
+          : options.depth - 1,
+    });
+
+    const arr = Array.from(this);
+    const arrayInspect = inspect(arr, newOptions);
+
+    return `UnionIterable(${arr.length}) {${arrayInspect.slice(1, -1)}}`;
+  }
+}