Working with Dgraph in Modus

dgraph-101/README.md

@@ -0,0 +1,45 @@
+# dgraph with modus 101
+
+Demo from 11/16/24 ModusHack Webinar.
+
+## Dgraph
+
+Start a local instance using Docker.
+
+```sh
+   docker run --name dgraph-101 -d -p "8080:8080" -p "9080:9080" -v ~/dgraph-101:/dgraph dgraph/standalone:latest
+```
+
+Start Ratel, a Graphical data explorer for Dgraph:
+```sh
+docker run --name ratel  -d -p "8000:8000"  dgraph/ratel:latest
+```
+
+## Data model
+
+Modus is code first, simply define you data using classes.
+In this example we are using `Product` and `Category` defined in `classes.ts` file.
+
+## Define your API.
+
+index.ts is where we export the functions that are exposed as GraphQL operations.
+
+This project defines the following operations:
+
+- upsertProduct
+- getProduct
+- deleteProduct
+
+Note that Modus is exposing `getProduct` as `product` to follow common coding and GraphQL practices.
+Modus also exposes `upsertProduct` and `deleteProduct` as a GraphQL Mutation automatically.
+
+- getProductsByCategory
+The data is saved in Dgraph as a graph.
+This function shows how to easily use the relationships in the graph.
+
+- searchProducts
+Finally we are using an AI model to create text embeddings for our Products.
+
+As the embeddings are stored in Dgraph for each Product entity, we can easily expose an API to search by natural language and similarity
+
+

dgraph-101/api-as/.gitignore

@@ -0,0 +1,15 @@
+# Ignore macOS system files
+.DS_Store
+
+# Ignore environment variable files
+.env
+.env.*
+
+# Ignore build output directories
+build/
+
+# Ignore node_modules folders
+node_modules/
+
+# Ignore logs generated by as-test
+logs/

dgraph-101/api-as/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "plugins": ["assemblyscript-prettier"]
+}

dgraph-101/api-as/asconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./node_modules/@hypermode/modus-sdk-as/plugin.asconfig.json",
+  "options": {
+    "transform": ["@hypermode/modus-sdk-as/transform", "json-as/transform"]
+  }
+}

dgraph-101/api-as/assembly/classes.ts

@@ -0,0 +1,31 @@
+
+/* 
+    This file contains the classes that are used in our App.
+    The classes are annotated with the @json decorator 
+    to be serialized and deserialized as json string.
+    @alias is used to rename the properties in the json string to match Dgraph best practices.
+*/
+
+
+@json
+export class Product {
+
+  @alias("Product.id")
+  id!: string;
+
+  @alias("Product.title")
+  title: string = "";
+
+  @alias("Product.description")
+  description: string = "";
+
+  @alias("Product.category")
+  @omitnull()
+  category: Category | null = null;
+}
+
+@json
+export class Category {
+  @alias("Category.name")
+  name: string = "";
+}

dgraph-101/api-as/assembly/dgraph-utils.ts

@@ -0,0 +1,125 @@
+/**
+ * Utility classes and functions used to interact with Dgraph.
+ */
+import { dgraph } from "@hypermode/modus-sdk-as"
+import { JSON } from "json-as"
+
+@json
+class Uid {
+  uid: string = "";
+}
+@json 
+class UidResult {
+  uids: Uid[] = [];
+}
+@json 
+export class ListOf<T> {
+    list: T[] = [];
+}
+export class NestedEntity {
+    predicate: string = "";
+    type: string = "";
+    id_field: string = "";
+    id_value: string | null = null;
+}
+
+/**
+ * modify the json payload to add the uid of the root and the nested entities when they exist in Dgraph
+ * for non existing entities, a blank uid is generated helping the interpretation of mutation response
+ */
+
+export function injectNodeUid(connection:string, payload: string, nested_entities: NestedEntity[]): string {
+
+    const root_entity = nested_entities[0];
+
+    payload = payload.replace("{", `{ \"dgraph.type\":\"${root_entity.type}\",`)
+
+    for (var i = 1; i < nested_entities.length; i++) {
+        const predicate = nested_entities[i].predicate;
+        const type = nested_entities[i].type;
+        payload = payload.replace(`${predicate}\":{`, `${predicate}\":{ \"dgraph.type\":\"${type}\",`);
+    }
+
+    for ( i = 0; i < nested_entities.length; i++) {
+        var locator = `${nested_entities[i].predicate}\":{`
+        if (i == 0) { 
+            locator = "{"
+        } 
+        if(nested_entities[i].id_value != null) {
+            const nodeUid = getEntityUid(connection,`${nested_entities[i].type}.${nested_entities[i].id_field}`, nested_entities[i].id_value!);
+            if (nodeUid != null) {
+
+                payload = payload.replace(`${locator}`, `${locator} "uid":"${nodeUid}",`)
+            } else {
+                payload = payload.replace(`${locator}`, `${locator} "uid": "_:${nested_entities[i].type}-${nested_entities[i].id_value!}",`)
+            }
+        } 
+    }
+  return payload
+}
+
+export function getEntityById<T>(connection: string, predicate: string, id: string, body: string): T | null{
+    const query = new dgraph.Query(`{
+      list(func: eq(${predicate}, "${id}")) {
+          ${body}
+        }
+      }`);
+    const response = dgraph.execute(connection, new dgraph.Request(query));
+    const data = JSON.parse<ListOf<T>>(response.Json);
+    if (data.list.length > 0) {
+      return data.list[0];
+    } 
+    return null;
+  }
+
+function getEntityUid(connection: string,predicate:string, value: string): string | null{
+
+    const query = new dgraph.Query(`{uids(func: eq(${predicate}, "${value}")) {uid}}`);
+    const response = dgraph.execute(connection, new dgraph.Request(query ))
+    const data = JSON.parse<UidResult>(response.Json)
+    if (data.uids.length == 0) {
+      return null
+    }
+    console.log(`${predicate} Uid: ${data.uids[0].uid}`)
+    return data.uids[0].uid
+  }
+
+  export function deleteNodePredicates(connection:string, criteria: string, predicates: string[]): void {
+
+    const query = new dgraph.Query(`{
+        node as var(func: ${criteria}) 
+    }`);
+    predicates.push("dgraph.type");
+    const del_nquads = predicates.map<string>((predicate) => `uid(node) <${predicate}> * .`).join("\n");
+    const mutation = new dgraph.Mutation("","","",del_nquads);
+
+    dgraph.execute(connection, new dgraph.Request(query, [mutation]));
+
+}
+
+export function searchBySimilarity<T>(connection:string, embedding: f32[],predicate:string, body:string, topK: i32): T[]{
+
+    const query = `
+    query search($vector: float32vector) {
+        var(func: similar_to(${predicate},${topK},$vector))  {    
+            vemb as Product.embedding 
+            dist as math((vemb - $vector) dot (vemb - $vector))
+            score as math(1 - (dist / 2.0))
+        } 
+
+        list(func:uid(score),orderdesc:val(score))  @filter(gt(val(score),0.25)){ 
+            ${body}
+        }
+    }`
+    const vars = new dgraph.Variables();
+    vars.set("$vector", JSON.stringify(embedding));
+
+    const dgraph_query = new dgraph.Query(query,vars);
+
+    const response = dgraph.execute(connection, new dgraph.Request(dgraph_query));
+    console.log(response.Json)
+    return JSON.parse<ListOf<T>>(response.Json).list
+  }
+
+
+

dgraph-101/api-as/assembly/embeddings.ts

@@ -0,0 +1,12 @@
+import { models } from "@hypermode/modus-sdk-as";
+import { EmbeddingsModel } from "@hypermode/modus-sdk-as/models/experimental/embeddings";
+
+const EMBEDDING_MODEL = "minilm";
+
+
+export function embedText(content: string[]): f32[][] {
+  const model = models.getModel<EmbeddingsModel>(EMBEDDING_MODEL);
+  const input = model.createInput(content);
+  const output = model.invoke(input);
+  return output.predictions;
+}

dgraph-101/api-as/assembly/index.ts

@@ -0,0 +1,91 @@
+import { JSON } from "json-as"
+import { dgraph } from "@hypermode/modus-sdk-as"
+import { Product } from "./classes"
+import { embedText } from "./embeddings"
+import { buildProductMutationJson } from "./product-helpers"
+import { deleteNodePredicates, ListOf, searchBySimilarity, getEntityById } from "./dgraph-utils"
+
+const DGRAPH_CONNECTION="dgraph-grpc"
+
+/**
+ * Add or update a new product to the database
+ */
+export function upsertProduct(product: Product): Map<string, string> | null {
+
+  var payload = buildProductMutationJson(DGRAPH_CONNECTION,product);
+
+  const embedding = embedText([product.description]);
+  payload = payload.replace("{", `{ \"Product.embedding\":\"${JSON.stringify(embedding[0])}\",`)
+
+  const mutations: dgraph.Mutation[] = [new dgraph.Mutation(payload)];
+  const uids = dgraph.execute(DGRAPH_CONNECTION, new dgraph.Request(null, mutations)).Uids;
+
+  return uids;
+}
+
+/**
+ * Get a product info by its id
+ */
+export function getProduct(id: string): Product | null{
+  const body = `
+    Product.id
+    Product.description
+    Product.title
+    Product.category {
+      Category.name
+    }`
+  return getEntityById<Product>(DGRAPH_CONNECTION, "Product.id", id, body);
+}
+/** 
+ * Delete a product by its id
+ */
+
+export function deleteProduct(id: string): void {
+  deleteNodePredicates(DGRAPH_CONNECTION, `eq(Product.id, "${id}")`, ["Product.id", "Product.description", "Product.category"]);
+}
+
+/**
+ * Get all products of a given category
+ */
+export function getProductsByCategory(category: string): Product[] {
+  const query = new dgraph.Query(`{
+    list(func: eq(Category.name, "${category}")) {
+      list:~Product.category {
+        Product.id
+        Product.description
+        Product.category {
+          Category.name
+        }
+      }
+    }
+  }`);
+  const response = dgraph.execute(DGRAPH_CONNECTION, new dgraph.Request(query));
+  const data = JSON.parse<ListOf<ListOf<Product>>>(response.Json);
+  if (data.list.length > 0) {
+    return data.list[0].list;
+  } 
+  return [];
+}
+
+
+/**
+ * Search products by similarity to a given text
+ */
+export function searchProducts(search: string): Product[]{
+  const embedding = embedText([search])[0];
+  const topK = 3;
+  const body = `
+    Product.id
+    Product.description
+    Product.title
+    Product.category {
+      Category.name
+    }
+  `
+  return searchBySimilarity<Product>(DGRAPH_CONNECTION,embedding,"Product.embedding",body, topK);
+
+}
+
+
+
+

dgraph-101/api-as/assembly/product-helpers.ts

@@ -0,0 +1,30 @@
+/**
+ * Helper functions for products classes.
+ */
+import { JSON } from "json-as"
+import { Product } from "./classes"
+import { injectNodeUid, NestedEntity   } from "./dgraph-utils"
+
+export function buildProductMutationJson(connection:string, product: Product): string {
+    var payload = JSON.stringify(product);
+    const nested_entities: NestedEntity[] = [
+      { 
+        predicate: "",
+        type: "Product",
+        id_field: "id",
+        id_value: product.id
+      }
+    ]
+    if (product.category != null) {
+      nested_entities.push({
+        predicate: "category",
+        type: "Category",
+        id_field: "name",
+        id_value: product.category!.name
+      });
+    }
+    payload = injectNodeUid(connection,payload, nested_entities);
+
+    return payload;
+
+  }

dgraph-101/api-as/assembly/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "assemblyscript/std/assembly.json",
+  "include": ["./**/*.ts"]
+}

dgraph-101/api-as/eslint.config.js

@@ -0,0 +1,11 @@
+// @ts-check
+
+import eslint from "@eslint/js";
+import tseslint from "typescript-eslint";
+import aseslint from "@hypermode/modus-sdk-as/tools/assemblyscript-eslint";
+
+export default tseslint.config(
+  eslint.configs.recommended,
+  ...tseslint.configs.recommended,
+  aseslint.config,
+);