Working with Dgraph in Modus

README.md

@@ -16,3 +16,4 @@ The following recipes have associated recorded content:
 | [modus-getting-started](modus-getting-started/) | [Getting Started With Modus video](https://www.youtube.com/watch?v=3CcJTXTmz88)                                                                    |
 | [modushack-data-models](modushack-data-models/) | [ModusHack: Working With Data & AI Models livestream](https://www.youtube.com/watch?v=gB-v7YWwkCw&list=PLzOEKEHv-5e3zgRGzDysyUm8KQklHQQgi&index=3) |
 | [modus-press](modus-press/)                     | Coming soon                                                                                                                                        |
+| [dgraph-101](dgraph-101/)                     | Coming soon                                                                                                                                        |

dgraph-101/README.md

@@ -0,0 +1,44 @@
+# dgraph with modus 101
+
+From ModusHack livestream - 11/19/24  .
+
+## Dgraph
+
+Start a local instance using Docker.
+
+```sh
+   docker run --name dgraph-101 -d -p "8080:8080" -p "9080:9080" -v ~/dgraph-101:/dgraph dgraph/standalone:latest
+```
+
+Start Ratel, a Graphical data explorer for Dgraph:
+```sh
+docker run --name ratel  -d -p "8000:8000"  dgraph/ratel:latest
+```
+
+## Data model
+
+Modus is code first, simply define you data using classes.
+In this example we are using `Product` and `Category` defined in `classes.ts` file.
+
+## Define your API.
+
+index.ts is where we export the functions that are exposed as GraphQL operations.
+
+This project defines the following operations:
+
+- upsertProduct
+- getProduct
+- deleteProduct
+
+Note that Modus is exposing `getProduct` as `product` to follow common coding and GraphQL practices.
+Modus also exposes `upsertProduct` and `deleteProduct` as a GraphQL Mutation automatically.
+
+- getProductsByCategory
+The data is saved in Dgraph as a graph.
+This function shows how to easily use the relationships in the graph.
+
+- searchProducts
+Finally we are using an AI model to create text embeddings for our Products: as the embeddings are stored in Dgraph for each Product entity, we can easily expose an API to search by natural language and similarity, leveraging DQL `similar_to` function.
+
+
+

dgraph-101/api-as/.gitignore

@@ -0,0 +1,15 @@
+# Ignore macOS system files
+.DS_Store
+
+# Ignore environment variable files
+.env
+.env.*
+
+# Ignore build output directories
+build/
+
+# Ignore node_modules folders
+node_modules/
+
+# Ignore logs generated by as-test
+logs/

dgraph-101/api-as/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "plugins": ["assemblyscript-prettier"]
+}

dgraph-101/api-as/asconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "./node_modules/@hypermode/modus-sdk-as/plugin.asconfig.json",
+  "options": {
+    "transform": ["@hypermode/modus-sdk-as/transform", "json-as/transform"]
+  }
+}

dgraph-101/api-as/assembly/classes.ts

@@ -0,0 +1,31 @@
+
+/* 
+    This file contains the classes that are used in our App.
+    The classes are annotated with the @json decorator 
+    to be serialized and deserialized as json string.
+    @alias is used to rename the properties in the json string to match Dgraph best practices.
+*/
+
+
+@json
+export class Product {
+
+  @alias("Product.id")
+  id!: string;
+
+  @alias("Product.title")
+  title: string = "";
+
+  @alias("Product.description")
+  description: string = "";
+
+  @alias("Product.category")
+  @omitnull()
+  category: Category | null = null;
+}
+
+@json
+export class Category {
+  @alias("Category.name")
+  name: string = "";
+}

dgraph-101/api-as/assembly/dgraph-utils.ts

@@ -0,0 +1,148 @@
+/**
+ * Utility classes and functions used to interact with Dgraph.
+ */
+import { dgraph } from "@hypermode/modus-sdk-as"
+import { JSON } from "json-as"
+import { JSON as JSON_TREE} from "assemblyscript-json/assembly/index"
+
+@json
+class Uid {
+  uid: string = "";
+}
+@json 
+class UidResult {
+  uids: Uid[] = [];
+}
+@json 
+export class ListOf<T> {
+    list: T[] = [];
+}
+export class Relationship {
+    predicate!: string;
+    type!: string;
+}
+export class NodeType {
+    id_field: string = ""
+    relationships : Relationship[] = [];
+}
+export class GraphSchema {
+    node_types: Map<string, NodeType> = new Map<string, NodeType>();
+}
+
+
+/**
+ * modify the json payload to add the uid of the root and the nested entities when they exist in Dgraph
+ * for non existing entities, a blank uid is generated helping the interpretation of mutation response
+ */
+
+export function injectNodeUid(connection:string, payload: string, root_type:string, schema: GraphSchema): string {
+
+    const root_node_type = schema.node_types.get(root_type);
+
+    const root = <JSON_TREE.Obj>(JSON_TREE.parse(payload));
+    injectNodeType(connection, root, root_type, schema);
+
+    console.log(root.toString())
+
+  return root.toString()
+}
+
+/**
+ * recursively inject the dgraph.type and uid of the entities in the json tree
+ */
+function injectNodeType(connection: string, entity: JSON_TREE.Obj | null, type: string, schema: GraphSchema): void {
+
+    if (entity != null) {
+      entity.set("dgraph.type", type);
+      const node_type = schema.node_types.get(type);
+      for (var i = 0; i < node_type.relationships.length; i++) {
+        const predicate = node_type.relationships[i].predicate;
+        const type = node_type.relationships[i].type;
+        injectNodeType(connection, entity.getObj(predicate), type, schema);
+      }
+
+      const id_field = entity.getString(node_type.id_field)
+      if (id_field != null) {
+        const id_value = entity.getString(node_type.id_field)!.toString();
+        if (id_value != null) {
+          const node_uid = getEntityUid(connection,`${node_type.id_field}`, id_value);
+          if (node_uid != null) {
+            entity.set("uid", node_uid);
+          } else {
+            entity.set("uid", `_:${type}-${id_value}`);
+          }
+        }
+      }
+    }
+}
+
+export function getEntityById<T>(connection: string, predicate: string, id: string, body: string): T | null{
+    const query = new dgraph.Query(`{
+      list(func: eq(${predicate}, "${id}")) {
+          ${body}
+        }
+      }`);
+    const response = dgraph.execute(connection, new dgraph.Request(query));
+    const data = JSON.parse<ListOf<T>>(response.Json);
+    if (data.list.length > 0) {
+      return data.list[0];
+    } 
+    return null;
+  }
+
+function getEntityUid(connection: string,predicate:string, value: string): string | null{
+
+    const query = new dgraph.Query(`{uids(func: eq(${predicate}, "${value}")) {uid}}`);
+    const response = dgraph.execute(connection, new dgraph.Request(query ))
+    const data = JSON.parse<UidResult>(response.Json)
+    if (data.uids.length == 0) {
+      return null
+    }
+    console.log(`${predicate} Uid: ${data.uids[0].uid}`)
+    return data.uids[0].uid
+  }
+
+  export function deleteNodePredicates(connection:string, criteria: string, predicates: string[]): void {
+
+    const query = new dgraph.Query(`{
+        node as var(func: ${criteria}) 
+    }`);
+    predicates.push("dgraph.type");
+    const del_nquads = predicates.map<string>((predicate) => `uid(node) <${predicate}> * .`).join("\n");
+    const mutation = new dgraph.Mutation("","","",del_nquads);
+
+    dgraph.execute(connection, new dgraph.Request(query, [mutation]));
+
+}
+
+export function searchBySimilarity<T>(connection:string, embedding: f32[],predicate:string, body:string, topK: i32): T[]{
+
+    const query = `
+    query search($vector: float32vector) {
+        var(func: similar_to(${predicate},${topK},$vector))  {    
+            vemb as Product.embedding 
+            dist as math((vemb - $vector) dot (vemb - $vector))
+            score as math(1 - (dist / 2.0))
+        } 
+
+        list(func:uid(score),orderdesc:val(score))  @filter(gt(val(score),0.25)){ 
+            ${body}
+        }
+    }`
+    const vars = new dgraph.Variables();
+    vars.set("$vector", JSON.stringify(embedding));
+
+    const dgraph_query = new dgraph.Query(query,vars);
+
+    const response = dgraph.execute(connection, new dgraph.Request(dgraph_query));
+    console.log(response.Json)
+    return JSON.parse<ListOf<T>>(response.Json).list
+  }
+
+
+  export function addEmbeddingToJson(payload: string, predicate:string, embedding: f32[]): string {
+    // Add the embedding to the payload at root level
+    // TO DO: extend to nested entities and use JSONpath
+    payload = payload.replace("{", `{ \"${predicate}\":\"${JSON.stringify(embedding)}\",`)
+    return payload;
+  }

dgraph-101/api-as/assembly/embeddings.ts

@@ -0,0 +1,12 @@
+import { models } from "@hypermode/modus-sdk-as";
+import { EmbeddingsModel } from "@hypermode/modus-sdk-as/models/experimental/embeddings";
+
+const EMBEDDING_MODEL = "minilm";
+
+
+export function embedText(content: string[]): f32[][] {
+  const model = models.getModel<EmbeddingsModel>(EMBEDDING_MODEL);
+  const input = model.createInput(content);
+  const output = model.invoke(input);
+  return output.predictions;
+}

dgraph-101/api-as/assembly/index.ts

@@ -0,0 +1,92 @@
+import { JSON } from "json-as"
+import { dgraph } from "@hypermode/modus-sdk-as"
+import { Product } from "./classes"
+import { embedText } from "./embeddings"
+import { buildProductMutationJson } from "./product-helpers"
+import { deleteNodePredicates, ListOf, searchBySimilarity, getEntityById , addEmbeddingToJson} from "./dgraph-utils"
+
+const DGRAPH_CONNECTION="dgraph-grpc"
+
+/**
+ * Add or update a new product to the database
+ */
+export function upsertProduct(product: Product): Map<string, string> | null {
+
+  var payload = buildProductMutationJson(DGRAPH_CONNECTION,product);
+
+  const embedding = embedText([product.description])[0];
+  payload = addEmbeddingToJson(payload, "Product.embedding",embedding);
+
+  const mutations: dgraph.Mutation[] = [new dgraph.Mutation(payload)];
+  const uids = dgraph.execute(DGRAPH_CONNECTION, new dgraph.Request(null, mutations)).Uids;
+
+  return uids;
+}
+
+/**
+ * Get a product info by its id
+ */
+export function getProduct(id: string): Product | null{
+  const body = `
+    Product.id
+    Product.description
+    Product.title
+    Product.category {
+      Category.name
+    }`
+  return getEntityById<Product>(DGRAPH_CONNECTION, "Product.id", id, body);
+}
+/** 
+ * Delete a product by its id
+ */
+
+export function deleteProduct(id: string): void {
+  deleteNodePredicates(DGRAPH_CONNECTION, `eq(Product.id, "${id}")`, ["Product.id", "Product.description", "Product.category"]);
+}
+
+/**
+ * Get all products of a given category
+ */
+export function getProductsByCategory(category: string): Product[] {
+  const query = new dgraph.Query(`{
+    list(func: eq(Category.name, "${category}")) {
+      list:~Product.category {
+        Product.id
+        Product.title
+        Product.description
+        Product.category {
+          Category.name
+        }
+      }
+    }
+  }`);
+  const response = dgraph.execute(DGRAPH_CONNECTION, new dgraph.Request(query));
+  const data = JSON.parse<ListOf<ListOf<Product>>>(response.Json);
+  if (data.list.length > 0) {
+    return data.list[0].list;
+  } 
+  return [];
+}
+
+
+/**
+ * Search products by similarity to a given text
+ */
+export function searchProducts(search: string): Product[]{
+  const embedding = embedText([search])[0];
+  const topK = 3;
+  const body = `
+    Product.id
+    Product.description
+    Product.title
+    Product.category {
+      Category.name
+    }
+  `
+  return searchBySimilarity<Product>(DGRAPH_CONNECTION,embedding,"Product.embedding",body, topK);
+
+}
+
+
+
+

dgraph-101/api-as/assembly/product-helpers.ts

@@ -0,0 +1,28 @@
+/**
+ * Helper functions for products classes.
+ */
+import { JSON } from "json-as"
+import { Product } from "./classes"
+import { injectNodeUid,  GraphSchema  } from "./dgraph-utils"
+
+const product_schema : GraphSchema = new GraphSchema();
+
+product_schema.node_types.set("Product", {
+  id_field: "Product.id",
+  relationships: [
+    {predicate: "Product.category", type: "Category"}
+  ]
+});
+product_schema.node_types.set("Category", {
+  id_field: "Category.name",
+  relationships: []
+});
+
+export function buildProductMutationJson(connection:string, product: Product): string {
+    var payload = JSON.stringify(product);
+
+    payload = injectNodeUid(connection,payload, "Product", product_schema);
+
+    return payload;
+
+  }