Merge pull request #18 from reworkd/rohan

Tagging improvements

Author: awtkns

README.md

@@ -84,6 +84,12 @@ if __name__ == '__main__':
     asyncio.run(main())
 ```
 
+Keep in mind that Tarsier tags different types of elements differently to help your LLM identify what actions are performable on each element. Specifically:
+- `[#ID]`: text-insertable fields (e.g. `textarea`, `input` with textual type)
+- `[@ID]`: hyperlinks (`<a>` tags)
+- `[$ID]`: other interactable elements (e.g. `button`, `select`)
+- `[ID]`: plain text (if you pass `tag_text_elements=True`)
+
 ## Local Development
 
 ### Setup

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tarsier"
-version = "0.4.4"
+version = "0.5.0"
 description = "Vision utilities for web interaction agents"
 authors = ["Rohan Pandey", "Adam Watkins", "Asim Shrestha"]
 readme = "README.md"

tarsier/tag_utils.ts

@@ -22,8 +22,11 @@ const isInteractable = (el: HTMLElement) =>
   (el.tagName.toLowerCase() === "input" && el.type !== "hidden") ||
   el.role === "button";
 
+const text_input_types = ["text", "password", "email", "search", "url", "tel", "number"];
 const isTextInsertable = (el: HTMLElement) =>
-  (["input", "textarea"].includes(el.tagName.toLowerCase()));
+  el.tagName.toLowerCase() === "textarea" ||
+  ((el.tagName.toLowerCase() === "input" &&
+    text_input_types.includes((el as HTMLInputElement).type)));
 
 const emptyTagWhitelist = ["input", "textarea", "select", "button"];
 const isEmpty = (el: HTMLElement) => {
@@ -107,14 +110,27 @@ function getElementXPath(element: HTMLElement | null) {
   return iframe_str + "//" + path_parts.join("/");
 }
 
-function create_tagged_span(idStr: string) {
+function create_tagged_span(idNum: number, el: HTMLElement) {
+  let idStr: string;
+  if (isInteractable(el)) {
+    if (isTextInsertable(el))
+      idStr = `[#${idNum}]`;
+    else if (el.tagName.toLowerCase() == 'a')
+      idStr = `[@${idNum}]`;
+    else
+      idStr = `[$${idNum}]`;
+  } else {
+    idStr = `[${idNum}]`;
+  }
+
   let idSpan = document.createElement("span");
   idSpan.id = "__tarsier_id";
   idSpan.style.all = "inherit";
   idSpan.style.display = "inline";
   idSpan.style.color = "white";
   idSpan.style.backgroundColor = "red";
   idSpan.textContent = idStr;
+  
   return idSpan;
 }
 
@@ -183,17 +199,20 @@ window.tagifyWebpage = (tagLeafTexts = false) => {
       continue;
     }
 
-    const idStr = isTextInsertable(el) ? `{${idNum}}` : `[${idNum}]`;
-    let idSpan = create_tagged_span(idStr);
+    let idSpan = create_tagged_span(idNum, el);
 
     if (isInteractable(el)) {
-      el.prepend(idSpan);
+      if (isTextInsertable(el) && el.parentElement) {
+        el.parentElement.insertBefore(idSpan, el);
+      } else {
+        el.prepend(idSpan);
+      }
       idNum++;
     } else if (tagLeafTexts) {
       for (let child of Array.from(el.childNodes)) {
         if (child.nodeType === Node.TEXT_NODE && /\S/.test(child.textContent || "")) {
           // This is a text node with non-whitespace text
-          let idSpan = create_tagged_span(idStr);
+          let idSpan = create_tagged_span(idNum, el);
           el.insertBefore(idSpan, child);
           idNum++;
         }