From a8380cda356ce011e5239c664ee6fe3ca96ceae1 Mon Sep 17 00:00:00 2001
From: huangjien <huangjien@gmail.com>
Date: Thu, 7 Nov 2024 18:53:40 +0000
Subject: [PATCH] add vision qa

---
 package.json              | 14 +++++++++++-
 src/JenkinsPanel.ts       |  2 +-
 src/extension.ts          | 47 ++++++++++++++++++++++++++++++++++++---
 src/getInfoFromJenkins.ts | 21 +++++++++++++++++
 4 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index e5c93f9..741220a 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "jenkins-log-reader",
   "displayName": "AI Log Reader",
   "description": "Read jenkins log, analyse with local AI.",
-  "version": "0.8.7",
+  "version": "0.8.8",
   "engines": {
     "vscode": "^1.95.0"
   },
@@ -115,6 +115,13 @@
       }
     ],
     "menus": {
+      "explorer/context": [
+        {
+          "when": "resourceExtname =~ /\\.(png|jpg|jpeg|gif|bmp|svg)$/i",
+          "command": "jenkins-log-reader.readImage",
+          "group": "navigation"
+        }
+      ],
       "editor/context": [
         {
           "when": "editorTextFocus && (editorLangId == groovy || editorLangId == jenkinsfile )",
@@ -124,6 +131,10 @@
       ]
     },
     "commands": [
+      {
+        "command": "jenkins-log-reader.readImage",
+        "title": "Analyse Image"
+      },
       {
         "command": "jenkins-log-reader.showResult",
         "title": "Show Analysis Result"
@@ -213,6 +224,7 @@
     "axios": "^1.7.7",
     "crypto": "^1.0.1",
     "marked": "^14.1.3",
+    "ollama": "^0.5.9",
     "openai": "^4.71.1",
     "tailwindcss": "^3.4.14"
   },
diff --git a/src/JenkinsPanel.ts b/src/JenkinsPanel.ts
index d9c2790..3d3839d 100644
--- a/src/JenkinsPanel.ts
+++ b/src/JenkinsPanel.ts
@@ -359,7 +359,7 @@ export class JenkinsPanel {
   }
 }
 
-function showStatusBarProgress(task: Promise<any>, title = "Processing...") {
+export function showStatusBarProgress(task: Promise<any>, title = "Processing...") {
   window.withProgress(
     {
       location: ProgressLocation.Notification,
diff --git a/src/extension.ts b/src/extension.ts
index 20b9ba1..1a1cac7 100644
--- a/src/extension.ts
+++ b/src/extension.ts
@@ -1,10 +1,12 @@
-import { ExtensionContext, window, commands, workspace, Range } from "vscode";
-import { JenkinsPanel } from "./JenkinsPanel";
+import { ExtensionContext, window, commands, workspace, Range, Uri } from "vscode";
+import { JenkinsPanel, showStatusBarProgress } from "./JenkinsPanel";
 import JenkinsSettings from "./JenkinsSettings";
-import { existsSync, mkdirSync } from "fs";
+import { existsSync, fstat, mkdirSync } from "fs";
 import { LogReaderResultWebViewProvider } from "./LogReaderResultWebViewProvider";
 import { LogReaderSettingWebViewProvider } from "./LogReaderSettingWebViewProvider";
 import { GroovyCodeFormat } from "./GroovyFormat";
+import * as fs from "fs";
+import { getAnalysis, getImageAnalysis } from "./getInfoFromJenkins";
 
 export function activate(context: ExtensionContext) {
   const storagePath = context.globalStorageUri.fsPath;
@@ -67,6 +69,7 @@ export function activate(context: ExtensionContext) {
   const resultViewProvider = setupResultWebviewProvider(context);
 
   registerCommandOfShowResult(context, resultViewProvider);
+  registerCommandOfReadImage(context, resultViewProvider);
   registerCommandOfFormatGrooby(context);
 }
 
@@ -89,6 +92,42 @@ function registerCommandOfFormatGrooby(context: ExtensionContext) {
   );
 }
 
+function registerCommandOfReadImage(context: ExtensionContext,
+  provider: LogReaderResultWebViewProvider) {
+  context.subscriptions.push(
+    commands.registerCommand("jenkins-log-reader.readImage", async (uri: Uri) => {
+      // get image file
+      // turn it into base64
+      // send to AI (llama3.2-vision)
+      // show result in result view
+      if (uri){
+
+      
+        const image_uri = uri
+
+        const base64String = fs.readFileSync(image_uri.fsPath).toString('base64')
+        const long_run_task = analyse_image(base64String, provider);
+        showStatusBarProgress(long_run_task, 'Analysing the image...')
+        
+      }
+    })
+  );
+}
+
+async function analyse_image(base64String: string, provider: LogReaderResultWebViewProvider) {
+  await getImageAnalysis("llama3.2-vision",
+    'I am a QA, testing a product, which name is Curam. Please help me to check the snapshot, this is a functional testing. Do you see anything wrong or something behave weird in it?',
+    base64String).then((information: string) => {
+      if (provider._view) {
+        // commands.executeCommand("jenkins-log-reader_result-view.focus");
+        if (information) {
+          provider.updateContent(information);
+          commands.executeCommand("jenkins-log-reader_result-view.focus");
+        }
+      }
+    });
+}
+
 function registerCommandOfShowResult(
   context: ExtensionContext,
   provider: LogReaderResultWebViewProvider
@@ -125,3 +164,5 @@ function setupSettingsViewProvider(context: ExtensionContext) {
 function getConfig(config_key: string): any {
   return workspace.getConfiguration().get(config_key);
 }
+
+
diff --git a/src/getInfoFromJenkins.ts b/src/getInfoFromJenkins.ts
index ea2f172..7b6aaf4 100644
--- a/src/getInfoFromJenkins.ts
+++ b/src/getInfoFromJenkins.ts
@@ -3,6 +3,7 @@ import OpenAI from "openai";
 import * as fs from "fs";
 import { createHash } from "crypto";
 import { JenkinsPanel } from "./JenkinsPanel";
+import ollama from "ollama";
 
 type Build = {
   url: string;
@@ -199,6 +200,26 @@ export async function getAnalysis(
     });
 }
 
+export async function getImageAnalysis(
+  model: string,
+  prompt: string,
+  data: string
+) {
+  
+  return await ollama.chat({
+      model: model,
+      messages: [{ role: "user", content: prompt, images: [data] }],
+      stream: false,
+    })
+    .then((ret) => {
+      const information = ret["message"]["content"];
+      return information;
+    })
+    .catch((err) => {
+      throw err;
+    });
+}
+
 export function digest(message: string) {
   return createHash("sha1")
     .update(message.replace(/\s/g, "").replace(" ", ""), "utf8")