add vision qa

package.json

@@ -2,7 +2,7 @@
   "name": "jenkins-log-reader",
   "displayName": "AI Log Reader",
   "description": "Read jenkins log, analyse with local AI.",
-  "version": "0.8.7",
+  "version": "0.8.8",
   "engines": {
     "vscode": "^1.95.0"
   },
@@ -115,6 +115,13 @@
       }
     ],
     "menus": {
+      "explorer/context": [
+        {
+          "when": "resourceExtname =~ /\\.(png|jpg|jpeg|gif|bmp|svg)$/i",
+          "command": "jenkins-log-reader.readImage",
+          "group": "navigation"
+        }
+      ],
       "editor/context": [
         {
           "when": "editorTextFocus && (editorLangId == groovy || editorLangId == jenkinsfile )",
@@ -124,6 +131,10 @@
       ]
     },
     "commands": [
+      {
+        "command": "jenkins-log-reader.readImage",
+        "title": "Analyse Image"
+      },
       {
         "command": "jenkins-log-reader.showResult",
         "title": "Show Analysis Result"
@@ -213,6 +224,7 @@
     "axios": "^1.7.7",
     "crypto": "^1.0.1",
     "marked": "^14.1.3",
+    "ollama": "^0.5.9",
     "openai": "^4.71.1",
     "tailwindcss": "^3.4.14"
   },

src/JenkinsPanel.ts

@@ -359,7 +359,7 @@ export class JenkinsPanel {
   }
 }
 
-function showStatusBarProgress(task: Promise<any>, title = "Processing...") {
+export function showStatusBarProgress(task: Promise<any>, title = "Processing...") {
   window.withProgress(
     {
       location: ProgressLocation.Notification,

src/extension.ts

@@ -1,10 +1,12 @@
-import { ExtensionContext, window, commands, workspace, Range } from "vscode";
-import { JenkinsPanel } from "./JenkinsPanel";
+import { ExtensionContext, window, commands, workspace, Range, Uri } from "vscode";
+import { JenkinsPanel, showStatusBarProgress } from "./JenkinsPanel";
 import JenkinsSettings from "./JenkinsSettings";
-import { existsSync, mkdirSync } from "fs";
+import { existsSync, fstat, mkdirSync } from "fs";
 import { LogReaderResultWebViewProvider } from "./LogReaderResultWebViewProvider";
 import { LogReaderSettingWebViewProvider } from "./LogReaderSettingWebViewProvider";
 import { GroovyCodeFormat } from "./GroovyFormat";
+import * as fs from "fs";
+import { getAnalysis, getImageAnalysis } from "./getInfoFromJenkins";
 
 export function activate(context: ExtensionContext) {
   const storagePath = context.globalStorageUri.fsPath;
@@ -67,6 +69,7 @@ export function activate(context: ExtensionContext) {
   const resultViewProvider = setupResultWebviewProvider(context);
 
   registerCommandOfShowResult(context, resultViewProvider);
+  registerCommandOfReadImage(context, resultViewProvider);
   registerCommandOfFormatGrooby(context);
 }
 
@@ -89,6 +92,42 @@ function registerCommandOfFormatGrooby(context: ExtensionContext) {
   );
 }
 
+function registerCommandOfReadImage(context: ExtensionContext,
+  provider: LogReaderResultWebViewProvider) {
+  context.subscriptions.push(
+    commands.registerCommand("jenkins-log-reader.readImage", async (uri: Uri) => {
+      // get image file
+      // turn it into base64
+      // send to AI (llama3.2-vision)
+      // show result in result view
+      if (uri){
+
+
+        const image_uri = uri
+
+        const base64String = fs.readFileSync(image_uri.fsPath).toString('base64')
+        const long_run_task = analyse_image(base64String, provider);
+        showStatusBarProgress(long_run_task, 'Analysing the image...')
+
+      }
+    })
+  );
+}
+
+async function analyse_image(base64String: string, provider: LogReaderResultWebViewProvider) {
+  await getImageAnalysis("llama3.2-vision",
+    'I am a QA, testing a product, which name is Curam. Please help me to check the snapshot, this is a functional testing. Do you see anything wrong or something behave weird in it?',
+    base64String).then((information: string) => {
+      if (provider._view) {
+        // commands.executeCommand("jenkins-log-reader_result-view.focus");
+        if (information) {
+          provider.updateContent(information);
+          commands.executeCommand("jenkins-log-reader_result-view.focus");
+        }
+      }
+    });
+}
+
 function registerCommandOfShowResult(
   context: ExtensionContext,
   provider: LogReaderResultWebViewProvider
@@ -125,3 +164,5 @@ function setupSettingsViewProvider(context: ExtensionContext) {
 function getConfig(config_key: string): any {
   return workspace.getConfiguration().get(config_key);
 }
+
+

src/getInfoFromJenkins.ts

@@ -3,6 +3,7 @@ import OpenAI from "openai";
 import * as fs from "fs";
 import { createHash } from "crypto";
 import { JenkinsPanel } from "./JenkinsPanel";
+import ollama from "ollama";
 
 type Build = {
   url: string;
@@ -199,6 +200,26 @@ export async function getAnalysis(
     });
 }
 
+export async function getImageAnalysis(
+  model: string,
+  prompt: string,
+  data: string
+) {
+
+  return await ollama.chat({
+      model: model,
+      messages: [{ role: "user", content: prompt, images: [data] }],
+      stream: false,
+    })
+    .then((ret) => {
+      const information = ret["message"]["content"];
+      return information;
+    })
+    .catch((err) => {
+      throw err;
+    });
+}
+
 export function digest(message: string) {
   return createHash("sha1")
     .update(message.replace(/\s/g, "").replace("　", ""), "utf8")