From 1f91ad34fdb90e47d44f02405d9d82cd0e546c26 Mon Sep 17 00:00:00 2001 From: Usamazbr Date: Sun, 16 Jun 2024 00:24:43 +0500 Subject: [PATCH 1/2] Intoducing Image file upload (basic functionality in frontend with backend already fully compatible) --- frontend/package-lock.json | 14 +- frontend/package.json | 2 +- frontend/src/api/api.ts | 23 ++- frontend/src/api/models.ts | 1 + .../QuestionInput/QuestionInput.module.css | 82 +++++++++-- .../QuestionInput/QuestionInput.tsx | 138 ++++++++++++++---- frontend/src/pages/chat/Chat.module.css | 15 ++ frontend/src/pages/chat/Chat.tsx | 39 ++++- 8 files changed, 260 insertions(+), 54 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 60a3b70dd2..9a2afffbbf 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,7 +11,7 @@ "@fluentui/react": "^8.105.3", "@fluentui/react-hooks": "^8.6.29", "@fluentui/react-icons": "^2.0.195", - "dompurify": "^3.0.8", + "dompurify": "^3.1.5", "lodash": "^4.17.21", "lodash-es": "^4.17.21", "plotly.js": "^2.32.0", @@ -4698,9 +4698,9 @@ } }, "node_modules/dompurify": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.8.tgz", - "integrity": "sha512-b7uwreMYL2eZhrSCRC4ahLTeZcPZxSmYfmcQGXGkXiZSNW1X85v+SDM5KsWcpivIiUBH47Ji7NtyUdpLeF5JZQ==" + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz", + "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" }, "node_modules/draw-svg-path": { "version": "1.0.0", @@ -17619,9 +17619,9 @@ } }, "dompurify": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.8.tgz", - "integrity": "sha512-b7uwreMYL2eZhrSCRC4ahLTeZcPZxSmYfmcQGXGkXiZSNW1X85v+SDM5KsWcpivIiUBH47Ji7NtyUdpLeF5JZQ==" + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.1.5.tgz", + "integrity": "sha512-lwG+n5h8QNpxtyrJW/gJWckL+1/DQiYMX8f7t8Z2AZTPw1esVrqjI63i7Zc2Gz0aKzLVMYC1V1PL/ky+aY/NgA==" }, "draw-svg-path": { "version": "1.0.0", diff --git a/frontend/package.json b/frontend/package.json index adf63e3bc9..bf85057001 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,7 +18,7 @@ "@fluentui/react": "^8.105.3", "@fluentui/react-hooks": "^8.6.29", "@fluentui/react-icons": "^2.0.195", - "dompurify": "^3.0.8", + "dompurify": "^3.1.5", "lodash": "^4.17.21", "lodash-es": "^4.17.21", "plotly.js": "^2.32.0", diff --git a/frontend/src/api/api.ts b/frontend/src/api/api.ts index 97beb98544..76556387eb 100644 --- a/frontend/src/api/api.ts +++ b/frontend/src/api/api.ts @@ -3,13 +3,34 @@ import { chatHistorySampleData } from '../constants/chatHistory' import { ChatMessage, Conversation, ConversationRequest, CosmosDBHealth, CosmosDBStatus, UserInfo } from './models' export async function conversationApi(options: ConversationRequest, abortSignal: AbortSignal): Promise { + const transformedMessages = options.messages.map(item => { + if (item.image) { + return { + ...item, + content: [ + { + type: "text", + text: item.content + }, + { + type: "image_url", + image_url: { + url: item.image + } + } + ] + }; + } else { + return item; + } + }); const response = await fetch('/conversation', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - messages: options.messages + messages: transformedMessages }), signal: abortSignal }) diff --git a/frontend/src/api/models.ts b/frontend/src/api/models.ts index 2b1465cef0..7a108d409b 100644 --- a/frontend/src/api/models.ts +++ b/frontend/src/api/models.ts @@ -48,6 +48,7 @@ export type ChatMessage = { id: string role: string content: string + image?: string end_turn?: boolean date: string feedback?: Feedback diff --git a/frontend/src/components/QuestionInput/QuestionInput.module.css b/frontend/src/components/QuestionInput/QuestionInput.module.css index b9dc041e56..595fcfd524 100644 --- a/frontend/src/components/QuestionInput/QuestionInput.module.css +++ b/frontend/src/components/QuestionInput/QuestionInput.module.css @@ -10,21 +10,30 @@ 0px 8px 16px rgba(0, 0, 0, 0.14), 0px 0px 2px rgba(0, 0, 0, 0.12); border-radius: 8px; + display: flex; + flex-direction: row; + align-items: center; + padding: 10px; } .questionInputTextArea { - width: 100%; + flex-grow: 1; + width: calc(100% - 60px); line-height: 40px; - margin-top: 10px; + margin-right: 10px; +} + +.buttonContainer { + display: flex; + flex-direction: column; + justify-content: center; + align-items: flex-end; + margin-top: 50px; margin-bottom: 10px; - margin-left: 12px; - margin-right: 12px; } .questionInputSendButtonContainer { - position: absolute; - right: 24px; - bottom: 20px; + margin-top: 2px; } .questionInputSendButton { @@ -40,21 +49,70 @@ color: #424242; } +.imageInputSendButtonContainer { + margin-bottom: 10px; +} + +.imageInputSendButton { + width: 24px; + height: 23px; + margin-right: 1px; +} + +.imageInputSendButton:hover { + cursor: pointer; + color: #2782c2; + transform: scale(1.2); + transition: transform 0.3s ease-in-out; +} + .questionInputBottomBorder { position: absolute; width: 100%; height: 4px; - left: 0%; - bottom: 0%; + left: 0; + bottom: 0; background: radial-gradient(106.04% 106.06% at 100.1% 90.19%, #0f6cbd 33.63%, #8dddd8 100%); border-bottom-left-radius: 8px; border-bottom-right-radius: 8px; } -.questionInputOptionsButton { +.imageWrapper { + position: relative; + width: auto; + max-width: 100px; + max-height: 100px; + overflow: hidden; + /* background: #ffffff; */ + box-shadow: 0px 8px 16px rgba(0, 0, 0, 0.14), 0px 0px 2px rgba(0, 0, 0, 0.12); + border-radius: 8px; + margin-right: 10px; + margin-bottom: 0; + padding: 0px; +} + +.fileDisplay { + width: 100%; + height: 100%; /* Ensure the image covers the entire container */ + object-fit: contain; + display: block; /* Remove any inline spacing */ + margin: 0; /* Remove any margin */ + padding: 0; /* Remove any padding */ +} + +.clearIcon { + color: red; + position: absolute; + top: 2px; + right: 2px; cursor: pointer; - width: 27px; - height: 30px; + opacity: 0; + transition: opacity 0.3s; + z-index: 2; +} + +.clearIcon:hover { + opacity: 1; } @media (max-width: 480px) { diff --git a/frontend/src/components/QuestionInput/QuestionInput.tsx b/frontend/src/components/QuestionInput/QuestionInput.tsx index d75e543239..1ef46fdbbd 100644 --- a/frontend/src/components/QuestionInput/QuestionInput.tsx +++ b/frontend/src/components/QuestionInput/QuestionInput.tsx @@ -1,13 +1,13 @@ -import { useState } from 'react' +import { useRef, useState } from 'react' import { Stack, TextField } from '@fluentui/react' -import { SendRegular } from '@fluentui/react-icons' +import { AddSquareRegular, SendRegular, DismissCircleFilled, AddSquareFilled } from '@fluentui/react-icons' import Send from '../../assets/Send.svg' import styles from './QuestionInput.module.css' interface Props { - onSend: (question: string, id?: string) => void + onSend: (question: string, id?: string, filebase64?: string) => void disabled: boolean placeholder?: string clearOnSend?: boolean @@ -15,21 +15,41 @@ interface Props { } export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conversationId }: Props) => { - const [question, setQuestion] = useState('') + const fileInputRef = useRef(null); + const [question, setQuestion] = useState(' ') + const [file, setFile] = useState(undefined) + const [fileName, setFileName] = useState('') + const [filebase64, setFileBase64] = useState('') + const [isHovered, setIsHovered] = useState(false); const sendQuestion = () => { - if (disabled || !question.trim()) { + if (disabled || (!question.trim() && !filebase64)) { return } if (conversationId) { - onSend(question, conversationId) - } else { - onSend(question) + if (filebase64) { + onSend(question, conversationId, filebase64) + } + else { + onSend(question, conversationId, undefined) + + } + } + else { + if (filebase64) { + onSend(question, undefined, filebase64) + } + else { + onSend(question, undefined, undefined) + } } if (clearOnSend) { setQuestion('') + setFile(undefined) + setFileName('') + setFileBase64('') } } @@ -44,34 +64,102 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conv setQuestion(newValue || '') } - const sendQuestionDisabled = disabled || !question.trim() + const convertToBase64 = (file: File) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => { + if (reader.result) { + setFileBase64(reader.result.toString()); + } + } + reader.onerror = (error) => { + console.error("There was an error reading the file!", error) + } + } + + const onFileChange = (event: React.ChangeEvent) => { + const file = event.target.files?.[0] + if (file) { + setFile(file) + setFileName(file.name) + convertToBase64(file) + event.target.value = '' + // sendQuestion() + } + } + + const sendQuestionDisabled = disabled || (!question.trim() && !filebase64) + const handleClearImage = () => { + setFileBase64('') + }; + const handleMouseEnter = () => { + setIsHovered(true); + }; + + const handleMouseLeave = () => { + setIsHovered(false); + }; return ( - - + + + {filebase64 && ( +
+ Uploaded File + +
+ )} +
+
+ + {isHovered ? ( + { + e.stopPropagation(); + fileInputRef.current?.click(); + }} + /> + ) : ( + + )} +
(e.key === 'Enter' || e.key === ' ' ? sendQuestion() : null)}> + onKeyDown={(e) => (e.key === 'Enter' || e.key === ' ' ? sendQuestion() : null)} + > {sendQuestionDisabled ? ( ) : ( Send Button )}
-
- +
+
+ ) } diff --git a/frontend/src/pages/chat/Chat.module.css b/frontend/src/pages/chat/Chat.module.css index 775311e904..bb8c15e786 100644 --- a/frontend/src/pages/chat/Chat.module.css +++ b/frontend/src/pages/chat/Chat.module.css @@ -82,6 +82,16 @@ display: flex; justify-content: flex-end; margin-bottom: 12px; + gap: 5px; + align-items: flex-start; + width: 100%; +} + +.chatImage { + border-radius: 5px; + height: 100px; + width: auto; + flex-shrink: 0; } .chatMessageUserMessage { @@ -102,6 +112,11 @@ white-space: pre-wrap; word-wrap: break-word; max-width: 80%; + flex-shrink: 0; + max-width: calc(100% - 160px); + overflow: hidden; + text-overflow: ellipsis; + margin-left: 24px; } .chatMessageGpt { diff --git a/frontend/src/pages/chat/Chat.tsx b/frontend/src/pages/chat/Chat.tsx index f3222d0689..f0975d1def 100644 --- a/frontend/src/pages/chat/Chat.tsx +++ b/frontend/src/pages/chat/Chat.tsx @@ -160,18 +160,36 @@ const Chat = () => { } } - const makeApiRequestWithoutCosmosDB = async (question: string, conversationId?: string) => { + const makeApiRequestWithoutCosmosDB = async (question: string, conversationId?: string, filebase64?: string) => { setIsLoading(true) setShowLoadingMessage(true) const abortController = new AbortController() abortFuncs.current.unshift(abortController) - const userMessage: ChatMessage = { + let userMessage: ChatMessage + if (filebase64) { + userMessage = { + id: uuid(), + role: 'user', + content: question, + date: new Date().toISOString(), + image: filebase64 + } + } + else { + userMessage = { id: uuid(), role: 'user', content: question, date: new Date().toISOString() + } } + // const userMessage: ChatMessage = { + // id: uuid(), + // role: 'user', + // content: question, + // date: new Date().toISOString() + // } let conversation: Conversation | null | undefined if (!conversationId) { @@ -284,7 +302,7 @@ const Chat = () => { return abortController.abort() } - const makeApiRequestWithCosmosDB = async (question: string, conversationId?: string) => { + const makeApiRequestWithCosmosDB = async (question: string, conversationId?: string, filebase64?: string) => { setIsLoading(true) setShowLoadingMessage(true) const abortController = new AbortController() @@ -777,7 +795,12 @@ const Chat = () => { <> {answer.role === 'user' ? (
-
{answer.content}
+ {answer?.content && answer.content.trim() && ( +
{answer.content}
+ )} + {answer?.image && ( + User sent image + )}
) : answer.role === 'assistant' ? (
@@ -825,7 +848,7 @@ const Chat = () => { )} - {isLoading && messages.length > 0 && ( + {isLoading && ( { clearOnSend placeholder="Type a new question..." disabled={isLoading} - onSend={(question, id) => { + onSend={(question, id, filebase64) => { appStateContext?.state.isCosmosDBAvailable?.cosmosDB - ? makeApiRequestWithCosmosDB(question, id) - : makeApiRequestWithoutCosmosDB(question, id) + ? makeApiRequestWithCosmosDB(question, id, filebase64) + : makeApiRequestWithoutCosmosDB(question, id, filebase64) }} conversationId={ appStateContext?.state.currentChat?.id ? appStateContext?.state.currentChat?.id : undefined From c6d6407e766ef43a81a4c08f8295a6144c5c73c3 Mon Sep 17 00:00:00 2001 From: Usamazbr Date: Sun, 16 Jun 2024 00:46:36 +0500 Subject: [PATCH 2/2] Image file pasting function added --- .../QuestionInput/QuestionInput.tsx | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/QuestionInput/QuestionInput.tsx b/frontend/src/components/QuestionInput/QuestionInput.tsx index 1ef46fdbbd..8d5b9e39f9 100644 --- a/frontend/src/components/QuestionInput/QuestionInput.tsx +++ b/frontend/src/components/QuestionInput/QuestionInput.tsx @@ -1,4 +1,4 @@ -import { useRef, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { Stack, TextField } from '@fluentui/react' import { AddSquareRegular, SendRegular, DismissCircleFilled, AddSquareFilled } from '@fluentui/react-icons' @@ -75,7 +75,23 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conv reader.onerror = (error) => { console.error("There was an error reading the file!", error) } + } + + // Function to handle paste event + const handlePaste = (e: ClipboardEvent) => { + const items = e.clipboardData?.items; + if (!items) return; + + for (let i = 0; i < items.length; i++) { + if (items[i].type.indexOf('image') !== -1) { + const blob = items[i].getAsFile(); + if (blob) { + convertToBase64(blob); + } + break; + } } + }; const onFileChange = (event: React.ChangeEvent) => { const file = event.target.files?.[0] @@ -88,6 +104,13 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend, conv } } + useEffect(()=>{ + document.addEventListener('paste', handlePaste); + return () => { + document.removeEventListener('paste', handlePaste); + }; + }, []) + const sendQuestionDisabled = disabled || (!question.trim() && !filebase64) const handleClearImage = () => { setFileBase64('')