diff --git a/build.gradle.kts b/build.gradle.kts index d1f039e7..5fa1c0ed 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -28,7 +28,7 @@ repositories { val jetty_version = "11.0.24" val slf4j_version = "2.0.16" -val skyenet_version = "1.2.16" +val skyenet_version = "1.2.17" val remoterobot_version = "0.11.23" val jackson_version = "2.17.2" diff --git a/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorAction.kt b/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorAction.kt index f0df3c1b..341dd714 100644 --- a/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorAction.kt +++ b/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorAction.kt @@ -12,9 +12,11 @@ import com.intellij.openapi.actionSystem.ActionUpdateThread import com.intellij.openapi.actionSystem.AnActionEvent import com.intellij.openapi.vfs.VirtualFile import com.simiacryptus.jopenai.models.chatModel -import com.simiacryptus.skyenet.apps.parse.CodeParsingModel import com.simiacryptus.skyenet.apps.parse.DocumentParserApp import com.simiacryptus.skyenet.apps.parse.DocumentParsingModel +import com.simiacryptus.skyenet.apps.parse.ParsingModel +import com.simiacryptus.skyenet.apps.parse.ParsingModel.DocumentData +import com.simiacryptus.skyenet.apps.parse.ParsingModelType import com.simiacryptus.skyenet.core.platform.Session import com.simiacryptus.skyenet.core.platform.file.DataStorage import com.simiacryptus.skyenet.webui.application.AppInfoData @@ -26,6 +28,7 @@ import java.nio.file.Path class DocumentDataExtractorAction : BaseAction() { val path = "/pdfExtractor" private var settings = DocumentParserApp.Settings() + private var modelType = ParsingModelType.Document override fun getActionUpdateThread() = ActionUpdateThread.BGT @@ -68,29 +71,24 @@ class DocumentDataExtractorAction : BaseAction() { return } val selectedFile = processableFiles.first() - val configDialog = DocumentDataExtractorConfigDialog(e.project, settings) + val configDialog = DocumentDataExtractorConfigDialog(e.project, settings, modelType) if (!configDialog.showAndGet()) return settings = configDialog.settings + modelType = configDialog.modelType as ParsingModelType val session = Session.newGlobalID() DataStorage.sessionPaths[session] = selectedFile.toFile.parentFile val smartModel = AppSettingsState.instance.smartModel.chatModel() - val parsingModel = when { - selectedFile.name.endsWith(".pdf", ignoreCase = true) -> DocumentParsingModel(smartModel, 0.1) - selectedFile.name.endsWith(".txt", ignoreCase = true) -> DocumentParsingModel(smartModel, 0.1) - selectedFile.name.endsWith(".html", ignoreCase = true) -> DocumentParsingModel(smartModel, 0.1) - selectedFile.name.endsWith(".htm", ignoreCase = true) -> DocumentParsingModel(smartModel, 0.1) - selectedFile.name.endsWith(".md", ignoreCase = true) -> DocumentParsingModel(smartModel, 0.1) - else -> CodeParsingModel(smartModel, 0.1) - } + val parsingModel = ParsingModelType.getImpl(smartModel, 0.1, modelType) - SessionProxyServer.Companion.chats[session] = object : DocumentParserApp( + SessionProxyServer.chats[session] = object : DocumentParserApp( applicationName = "Document Extractor", path = this@DocumentDataExtractorAction.path, api = this@DocumentDataExtractorAction.api, fileInputs = processableFiles.map { it.toNioPath() }, - parsingModel = parsingModel, + parsingModel = parsingModel as ParsingModel, + fastMode = settings.fastMode, ) { override fun initSettings(session: Session): T = this@DocumentDataExtractorAction.settings as T override val root: File get() = selectedFile.parent.toFile diff --git a/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorConfigDialog.kt b/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorConfigDialog.kt index a8a7a00a..2bb492a5 100644 --- a/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorConfigDialog.kt +++ b/src/main/kotlin/com/github/simiacryptus/aicoder/actions/knowledge/DocumentDataExtractorConfigDialog.kt @@ -5,11 +5,13 @@ import com.intellij.openapi.ui.DialogWrapper import com.intellij.ui.components.JBCheckBox import com.intellij.ui.components.JBTextField import com.simiacryptus.skyenet.apps.parse.DocumentParserApp +import com.simiacryptus.skyenet.apps.parse.ParsingModelType import javax.swing.* class DocumentDataExtractorConfigDialog( project: Project?, - var settings: DocumentParserApp.Settings + var settings: DocumentParserApp.Settings, + var modelType: ParsingModelType<*> ) : DialogWrapper(project) { private val dpiField = JBTextField(settings.dpi.toString()) @@ -20,6 +22,11 @@ class DocumentDataExtractorConfigDialog( private val saveImageFilesCheckbox = JBCheckBox("Save Image Files", settings.saveImageFiles) private val saveTextFilesCheckbox = JBCheckBox("Save Text Files", settings.saveTextFiles) private val saveFinalJsonCheckbox = JBCheckBox("Save Final JSON", settings.saveFinalJson) + private val fastModeCheckbox = JBCheckBox("Fast Mode", settings.fastMode) + private val addLineNumbersCheckbox = JBCheckBox("Add Line Numbers", settings.addLineNumbers) + private val modelTypeComboBox = JComboBox(ParsingModelType.values().toTypedArray()).apply { + selectedItem = modelType + } init { init() @@ -29,7 +36,7 @@ class DocumentDataExtractorConfigDialog( override fun createCenterPanel(): JComponent { val panel = JPanel() panel.layout = BoxLayout(panel, BoxLayout.Y_AXIS) - + panel.add(createLabeledField("Parsing Model:", modelTypeComboBox)) panel.add(createLabeledField("DPI:", dpiField)) panel.add(createLabeledField("Max Pages:", maxPagesField)) panel.add(createLabeledField("Output Format:", outputFormatField)) @@ -38,6 +45,8 @@ class DocumentDataExtractorConfigDialog( panel.add(saveImageFilesCheckbox) panel.add(saveTextFilesCheckbox) panel.add(saveFinalJsonCheckbox) + panel.add(fastModeCheckbox) + panel.add(addLineNumbersCheckbox) return panel } @@ -60,8 +69,11 @@ class DocumentDataExtractorConfigDialog( showImages = showImagesCheckbox.isSelected, saveImageFiles = saveImageFilesCheckbox.isSelected, saveTextFiles = saveTextFilesCheckbox.isSelected, - saveFinalJson = saveFinalJsonCheckbox.isSelected + saveFinalJson = saveFinalJsonCheckbox.isSelected, + fastMode = fastModeCheckbox.isSelected, + addLineNumbers = addLineNumbersCheckbox.isSelected ) + modelType = modelTypeComboBox.selectedItem as ParsingModelType<*> super.doOKAction() } } \ No newline at end of file diff --git a/src/main/kotlin/com/github/simiacryptus/aicoder/ui/SettingsWidgetFactory.kt b/src/main/kotlin/com/github/simiacryptus/aicoder/ui/SettingsWidgetFactory.kt index 8d6201bf..d2cd421d 100644 --- a/src/main/kotlin/com/github/simiacryptus/aicoder/ui/SettingsWidgetFactory.kt +++ b/src/main/kotlin/com/github/simiacryptus/aicoder/ui/SettingsWidgetFactory.kt @@ -40,16 +40,26 @@ class SettingsWidgetFactory : StatusBarWidgetFactory { private var project: Project? = null private val sessionsList = JBList() private val sessionsListModel = DefaultListModel() - private fun createModelTree(title: String, selectedModel: String?): JTree { + private fun createModelTree(title: String, selectedModel: String?): Tree { val root = DefaultMutableTreeNode(title) - val providers = models().groupBy { it.second.provider } + // Filter models by providers that have API keys set + val providers = models() + .filter { model -> + val providerName = model.second.provider.name + AppSettingsState.instance.apiKey?.get(providerName)?.isNotEmpty() == true + } + .groupBy { it.second.provider } + for ((provider, models) in providers) { val providerNode = DefaultMutableTreeNode(provider.name) for (model in models) { val modelNode = DefaultMutableTreeNode(model.second.modelName) providerNode.add(modelNode) } - root.add(providerNode) + // Only add provider node if it has models + if (providerNode.childCount > 0) { + root.add(providerNode) + } } val treeModel = DefaultTreeModel(root) val tree = Tree(treeModel)