Skip to content

Commit 833baab

Browse files
acharneskiacharneski
authored
wip (#37)
* wip * wip * wip * wip * wip * Add ImageTableTask and TableCompilationTask Introduces ImageTableTask for generating AI-created image grids and TableCompilationTask for AI-driven table compilation. Registers both tasks in TaskType, updates WriteHtmlTask to use child image chat clients, and refactors SessionTask to use the correct UI link method. * rename * Add MinkowskiCausalLayer and MinkowskiRBFLayer analyses Added comprehensive markdown analyses for MinkowskiCausalLayer and MinkowskiRBFLayer, including formal definitions, gradient derivations, stability, Lipschitz, and numerical analyses, as well as reference implementations in PyTorch, JAX, and mathematical pseudocode. Updated NeuralNetworkLayerTask.kt to support or reference these new layer types. * Update nn_layer_20251128122952.md * wip * wip * wip
1 parent 20a2e01 commit 833baab

File tree

100 files changed

+11496
-511
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+11496
-511
lines changed

‎core/src/main/kotlin/com/simiacryptus/cognotik/input/PDFReader.kt‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,29 @@ import org.apache.pdfbox.pdmodel.PDDocument
55
import org.apache.pdfbox.rendering.PDFRenderer
66
import org.apache.pdfbox.text.PDFTextStripper
77
import java.awt.image.BufferedImage
8-
import java.io.File
8+
import java.io.File
9+
import javax.imageio.ImageIO
10+
import javax.imageio.spi.IIORegistry
11+
import javax.imageio.spi.ImageReaderSpi
12+
import javax.imageio.spi.ImageWriterSpi
13+
import java.util.ServiceLoader
914

10-
class PDFReader(pdfFile: File) : PaginatedDocumentReader, RenderableDocumentReader {
15+
class PDFReader(pdfFile: File) : PaginatedDocumentReader, RenderableDocumentReader {
1116
private val document: PDDocument = Loader.loadPDF(pdfFile)
1217
private val renderer: PDFRenderer = PDFRenderer(document)
18+
companion object {
19+
init {
20+
val registry = IIORegistry.getDefaultInstance()
21+
val loader = PDFReader::class.java.classLoader
22+
ServiceLoader.load(ImageReaderSpi::class.java, loader).forEach {
23+
registry.registerServiceProvider(it)
24+
}
25+
ServiceLoader.load(ImageWriterSpi::class.java, loader).forEach {
26+
registry.registerServiceProvider(it)
27+
}
28+
}
29+
}
30+
1331
override fun getText(): String {
1432
val stripper = PDFTextStripper().apply { sortByPosition = true }
1533
return stripper.getText(document)
@@ -27,7 +45,14 @@ class PDFReader(pdfFile: File) : PaginatedDocumentReader, RenderableDocumentRead
2745
}
2846

2947
override fun renderImage(pageIndex: Int, dpi: Float): BufferedImage {
30-
return renderer.renderImageWithDPI(pageIndex, dpi)
48+
val currentThread = Thread.currentThread()
49+
val originalClassLoader = currentThread.contextClassLoader
50+
try {
51+
currentThread.contextClassLoader = this::class.java.classLoader
52+
return renderer.renderImageWithDPI(pageIndex, dpi)
53+
} finally {
54+
currentThread.contextClassLoader = originalClassLoader
55+
}
3156
}
3257

3358
override fun close() {

‎desktop/src/main/resources/welcome/modules/task-config.js‎

Lines changed: 54 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,20 @@ class TaskConfigManager {
4747
category: 'File Operations',
4848
configFields: []
4949
},
50+
{
51+
id: 'TableCompilation',
52+
name: 'Table Compilation',
53+
description: 'Compile data into tables',
54+
category: 'File Operations',
55+
configFields: []
56+
},
57+
{
58+
id: 'ImageTable',
59+
name: 'Image Table',
60+
description: 'Generate tables containing images',
61+
category: 'File Operations',
62+
configFields: []
63+
},
5064
{
5165
id: 'VectorSearch',
5266
name: 'Vector Search Task',
@@ -610,13 +624,6 @@ class TaskConfigManager {
610624
}
611625
]
612626
},
613-
{
614-
id: 'NarrativeReasoning',
615-
name: 'Narrative Reasoning',
616-
description: 'Understand and analyze narrative structures',
617-
category: 'Writing',
618-
configFields: []
619-
},
620627
{
621628
id: 'NarrativeGeneration',
622629
name: 'Narrative Generation',
@@ -631,6 +638,20 @@ class TaskConfigManager {
631638
category: 'Reasoning',
632639
configFields: []
633640
},
641+
{
642+
id: 'MathematicalReasoning',
643+
name: 'Mathematical Reasoning',
644+
description: 'Solve mathematical problems and proofs',
645+
category: 'Reasoning',
646+
configFields: []
647+
},
648+
{
649+
id: 'NeuralNetworkLayer',
650+
name: 'Neural Network Layer',
651+
description: 'Simulate or design neural network layers',
652+
category: 'Reasoning',
653+
configFields: []
654+
},
634655
{
635656
id: 'EthicalReasoning',
636657
name: 'Ethical Reasoning',
@@ -715,20 +736,27 @@ class TaskConfigManager {
715736
category: 'Writing',
716737
configFields: []
717738
},
739+
{
740+
id: 'SoftwareDesignDocument',
741+
name: 'Software Design Document',
742+
description: 'Generate software design documentation',
743+
category: 'Writing',
744+
configFields: []
745+
},
746+
{
747+
id: 'ComicBookGeneration',
748+
name: 'Comic Book Generation',
749+
description: 'Generate comic book pages and stories',
750+
category: 'Writing',
751+
configFields: []
752+
},
718753
{
719754
id: 'GenerateImage',
720755
name: 'Generate Image',
721756
description: 'Generate images using AI image generation',
722757
category: 'File Operations',
723758
configFields: []
724759
},
725-
{
726-
id: 'CommandSession',
727-
name: 'Command Session Task',
728-
description: 'Execute a series of commands in a session',
729-
category: 'Execution & Automation',
730-
configFields: []
731-
},
732760
{
733761
id: 'SubPlanning',
734762
name: 'Sub-Planning',
@@ -814,6 +842,13 @@ class TaskConfigManager {
814842
description: 'Add illustrations and visual elements to documents',
815843
category: 'File Operations',
816844
configFields: []
845+
},
846+
{
847+
id: 'GenerateQRImage',
848+
name: 'Generate QR Image',
849+
description: 'Generate QR codes',
850+
category: 'File Operations',
851+
configFields: []
817852
}
818853
];
819854
}
@@ -946,7 +981,7 @@ class TaskConfigManager {
946981

947982
// Populate model dropdown
948983
this.populateTaskModelDropdown(modal, configModel);
949-
984+
950985
// Set up sub-tasks event handlers if this is a SubPlanning task
951986
if (taskType.id === 'SubPlanning') {
952987
this.setupSubTasksHandlers(modal, existingConfig);
@@ -1053,6 +1088,7 @@ class TaskConfigManager {
10531088
</div>
10541089
`;
10551090
}
1091+
10561092
// Set up event handlers for sub-tasks management
10571093
setupSubTasksHandlers(modal, existingConfig) {
10581094
// Initialize sub-tasks data on modal
@@ -1085,6 +1121,7 @@ class TaskConfigManager {
10851121
this.attachSubTaskItemHandlers(modal, item);
10861122
});
10871123
}
1124+
10881125
// Attach event handlers to a sub-task item
10891126
attachSubTaskItemHandlers(modal, item) {
10901127
const key = item.dataset.key;
@@ -1128,6 +1165,7 @@ class TaskConfigManager {
11281165
});
11291166
}
11301167
}
1168+
11311169
// Show task type selection dialog
11321170
showTaskTypeSelectionDialog() {
11331171
return new Promise((resolve, reject) => {
@@ -1322,6 +1360,6 @@ class TaskConfigManager {
13221360
}
13231361
}
13241362

1325-
if (typeof module !== 'undefined' && module.exports) {
1363+
if (typeof module !== 'undefined' && module.exports) {
13261364
module.exports = {TaskConfigManager};
13271365
}

‎gradle.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
pluginName=Cognotik - Open Source Agentic Power Tools
22
pluginRepositoryUrl=https://github.com/SimiaCryptus/Cognotik
33
libraryGroup=com.simiacryptus
4-
libraryVersion=2.0.30
4+
libraryVersion=2.0.31
55
# Maven Central Publishing
66
cognotikGroup=com.cognotik
7-
cognotikVersion=2.0.30
7+
cognotikVersion=2.0.31
88
# Signing (set these in ~/.gradle/gradle.properties or as environment variables)
99
# signing.keyId=<last 8 chars of key>
1010
# signing.password=<key password>

‎intellij/build.gradle.kts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ dependencies {
6666
implementation(libs.slf4j.api)
6767
implementation(libs.logback.classic)
6868

69+
implementation("com.github.jai-imageio:jai-imageio-core:1.4.0")
70+
implementation("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0")
71+
implementation("org.apache.pdfbox:jbig2-imageio:3.0.4")
72+
6973
testImplementation(platform(libs.junit.bom))
7074
testImplementation(libs.junit.jupiter.api)
7175
testRuntimeOnly(libs.junit.jupiter.engine)

‎intellij/src/main/kotlin/cognotik/actions/chat/GenericChatAction.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ class GenericChatAction : BaseAction() {
3838
)
3939
SessionProxyServer.agents[session] = ChatSocketManager(
4040
session = session,
41-
model = AppSettingsState.instance.smartChatClient,
42-
parsingModel = AppSettingsState.instance.fastChatClient,
41+
smartModel = AppSettingsState.instance.smartChatClient,
42+
fastModel = AppSettingsState.instance.fastChatClient,
4343
systemPrompt = systemPrompt,
4444
applicationClass = ApplicationServer::class.java,
4545
storage = ApplicationServices.fileApplicationServices().dataStorageFactory,

‎intellij/src/main/kotlin/cognotik/actions/chat/ImageChatAction.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ class ImageChatAction : BaseAction() {
104104
private val codeFiles: Set<Path>
105105
) : ChatSocketManager(
106106
session = session,
107-
model = model,
108-
parsingModel = parsingModel,
107+
smartModel = model,
108+
fastModel = parsingModel,
109109
systemPrompt = "",
110110
applicationClass = ApplicationServer::class.java,
111111
storage = ApplicationServices.fileApplicationServices().dataStorageFactory,

‎intellij/src/main/kotlin/cognotik/actions/chat/ModifyFilesAction.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,8 @@ open class ModifyFilesAction(
119119
private val showLineNumbers: Boolean = false
120120
) : ChatSocketManager(
121121
session = session,
122-
model = model,
123-
parsingModel = parsingModel,
122+
smartModel = model,
123+
fastModel = parsingModel,
124124
systemPrompt = "",
125125
applicationClass = ApplicationServer::class.java,
126126
storage = ApplicationServices.fileApplicationServices().dataStorageFactory,

‎intellij/src/main/kotlin/cognotik/actions/chat/MultiCodeChatAction.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,8 @@ class MultiCodeChatAction : BaseAction() {
9898
private val codeFiles: Set<Path>
9999
) : ChatSocketManager(
100100
session = session,
101-
model = model,
102-
parsingModel = parsingModel,
101+
smartModel = model,
102+
fastModel = parsingModel,
103103
systemPrompt = "",
104104
applicationClass = ApplicationServer::class.java,
105105
storage = ApplicationServices.fileApplicationServices().dataStorageFactory,

‎intellij/src/main/kotlin/cognotik/actions/chat/SmartChatAction.kt‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package cognotik.actions.chat
2+
3+
import cognotik.actions.BaseAction
4+
import com.intellij.openapi.actionSystem.ActionUpdateThread
5+
import com.intellij.openapi.actionSystem.AnActionEvent
6+
import com.intellij.openapi.application.ApplicationManager
7+
import com.simiacryptus.cognotik.CognotikAppServer
8+
import com.simiacryptus.cognotik.config.AppSettingsState
9+
import com.simiacryptus.cognotik.platform.ApplicationServices
10+
import com.simiacryptus.cognotik.platform.Session
11+
import com.simiacryptus.cognotik.util.BrowseUtil.browse
12+
import com.simiacryptus.cognotik.util.LoggerFactory
13+
import com.simiacryptus.cognotik.util.SessionProxyServer
14+
import com.simiacryptus.cognotik.util.UITools
15+
import com.simiacryptus.cognotik.webui.application.AppInfoData
16+
import com.simiacryptus.cognotik.webui.application.ApplicationServer
17+
import com.simiacryptus.cognotik.webui.chat.SmartChatSocketManager
18+
import java.text.SimpleDateFormat
19+
20+
/**
21+
* Smart Chat Action that provides enhanced chat functionality with:
22+
* - Automatic history summarization when conversation gets too long
23+
* - Query elevation from fast model to smart model for complex queries
24+
*/
25+
class SmartChatAction : BaseAction() {
26+
override fun getActionUpdateThread() = ActionUpdateThread.BGT
27+
28+
private val systemPrompt = """
29+
You are a helpful AI assistant with expertise in software development, coding, and technical problem-solving.
30+
You provide clear, accurate, and well-structured responses.
31+
When discussing code, you explain your reasoning and suggest best practices.
32+
""".trimIndent()
33+
34+
override fun handle(e: AnActionEvent) {
35+
val project = e.project ?: return
36+
37+
try {
38+
UITools.runAsync(project, "Initializing Smart Chat", true) { progress ->
39+
progress.isIndeterminate = true
40+
progress.text = "Setting up smart chat session..."
41+
42+
val session = Session.newGlobalID()
43+
SessionProxyServer.metadataStorage.setSessionName(
44+
null,
45+
session,
46+
"Smart Chat @ ${SimpleDateFormat("HH:mm:ss").format(System.currentTimeMillis())}"
47+
)
48+
SessionProxyServer.agents[session] = SmartChatSocketManager(
49+
session = session,
50+
smartModel = AppSettingsState.instance.smartChatClient,
51+
fastModel = AppSettingsState.instance.fastChatClient,
52+
systemPrompt = systemPrompt,
53+
applicationClass = ApplicationServer::class.java,
54+
storage = ApplicationServices.fileApplicationServices().dataStorageFactory,
55+
budget = 2.0,
56+
maxHistoryTokens = 4000,
57+
targetSummaryTokens = 1000,
58+
preserveRecentMessages = 4
59+
)
60+
ApplicationServer.appInfoMap[session] = AppInfoData(
61+
applicationName = "Smart Chat",
62+
inputCnt = 0,
63+
stickyInput = true,
64+
loadImages = false,
65+
showMenubar = false
66+
)
67+
68+
val uri = CognotikAppServer.getServer().server.uri.resolve("/#$session")
69+
ApplicationManager.getApplication().executeOnPooledThread {
70+
try {
71+
BaseAction.log.info("Opening browser to $uri")
72+
browse(uri)
73+
} catch (e: Throwable) {
74+
UITools.error(log, "Failed to open browser", e)
75+
}
76+
}
77+
}
78+
} catch (e: Throwable) {
79+
log.warn("Error opening browser", e)
80+
}
81+
}
82+
83+
companion object {
84+
private val log = LoggerFactory.getLogger(SmartChatAction::class.java)
85+
}
86+
}

0 commit comments

Comments
 (0)