Skip to content

Commit 78e7a21

Browse files
acharneskiacharneski
committed
1.2.18
1 parent 59f93bd commit 78e7a21

File tree

11 files changed

+399
-87
lines changed

11 files changed

+399
-87
lines changed

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Gradle Releases -> https://github.com/gradle/gradle/releases
22
libraryGroup=com.simiacryptus.skyenet
3-
libraryVersion=1.2.17
3+
libraryVersion=1.2.18
44
gradleVersion=7.6.1
55
kotlin.daemon.jvmargs=-Xmx4g

‎webui/src/main/kotlin/com/simiacryptus/skyenet/Retryable.kt‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,25 +19,24 @@ open class Retryable(
1919
set(tabLabel, process(container))
2020
}
2121

22+
fun retry() {
23+
val idx = tabs.size
24+
val label = label(idx)
25+
val content = StringBuilder("Retrying..." + SessionTask.spinner)
26+
tabs.add(label to content)
27+
update()
28+
val newResult = process(content)
29+
content.clear()
30+
set(label, newResult)
31+
}
32+
2233
override fun renderTabButtons(): String = """
2334
<div class="tabs">${
2435
tabs.withIndex().joinToString("\n") { (index, _) ->
2536
val tabId = "$index"
2637
"""<button class="tab-button" data-for-tab="$tabId">${index + 1}</button>"""
2738
}
28-
}
29-
${
30-
ui.hrefLink("♻") {
31-
val idx = tabs.size
32-
val label = label(idx)
33-
val content = StringBuilder("Retrying..." + SessionTask.spinner)
34-
tabs.add(label to content)
35-
update()
36-
val newResult = process(content)
37-
content.clear()
38-
set(label, newResult)
39-
}
40-
}
39+
}${ui.hrefLink("♻") { retry() }}
4140
</div>
4241
"""
4342

‎webui/src/main/kotlin/com/simiacryptus/skyenet/TabbedDisplay.kt‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,18 @@ open class TabbedDisplay(
1414
}
1515

1616
val size: Int get() = tabs.size
17-
open fun render() = if (tabs.isEmpty()) "<div/>" else """
18-
<div class="tabs-container" id="${UUID.randomUUID()}">
19-
${renderTabButtons()}
20-
${
21-
tabs.toTypedArray().withIndex().joinToString("\n")
22-
{ (idx, t) -> renderContentTab(t, idx) }
17+
val tabId = UUID.randomUUID()
18+
open fun render() = if (tabs.isEmpty()) "<div/>" else {
19+
"""
20+
<div class="tabs-container" id="$tabId">
21+
${renderTabButtons()}
22+
${
23+
tabs.toTypedArray().withIndex().joinToString("\n")
24+
{ (idx, t) -> renderContentTab(t, idx) }
25+
}
26+
</div>
27+
"""
2328
}
24-
</div>
25-
"""
2629

2730
val container: StringBuilder by lazy {
2831
log.debug("Initializing container with rendered content")

‎webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/DocumentParserApp.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ open class DocumentParserApp(
3131
val reader: (File) -> DocumentReader = {
3232
when {
3333
it.name.endsWith(".pdf", ignoreCase = true) -> PDFReader(it)
34+
it.name.endsWith(".html", ignoreCase = true) -> HTMLReader(it)
35+
it.name.endsWith(".htm", ignoreCase = true) -> HTMLReader(it)
3436
else -> TextReader(it)
3537
}
3638
},

‎webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/HTMLReader.kt‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package com.simiacryptus.skyenet.apps.parse
2+
3+
import org.jsoup.Jsoup
4+
import org.jsoup.nodes.Document
5+
import java.awt.image.BufferedImage
6+
import java.io.File
7+
8+
class HTMLReader(private val htmlFile: File) : DocumentParserApp.DocumentReader {
9+
private val document: Document = Jsoup.parse(htmlFile, "UTF-8")
10+
private val pages: List<String> = splitIntoPages(document.body().text())
11+
private lateinit var settings: DocumentParserApp.Settings
12+
13+
fun configure(settings: DocumentParserApp.Settings) {
14+
this.settings = settings
15+
}
16+
17+
override fun getPageCount(): Int = pages.size
18+
19+
override fun getText(startPage: Int, endPage: Int): String {
20+
val text = pages.subList(startPage, endPage.coerceAtMost(pages.size)).joinToString("\n")
21+
return if (::settings.isInitialized && settings.addLineNumbers) {
22+
text.lines().mapIndexed { index, line ->
23+
"${(index + 1).toString().padStart(6)}: $line"
24+
}.joinToString("\n")
25+
} else text
26+
}
27+
28+
override fun renderImage(pageIndex: Int, dpi: Float): BufferedImage {
29+
throw UnsupportedOperationException("HTML files do not support image rendering")
30+
}
31+
32+
override fun close() {
33+
// No resources to close for HTML files
34+
}
35+
36+
private fun splitIntoPages(text: String, maxChars: Int = 16000): List<String> {
37+
if (text.length <= maxChars) return listOf(text)
38+
39+
// Split on paragraph boundaries when possible
40+
val paragraphs = text.split(Regex("\\n\\s*\\n"))
41+
42+
val pages = mutableListOf<String>()
43+
var currentPage = StringBuilder()
44+
45+
for (paragraph in paragraphs) {
46+
if (currentPage.length + paragraph.length > maxChars) {
47+
if (currentPage.isNotEmpty()) {
48+
pages.add(currentPage.toString())
49+
currentPage = StringBuilder()
50+
}
51+
// If a single paragraph is longer than maxChars, split it
52+
if (paragraph.length > maxChars) {
53+
val words = paragraph.split(" ")
54+
var currentChunk = StringBuilder()
55+
56+
for (word in words) {
57+
if (currentChunk.length + word.length > maxChars) {
58+
pages.add(currentChunk.toString())
59+
currentChunk = StringBuilder()
60+
}
61+
if (currentChunk.isNotEmpty()) currentChunk.append(" ")
62+
currentChunk.append(word)
63+
}
64+
if (currentChunk.isNotEmpty()) {
65+
currentPage.append(currentChunk)
66+
}
67+
} else {
68+
currentPage.append(paragraph)
69+
}
70+
} else {
71+
if (currentPage.isNotEmpty()) currentPage.append("\n\n")
72+
currentPage.append(paragraph)
73+
}
74+
}
75+
76+
if (currentPage.isNotEmpty()) {
77+
pages.add(currentPage.toString())
78+
}
79+
80+
return pages
81+
}
82+
}

‎webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/LogDataParsingModel.kt‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,27 +34,55 @@ open class LogDataParsingModel(
3434

3535
override fun getFastParser(api: API): (String) -> LogData {
3636
val patternGenerator = LogPatternGenerator(parsingModel, temperature)
37-
3837
return { originalText ->
39-
var remainingText = originalText
40-
var result: LogData? = null
41-
var iterationCount = 0
38+
parseText(originalText, patternGenerator, api, emptyList())
39+
}
40+
}
4241

43-
try {
44-
while (remainingText.isNotBlank() && iterationCount++ < maxIterations) {
45-
val patterns = patternGenerator.generatePatterns(api, remainingText)
46-
if (patterns.isEmpty()) break
47-
val applyPatterns = applyPatterns(remainingText, (result?.patterns ?: emptyList()) + patterns)
48-
result = applyPatterns.first
49-
remainingText = applyPatterns.second
42+
override fun getSmartParser(api: API): (LogData, String) -> LogData {
43+
val patternGenerator = LogPatternGenerator(parsingModel, temperature)
44+
return { runningDocument, prompt ->
45+
parseText(prompt, patternGenerator, api, runningDocument.patterns ?: emptyList())
46+
}
47+
}
48+
49+
private fun parseText(
50+
originalText: String,
51+
patternGenerator: LogPatternGenerator,
52+
api: API,
53+
existingPatterns: List<PatternData>
54+
): LogData {
55+
var remainingText = originalText
56+
var result: LogData? = null
57+
var iterationCount = 0
58+
var currentPatterns = existingPatterns
59+
try {
60+
// First try with existing patterns
61+
if (currentPatterns.isNotEmpty()) {
62+
val applyPatterns = applyPatterns(remainingText, currentPatterns)
63+
result = applyPatterns.first
64+
remainingText = applyPatterns.second
65+
}
66+
// Then generate new patterns for remaining text
67+
while (remainingText.isNotBlank() && iterationCount++ < maxIterations) {
68+
val newPatterns = patternGenerator.generatePatterns(api, remainingText)
69+
if (newPatterns.isEmpty()) break
70+
currentPatterns = (currentPatterns + newPatterns).distinctBy { it.regex }
71+
val applyPatterns = applyPatterns(remainingText, currentPatterns)
72+
result = if (result != null) {
73+
merge(result, applyPatterns.first)
74+
} else {
75+
applyPatterns.first
5076
}
51-
} catch (e: Exception) {
52-
log.error("Error parsing log data", e)
77+
remainingText = applyPatterns.second
5378
}
54-
result ?: LogData()
79+
} catch (e: Exception) {
80+
log.error("Error parsing log data", e)
5581
}
82+
return result ?: LogData()
5683
}
5784

85+
5886
private fun applyPatterns(text: String, patterns: List<PatternData>): Pair<LogData, String> {
5987
val patterns = patterns.filter { it.regex != null }.groupBy { it.id }.map { it.value.first() }
6088
val matches = patterns.flatMap { pattern ->

‎webui/src/main/kotlin/com/simiacryptus/skyenet/apps/parse/TextReader.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ class TextReader(private val textFile: File) : DocumentParserApp.DocumentReader
4141
var fitness = -((leftSize.toDouble() / text.length) * Math.log1p(rightSize.toDouble() / text.length) +
4242
(rightSize.toDouble() / text.length) * Math.log1p(leftSize.toDouble() / text.length))
4343
if (lines[i].isEmpty()) fitness *= 2
44-
i to fitness.toDouble()
44+
i to fitness
4545
}.toTypedArray().toMutableList()
4646

47-
var bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2
47+
val bestSplitIndex = splitFitnesses.minByOrNull { it.second }?.first ?: lines.size / 2
4848
val leftText = lines.subList(0, bestSplitIndex).joinToString("\n")
4949
val rightText = lines.subList(bestSplitIndex, lines.size).joinToString("\n")
5050
return splitIntoPages(leftText, maxChars) + splitIntoPages(rightText, maxChars)

‎webui/src/main/kotlin/com/simiacryptus/skyenet/apps/plan/CommandAutoFixTask.kt‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,12 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na
6060
api2: OpenAIClient,
6161
planSettings: PlanSettings
6262
) {
63+
var autoRetries = if(planSettings.autoFix) 5 else 0
6364
val semaphore = Semaphore(0)
6465
val hasError = AtomicBoolean(false)
6566
val onComplete = { semaphore.release() }
66-
Retryable(agent.ui, task = task) {
67+
lateinit var retryable: Retryable
68+
retryable = Retryable(agent.ui, task = task) {
6769
val task = agent.ui.newTask(false).apply { it.append(placeholder) }
6870
this.planTask?.commands?.forEachIndexed { index, commandWithDir ->
6971
val alias = commandWithDir.command.firstOrNull()
@@ -122,19 +124,22 @@ ${planSettings.commandAutoFixCommands?.joinToString("\n") { " * ${File(it).na
122124
)
123125
}
124126
resultFn("All Command Auto Fix tasks completed")
125-
task.add(if (!hasError.get()) {
126-
onComplete()
127-
MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false)
128-
} else {
129-
MarkdownUtil.renderMarkdown(
130-
"## Some Command Auto Fix tasks failed\n",
131-
ui = agent.ui
132-
) + acceptButtonFooter(
133-
agent.ui
134-
) {
127+
task.add(
128+
if (!hasError.get()) {
135129
onComplete()
136-
}
137-
})
130+
MarkdownUtil.renderMarkdown("## All Command Auto Fix tasks completed successfully\n", ui = agent.ui, tabs = false)
131+
} else {
132+
val s = MarkdownUtil.renderMarkdown(
133+
"## Some Command Auto Fix tasks failed\n",
134+
ui = agent.ui
135+
) + acceptButtonFooter(
136+
agent.ui
137+
) {
138+
onComplete()
139+
}
140+
if(autoRetries-- > 0) retryable.retry()
141+
s
142+
})
138143
task.placeholder
139144
}
140145
try {

0 commit comments

Comments
 (0)