diff --git a/.gitattributes b/.gitattributes index 304c9c2c0a..2a1dc7c60d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ -* text=auto -*.bat eol=crlf -gradlew text eol=lf +* text=auto +/gradlew.bat text eol=crlf +/gradlew text eol=lf diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0665932b0e..732279ce5c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -73,6 +73,7 @@ kotlinx-serialization-json-jvm = { module = "org.jetbrains.kotlinx:kotlinx-seria kotlinx-serialization-properties = { module = "org.jetbrains.kotlinx:kotlinx-serialization-properties", version.ref = "serialization" } kotlinx-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlinx-datetime" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } +kotlinx-coroutines-core-linuxx64 = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core-linuxx64", version.ref = "kotlinx-coroutines" } ktoml-core = { module = "com.akuleshov7:ktoml-core", version.ref = "ktoml" } ktoml-file = { module = "com.akuleshov7:ktoml-file", version.ref = "ktoml" } diff --git a/save-agent/build.gradle.kts b/save-agent/build.gradle.kts index ab2836a56c..383f47916a 100644 --- a/save-agent/build.gradle.kts +++ b/save-agent/build.gradle.kts @@ -39,6 +39,7 @@ kotlin { implementation(libs.kotlinx.serialization.properties) implementation(libs.okio) implementation(libs.kotlinx.datetime) + implementation(libs.kotlinx.coroutines.core.linuxx64) } } val linuxX64Test by getting { diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt index ac73e8ddb2..a195809eb6 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt @@ -49,21 +49,21 @@ internal suspend fun SaveAgent.downloadTestResources(config: BackendConfig, targ } /** - * Download additional resources from [additionalResourcesAsString] into [targetDirectory] + * Download additional resources from [additionalFiles] into [targetDirectory] * * @param baseUrl * @param targetDirectory - * @param additionalResourcesAsString + * @param additionalFiles * @param executionId * @return result */ internal suspend fun SaveAgent.downloadAdditionalResources( baseUrl: String, targetDirectory: Path, - additionalResourcesAsString: String, + additionalFiles: List, executionId: String, ) = runCatching { - FileKey.parseList(additionalResourcesAsString) + additionalFiles .map { fileKey -> val result = httpClient.downloadFile( "$baseUrl/internal/files/download?executionId=$executionId", diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt index 50c8ec9633..07e73c0394 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt @@ -6,13 +6,16 @@ import com.saveourtool.save.agent.utils.* import com.saveourtool.save.agent.utils.readFile import com.saveourtool.save.agent.utils.requiredEnv import com.saveourtool.save.agent.utils.sendDataToBackend +import com.saveourtool.save.core.config.resolveSaveOverridesTomlConfig import com.saveourtool.save.core.files.getWorkingDirectory import com.saveourtool.save.core.logging.describe +import com.saveourtool.save.core.logging.logTrace import com.saveourtool.save.core.plugin.Plugin import com.saveourtool.save.core.result.CountWarnings import com.saveourtool.save.core.utils.ExecutionResult import com.saveourtool.save.core.utils.ProcessBuilder import com.saveourtool.save.core.utils.runIf +import com.saveourtool.save.domain.FileKey import com.saveourtool.save.domain.TestResultDebugInfo import com.saveourtool.save.plugins.fix.FixPlugin import com.saveourtool.save.reporter.Report @@ -29,6 +32,7 @@ import io.ktor.client.statement.* import io.ktor.http.* import io.ktor.utils.io.core.* import okio.FileSystem +import okio.Path import okio.Path.Companion.toPath import okio.buffer @@ -93,7 +97,7 @@ class SaveAgent(private val config: AgentConfiguration, logDebugCustom("Will now download tests") val executionId = requiredEnv(AgentEnvName.EXECUTION_ID) val targetDirectory = config.testSuitesDir.toPath() - downloadTestResources(config.backend, targetDirectory, executionId).runIf({ isFailure }) { + downloadTestResources(config.backend, targetDirectory, executionId).runIf(failureResultPredicate) { logErrorCustom("Unable to download tests for execution $executionId: ${exceptionOrNull()?.describe()}") state.value = AgentState.CRASHED return@launch @@ -101,13 +105,30 @@ class SaveAgent(private val config: AgentConfiguration, logInfoCustom("Downloaded all tests for execution $executionId to $targetDirectory") logDebugCustom("Will now download additional resources") - val additionalFilesList = requiredEnv(AgentEnvName.ADDITIONAL_FILES_LIST) - downloadAdditionalResources(config.backend.url, targetDirectory, additionalFilesList, executionId).runIf({ isFailure }) { - logErrorCustom("Unable to download resources for execution $executionId based on list [$additionalFilesList]: ${exceptionOrNull()?.describe()}") + val additionalFiles = FileKey.parseList(requiredEnv(AgentEnvName.ADDITIONAL_FILES_LIST)) + downloadAdditionalResources(config.backend.url, targetDirectory, additionalFiles, executionId).runIf(failureResultPredicate) { + logErrorCustom("Unable to download resources for execution $executionId based on list [$additionalFiles]: ${exceptionOrNull()?.describe()}") state.value = AgentState.CRASHED return@launch } + logInfoCustom("Downloaded all additional resources for execution $executionId to $targetDirectory") + // a temporary workaround for python integration + logDebugCustom("Will execute additionally setup of evaluated tool for execution $executionId if it's required") + executeAdditionallySetup(targetDirectory, additionalFiles).runIf(failureResultPredicate) { + logErrorCustom("Unable to execute additionally setup for $executionId: ${exceptionOrNull()?.describe()}") + state.value = AgentState.CRASHED + return@launch + } + logInfoCustom("Additionally setup has completed for execution $executionId") + + logDebugCustom("Will create `save-overrides.toml` for execution $executionId if it's required") + prepareSaveOverridesToml(targetDirectory).runIf(failureResultPredicate) { + logErrorCustom("Unable to prepare `save-overrides.toml` for $executionId: ${exceptionOrNull()?.describe()}") + state.value = AgentState.CRASHED + return@launch + } + logInfoCustom("Created `save-overrides.toml` for execution $executionId based on configuration provided by orchestrator") state.value = AgentState.STARTING } return coroutineScope.launch { startHeartbeats(this) } @@ -122,6 +143,58 @@ class SaveAgent(private val config: AgentConfiguration, coroutineScope.cancel() } + // a temporary workaround for python integration + private fun executeAdditionallySetup(targetDirectory: Path, additionalFiles: List) = runCatching { + additionalFiles + .singleOrNull { it.name == "setup.sh" } + ?.let { fileKey -> + val targetFile = targetDirectory / fileKey.name + logDebugCustom("Additionally setup of evaluated tool by $targetFile") + val setupResult = ProcessBuilder(true, FileSystem.SYSTEM) + .exec( + "./$targetFile", + "", + null, + SETUP_SH_TIMEOUT + ) + if (setupResult.code != 0) { + throw IllegalStateException("${fileKey.name} is failed with error: ${setupResult.stderr}") + } + logTrace("$fileKey is executed successfully. Output: ${setupResult.stdout}") + } + } + + // prepare save-overrides.toml based on config.save.* + private fun prepareSaveOverridesToml(targetDirectory: Path) = runCatching { + with(config.save) { + val generalConfig = buildMap { + overrideExecCmd?.let { put("execCmd", it) } + batchSize?.let { put("batchSize", it) } + batchSeparator?.let { put("batchSeparator", it) } + }.map { (key, value) -> "$key = $value" } + val fixAndWarnConfigs = buildMap { + overrideExecFlags?.let { put("execFlags", it) } + }.map { (key, value) -> "$key = $value" } + val saveOverridesTomlContent = buildString { + if (generalConfig.isNotEmpty()) { + appendLine("[general]") + generalConfig.forEach { appendLine(it) } + } + if (fixAndWarnConfigs.isNotEmpty()) { + appendLine("[fix]") + fixAndWarnConfigs.forEach { appendLine(it) } + appendLine("[warn]") + fixAndWarnConfigs.forEach { appendLine(it) } + } + } + if (saveOverridesTomlContent.isNotEmpty()) { + FileSystem.SYSTEM.write(targetDirectory.resolveSaveOverridesTomlConfig(), true) { + writeUtf8(saveOverridesTomlContent) + } + } + } + } + @Suppress("WHEN_WITHOUT_ELSE") // when with sealed class private suspend fun startHeartbeats(coroutineScope: CoroutineScope) { logInfoCustom("Scheduling heartbeats") @@ -207,7 +280,6 @@ class SaveAgent(private val config: AgentConfiguration, } } - @Suppress("MagicNumber") private fun runSave(cliArgs: String): ExecutionResult { val fullCliCommand = buildString { append(config.cliCommand) @@ -237,7 +309,7 @@ class SaveAgent(private val config: AgentConfiguration, fullCliCommand, "", config.logFilePath.toPath(), - 1_000_000L + SAVE_CLI_TIMEOUT ) } @@ -374,4 +446,10 @@ class SaveAgent(private val config: AgentConfiguration, contentType(ContentType.Application.Json) setBody(AgentVersion(config.id, SAVE_CLOUD_VERSION)) } + + companion object { + private const val SAVE_CLI_TIMEOUT = 1_000_000L + private const val SETUP_SH_TIMEOUT = 1_000L + private val failureResultPredicate: Result<*>.() -> Boolean = { isFailure } + } }