Merge remote-tracking branch 'origin/staging' into preview/custom-htt…

…p-service-spi # Conflicts: # docs/Changelog.md
TouK · Mar 4, 2025 · ae67727 · ae67727
2 parents 8172b4b + 1e8cf03
commit ae67727
Show file tree

Hide file tree

Showing 126 changed files with 1,463 additions and 798 deletions.
diff --git a/...s/src/test/scala/pl/touk/nussknacker/engine/benchmarks/interpreter/InterpreterSetup.scala b/...s/src/test/scala/pl/touk/nussknacker/engine/benchmarks/interpreter/InterpreterSetup.scala
@@ -3,24 +3,28 @@ package pl.touk.nussknacker.engine.benchmarks.interpreter
 import cats.Monad
 import cats.data.Validated.{Invalid, Valid}
 import cats.data.ValidatedNel
-import pl.touk.nussknacker.engine.{api, CustomProcessValidatorLoader, InterpretationResult}
+import pl.touk.nussknacker.engine.{api, CustomProcessValidatorLoader, InterpretationResult, RuntimeMode}
 import pl.touk.nussknacker.engine.Interpreter.InterpreterShape
 import pl.touk.nussknacker.engine.api._
-import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, DesignerWideComponentId, UnboundedStreamComponent}
+import pl.touk.nussknacker.engine.api.component.{
+  ComponentDefinition,
+  DesignerWideComponentId,
+  NodesDeploymentData,
+  UnboundedStreamComponent
+}
 import pl.touk.nussknacker.engine.api.context.ProcessCompilationError
 import pl.touk.nussknacker.engine.api.exception.NuExceptionInfo
 import pl.touk.nussknacker.engine.api.process._
 import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess
 import pl.touk.nussknacker.engine.compile.ProcessCompilerData
 import pl.touk.nussknacker.engine.compiledgraph.part.ProcessPart
-import pl.touk.nussknacker.engine.definition.component.{ComponentDefinitionWithImplementation, Components}
+import pl.touk.nussknacker.engine.definition.component.Components
 import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode
 import pl.touk.nussknacker.engine.definition.model.{ModelDefinition, ModelDefinitionWithClasses}
 import pl.touk.nussknacker.engine.dict.SimpleDictRegistry
 import pl.touk.nussknacker.engine.modelconfig.ComponentsUiConfig
 import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector
 import pl.touk.nussknacker.engine.testing.ModelDefinitionBuilder
-import pl.touk.nussknacker.engine.util.Implicits._
 
 import scala.language.higherKinds
 import scala.reflect.ClassTag
@@ -73,8 +77,9 @@ class InterpreterSetup[T: ClassTag] {
       List.empty,
       getClass.getClassLoader,
       ProductionServiceInvocationCollector,
-      ComponentUseCase.EngineRuntime,
-      CustomProcessValidatorLoader.emptyCustomProcessValidator
+      RuntimeMode.Live,
+      CustomProcessValidatorLoader.emptyCustomProcessValidator,
+      NodesDeploymentData.empty,
     )
   }
 

diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/Service.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/Service.scala
@@ -1,7 +1,7 @@
 package pl.touk.nussknacker.engine.api
 
 import pl.touk.nussknacker.engine.api.component.{AllProcessingModesComponent, Component}
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -37,7 +37,7 @@ trait ServiceInvoker {
   def invoke(context: Context)(
       implicit ec: ExecutionContext,
       collector: ServiceInvocationCollector,
-      componentUseCase: ComponentUseCase
+      componentUseContext: ComponentUseContext,
   ): Future[Any]
 
 }
diff --git a/...nts-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/...nts-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala
@@ -4,7 +4,9 @@ import io.circe.{Decoder, Encoder}
 import pl.touk.nussknacker.engine.api.NodeId
 import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData
 
-final case class NodesDeploymentData(dataByNodeId: Map[NodeId, NodeDeploymentData])
+final case class NodesDeploymentData(private val dataByNodeId: Map[NodeId, NodeDeploymentData]) {
+  def get(nodeId: NodeId): Option[NodeDeploymentData] = dataByNodeId.get(nodeId)
+}
 
 object NodesDeploymentData {
 

diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseCase.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseCase.scala
diff --git a/...nents-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseContext.scala b/...nents-api/src/main/scala/pl/touk/nussknacker/engine/api/process/ComponentUseContext.scala
@@ -0,0 +1,26 @@
+package pl.touk.nussknacker.engine.api.process
+
+import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData
+
+/**
+  * Specifies the mode a node is used/invoked. It can be one of the following values:
+  * <ul>
+  * <li>LiveRuntime - component is invoked in live engine, eg. Flink.</li>
+  * <li>ScenarioTesting - component is invoked in test mode to collect results for test data. Can be used to stub real implementation.</li>
+  * </ul>
+  */
+sealed trait ComponentUseContext {
+  def deploymentData: Option[NodeDeploymentData]
+}
+
+object ComponentUseContext {
+
+  case class LiveRuntime(nodeData: Option[NodeDeploymentData]) extends ComponentUseContext {
+    override def deploymentData: Option[NodeDeploymentData] = nodeData
+  }
+
+  case object ScenarioTesting extends ComponentUseContext {
+    override def deploymentData: Option[NodeDeploymentData] = None
+  }
+
+}
diff --git a/...ents/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala b/...ents/openapi/src/it/scala/pl/touk/nussknacker/openapi/functional/OpenAPIServiceSpec.scala
@@ -8,14 +8,14 @@ import org.scalatest.funsuite.FixtureAnyFunSuite
 import org.scalatest.matchers.should.Matchers
 import pl.touk.nussknacker.engine.api._
 import pl.touk.nussknacker.engine.api.parameter.ParameterName
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.test.EmptyInvocationCollector.Instance
 import pl.touk.nussknacker.engine.api.typed.TypedMap
 import pl.touk.nussknacker.engine.util.ResourceLoader
 import pl.touk.nussknacker.engine.util.runtimecontext.TestEngineRuntimeContext
 import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersAndReturnType
 import pl.touk.nussknacker.http.backend.FixedAsyncHttpClientBackendProvider
-import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecurityConfig, SecuritySchemeName}
+import pl.touk.nussknacker.openapi.{ApiKeySecret, OpenAPIServicesConfig, SecuritySchemeName}
 import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
 import pl.touk.nussknacker.openapi.parser.SwaggerParser
 import pl.touk.nussknacker.test.PatientScalaFutures
@@ -31,9 +31,9 @@ class OpenAPIServiceSpec
     with LazyLogging
     with PatientScalaFutures {
 
-  implicit val componentUseCase: ComponentUseCase = ComponentUseCase.EngineRuntime
-  implicit val metaData: MetaData                 = MetaData("testProc", StreamMetaData())
-  implicit val context: Context                   = Context("testContextId", Map.empty)
+  implicit val componentUseContext: ComponentUseContext = ComponentUseContext.LiveRuntime(None)
+  implicit val metaData: MetaData                       = MetaData("testProc", StreamMetaData())
+  implicit val context: Context                         = Context("testContextId", Map.empty)
   val jobData = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))
 
   type FixtureParam = EagerServiceWithStaticParametersAndReturnType

diff --git a/...onents/openapi/src/main/scala/pl/touk/nussknacker/openapi/enrichers/SwaggerEnricher.scala b/...onents/openapi/src/main/scala/pl/touk/nussknacker/openapi/enrichers/SwaggerEnricher.scala
@@ -4,7 +4,7 @@ import org.asynchttpclient.DefaultAsyncHttpClient
 import pl.touk.nussknacker.engine.api.{ContextId, MetaData}
 import pl.touk.nussknacker.engine.api.definition.Parameter
 import pl.touk.nussknacker.engine.api.parameter.ParameterName
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext
 import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
 import pl.touk.nussknacker.engine.api.typed.typing
@@ -63,7 +63,7 @@ class SwaggerEnricher(
       collector: ServiceInvocationCollector,
       contextId: ContextId,
       metaData: MetaData,
-      componentUseCase: ComponentUseCase
+      componentUseContext: ComponentUseContext
   ): Future[AnyRef] =
     measuring {
       swaggerHttpService.invoke(

diff --git a/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala b/components/openapi/src/test/scala/pl/touk/nussknacker/openapi/BaseOpenAPITest.scala
@@ -4,7 +4,7 @@ import cats.data.Validated
 import cats.data.Validated.{Invalid, Valid}
 import org.apache.commons.io.IOUtils
 import pl.touk.nussknacker.engine.api._
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.util.runtimecontext.TestEngineRuntimeContext
 import pl.touk.nussknacker.engine.util.service.EagerServiceWithStaticParametersAndReturnType
 import pl.touk.nussknacker.openapi.enrichers.{SwaggerEnricherCreator, SwaggerEnrichers}
@@ -19,9 +19,9 @@ trait BaseOpenAPITest {
 
   protected val baseConfig: OpenAPIServicesConfig = OpenAPIServicesConfig(new URL("http://foo"))
 
-  implicit val componentUseCase: ComponentUseCase = ComponentUseCase.EngineRuntime
-  implicit val metaData: MetaData                 = MetaData("testProc", StreamMetaData())
-  implicit val context: Context                   = Context("testContextId", Map.empty)
+  implicit val componentUseContext: ComponentUseContext = ComponentUseContext.LiveRuntime(None)
+  implicit val metaData: MetaData                       = MetaData("testProc", StreamMetaData())
+  implicit val context: Context                         = Context("testContextId", Map.empty)
   private val jobData        = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))
   private val runtimeContext = TestEngineRuntimeContext(jobData)
 

diff --git a/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvoker.scala b/components/sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvoker.scala
@@ -1,9 +1,8 @@
 package pl.touk.nussknacker.sql.service
 
 import pl.touk.nussknacker.engine.api.{Context, Params, ServiceInvoker}
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
-import pl.touk.nussknacker.engine.api.typed.typing
 import pl.touk.nussknacker.engine.util.service.AsyncExecutionTimeMeasurement
 import pl.touk.nussknacker.sql.db.WithDBConnectionPool
 import pl.touk.nussknacker.sql.db.query._
@@ -34,7 +33,7 @@ class DatabaseEnricherInvoker(
   override def invoke(context: Context)(
       implicit ec: ExecutionContext,
       collector: ServiceInvocationCollector,
-      componentUseCase: ComponentUseCase
+      componentUseContext: ComponentUseContext,
   ): Future[queryExecutor.QueryResult] = {
     getTimeMeasurement().measuring {
       queryDatabase(queryArgumentsExtractor(argsCount, params, context))

diff --git a/...sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvokerWithCache.scala b/...sql/src/main/scala/pl/touk/nussknacker/sql/service/DatabaseEnricherInvokerWithCache.scala
@@ -2,9 +2,8 @@ package pl.touk.nussknacker.sql.service
 
 import com.github.benmanes.caffeine.cache.{AsyncCache, Caffeine}
 import pl.touk.nussknacker.engine.api.{Context, Params}
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
-import pl.touk.nussknacker.engine.api.typed.typing
 import pl.touk.nussknacker.engine.util.service.AsyncExecutionTimeMeasurement
 import pl.touk.nussknacker.sql.db.query.{QueryArguments, QueryResultStrategy}
 import pl.touk.nussknacker.sql.db.schema.TableDefinition
@@ -53,7 +52,7 @@ class DatabaseEnricherInvokerWithCache(
   override def invoke(context: Context)(
       implicit ec: ExecutionContext,
       collector: ServiceInvocationCollector,
-      componentUseCase: ComponentUseCase
+      componentUseContext: ComponentUseContext,
   ): Future[queryExecutor.QueryResult] = {
     getTimeMeasurement().measuring {
       val queryArguments = queryArgumentsExtractor(argsCount, params, context)

diff --git a/...ents/sql/src/test/scala/pl/touk/nussknacker/sql/utils/BaseDatabaseQueryEnricherTest.scala b/...ents/sql/src/test/scala/pl/touk/nussknacker/sql/utils/BaseDatabaseQueryEnricherTest.scala
@@ -9,7 +9,7 @@ import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.EmptyProce
 import pl.touk.nussknacker.engine.api.context.ValidationContext
 import pl.touk.nussknacker.engine.api.context.transformation.{FailedToDefineParameter, OutputVariableNameValue}
 import pl.touk.nussknacker.engine.api.parameter.ParameterName
-import pl.touk.nussknacker.engine.api.process.ComponentUseCase
+import pl.touk.nussknacker.engine.api.process.ComponentUseContext
 import pl.touk.nussknacker.engine.api.test.EmptyInvocationCollector
 import pl.touk.nussknacker.engine.api.test.InvocationCollectors.ServiceInvocationCollector
 import pl.touk.nussknacker.engine.api.typed.typing
@@ -19,11 +19,11 @@ import scala.concurrent.ExecutionContext
 
 trait BaseDatabaseQueryEnricherTest extends AnyFunSuite with Matchers with BeforeAndAfterAll {
 
-  implicit val ec: ExecutionContext                  = ExecutionContext.Implicits.global
-  implicit val context: Context                      = Context("", Map.empty)
-  implicit val metaData: MetaData                    = MetaData("", StreamMetaData())
-  implicit val collector: ServiceInvocationCollector = EmptyInvocationCollector.Instance
-  implicit val componentUseCase: ComponentUseCase    = ComponentUseCase.TestRuntime
+  implicit val ec: ExecutionContext                     = ExecutionContext.Implicits.global
+  implicit val context: Context                         = Context("", Map.empty)
+  implicit val metaData: MetaData                       = MetaData("", StreamMetaData())
+  implicit val collector: ServiceInvocationCollector    = EmptyInvocationCollector.Instance
+  implicit val componentUseContext: ComponentUseContext = ComponentUseContext.ScenarioTesting
 
   val jobData: JobData = JobData(metaData, ProcessVersion.empty.copy(processName = metaData.name))
 

diff --git a/designer/client/src/components/modals/AdvancedParametersSection.tsx b/designer/client/src/components/modals/AdvancedParametersSection.tsx
@@ -6,19 +6,19 @@ import { Typography } from "@mui/material";
 import AccordionDetails from "@mui/material/AccordionDetails";
 
 interface Props {
-    nodeId: string;
+    componentId: string;
 }
 
-export function AdvancedParametersSection({ children, nodeId }: PropsWithChildren<Props>): JSX.Element {
+export function AdvancedParametersSection({ children, componentId }: PropsWithChildren<Props>): JSX.Element {
     return (
         <Accordion disableGutters elevation={0} sx={{ border: 0, "&::before": { display: "none" } }}>
             <AccordionSummary
                 expandIcon={<ExpandMoreIcon sx={{ color: "inherit" }} />}
-                aria-controls={`${nodeId}-content`}
-                id={`${nodeId}-header`}
+                aria-controls={`${componentId}-content`}
+                id={`${componentId}-header`}
                 sx={{ flexDirection: "row-reverse", border: 0 }}
             >
-                <Typography>{nodeId}</Typography>
+                <Typography>{componentId}</Typography>
             </AccordionSummary>
             <AccordionDetails>{children}</AccordionDetails>
         </Accordion>