[Backport #19822] Pull interface package when replaying Exercice by interface (#19833)

* [Backport #19822] Pull interface package when replaying Exercice by interface (#19822)

fix DACH-NY/canton#20645

* fmt

* disable ReinterpretTestV2

Author: remyhaemmerle-da

sdk/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/preprocessing/Preprocessor.scala

@@ -5,6 +5,7 @@ package com.daml.lf
 package engine
 package preprocessing
 
+import com.daml.lf.command.ReplayCommand
 import com.daml.lf.data.{ImmArray, Ref}
 import com.daml.lf.language.{Ast, LookupError}
 import com.daml.lf.speedy.SValue
@@ -163,10 +164,21 @@ private[engine] final class Preprocessor(
 
   private[engine] def preprocessReplayCommand(
       cmd: command.ReplayCommand
-  ): Result[speedy.Command] =
-    safelyRun(pullTemplatePackage(List(cmd.templateId))) {
+  ): Result[speedy.Command] = {
+    def templateAndInterfaceIds =
+      cmd match {
+        case ReplayCommand.Create(templateId, _) => List(templateId)
+        case ReplayCommand.Exercise(templateId, interfaceId, _, _, _) =>
+          templateId :: interfaceId.toList
+        case ReplayCommand.ExerciseByKey(templateId, _, _, _) => List(templateId)
+        case ReplayCommand.Fetch(templateId, _) => List(templateId)
+        case ReplayCommand.FetchByKey(templateId, _) => List(templateId)
+        case ReplayCommand.LookupByKey(templateId, _) => List(templateId)
+      }
+    safelyRun(pullTemplatePackage(templateAndInterfaceIds)) {
       commandPreprocessor.unsafePreprocessReplayCommand(cmd)
     }
+  }
 
   /** Translates a complete transaction. Assumes no contract ID suffixes are used */
   def translateTransactionRoots(

sdk/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/ReinterpretTest.scala

@@ -24,7 +24,7 @@ import org.scalatest.matchers.should.Matchers
 import scala.language.implicitConversions
 
 class ReinterpretTestV1 extends ReinterpretTest(LanguageMajorVersion.V1)
-class ReinterpretTestV2 extends ReinterpretTest(LanguageMajorVersion.V2)
+//class ReinterpretTestV2 extends ReinterpretTest(LanguageMajorVersion.V2)
 
 class ReinterpretTest(majorLanguageVersion: LanguageMajorVersion)
     extends AnyWordSpec
@@ -62,13 +62,15 @@ class ReinterpretTest(majorLanguageVersion: LanguageMajorVersion)
         )
     )
 
-  private val engine = new Engine(
+  private def freshEngine = new Engine(
     EngineConfig(
       allowedLanguageVersions = language.LanguageVersion.AllVersions(majorLanguageVersion),
       requireSuffixedGlobalContractId = true,
     )
   )
 
+  private val engine = freshEngine
+
   def Top(xs: Shape*) = Shape.Top(xs.toList)
   def Exercise(xs: Shape*) = Shape.Exercise(xs.toList)
   def Rollback(xs: Shape*) = Shape.Rollback(xs.toList)
@@ -171,6 +173,54 @@ class ReinterpretTest(majorLanguageVersion: LanguageMajorVersion)
       Shape.ofTransaction(tx.transaction) shouldBe Top(Rollback(Exercise(Create())))
     }
   }
+
+  "exercise by interface pull interfaceId package " in {
+    val templatePkgId = Ref.PackageId.assertFromString("-template-package-")
+    val templateId = Ref.Identifier(templatePkgId, "A:T")
+    val interfacePkgId = Ref.PackageId.assertFromString("-interface-package-")
+    val interfaceId = Ref.Identifier(interfacePkgId, "B:I")
+
+    val testCases = Table(
+      "cmd" -> "packgeIds",
+      ReplayCommand.Create(templateId, ValueUnit) -> List(templatePkgId),
+      ReplayCommand.Exercise(templateId, None, toContractId("cid"), "myChoice", ValueUnit) -> List(
+        templatePkgId
+      ),
+      ReplayCommand.Exercise(
+        templateId,
+        Some(interfaceId),
+        toContractId("cid"),
+        "myChoice",
+        ValueUnit,
+      ) -> List(templatePkgId, interfacePkgId),
+      ReplayCommand.ExerciseByKey(templateId, ValueUnit, "MyChoide", ValueUnit) -> List(
+        templatePkgId
+      ),
+      ReplayCommand.Fetch(templateId, toContractId("cid")) -> List(templatePkgId),
+      ReplayCommand.FetchByKey(templateId, ValueUnit) -> List(templatePkgId),
+      ReplayCommand.LookupByKey(templateId, ValueUnit) -> List(templatePkgId),
+    )
+
+    forEvery(testCases) { (cmd, pkgIds) =>
+      val emptyPackage =
+        Package(Map.empty, Set.empty, LanguageMajorVersion.V1.maxStableVersion, None)
+      var queriedPackageIds = Set.empty[Ref.PackageId]
+      val trackPackageQueries: PartialFunction[Ref.PackageId, Package] = { pkgId =>
+        queriedPackageIds = queriedPackageIds + pkgId
+        emptyPackage
+      }
+      freshEngine
+        .reinterpret(
+          submitters,
+          cmd,
+          Some(seed),
+          time,
+          time,
+        )
+        .consume(pkgs = trackPackageQueries)
+      pkgIds.toSet shouldBe queriedPackageIds
+    }
+  }
 }
 
 object ReinterpretTest {

sdk/daml-script/test/BUILD.bazel

@@ -12,7 +12,10 @@ load("//bazel_tools:haskell.bzl", "da_haskell_test")
 load("@build_environment//:configuration.bzl", "sdk_version")
 load("@os_info//:os_info.bzl", "is_windows")
 load("//rules_daml:daml.bzl", "daml_compile")
-load("//daml-lf/language:daml-lf.bzl", "LF_DEV_VERSIONS", "LF_MAJOR_VERSIONS", "lf_version_latest", "mangle_for_damlc")
+load("//daml-lf/language:daml-lf.bzl", "LF_DEV_VERSIONS", "lf_version_latest", "mangle_for_damlc")
+
+# Test Only LF 1.x
+LF_MAJOR_VERSIONS = ["1"]
 
 [
     genrule(
@@ -21,6 +24,8 @@ load("//daml-lf/language:daml-lf.bzl", "LF_DEV_VERSIONS", "LF_MAJOR_VERSIONS", "
             glob(["**/*.daml"]) + [
                 "//daml-script/daml:daml-script-{}.dar".format(target),
                 "//docs:source/daml-script/template-root/src/ScriptExample.daml",
+                "//daml-script/test:template.dar",
+                "//daml-script/test:retrointerface.dar",
             ],
         outs = ["script-test-v{}.dar".format(major)],
         cmd = """
@@ -33,6 +38,8 @@ load("//daml-lf/language:daml-lf.bzl", "LF_DEV_VERSIONS", "LF_MAJOR_VERSIONS", "
       cp -L $(location :daml/TestExceptions.daml) $$TMP_DIR/daml
       cp -L $(location :daml/TestInterfaces.daml) $$TMP_DIR/daml
       cp -L $(location //docs:source/daml-script/template-root/src/ScriptExample.daml) $$TMP_DIR/daml
+      cp -L $(location //daml-script/test:template.dar) $$TMP_DIR/
+      cp -L $(location //daml-script/test:retrointerface.dar) $$TMP_DIR/
       cat << EOF >> $$TMP_DIR/daml/ScriptExample.daml
 initializeFixed : Script ()
 initializeFixed = do
@@ -52,6 +59,9 @@ dependencies:
   - daml-stdlib
   - daml-prim
   - daml-script-{target}.dar
+data-dependencies:
+  - template.dar
+  - retrointerface.dar
 build-options:
   - --target={target}
 EOF
@@ -330,6 +340,19 @@ daml_compile(
     version = "2.0.0",
 )
 
+daml_compile(
+    name = "template",
+    srcs = [":daml/Template.daml"],
+    target = "1.15",
+)
+
+daml_compile(
+    name = "retrointerface",
+    srcs = [":daml/RetroInterface.daml"],
+    data_dependencies = ["//daml-script/test:template.dar"],
+    target = "1.15",
+)
+
 da_scala_test_suite(
     name = "test",
     size = "large",
@@ -340,6 +363,8 @@ da_scala_test_suite(
     data = [
         ":script-test-no-ledger.dar",
         "//daml-script/runner:daml-script-binary",
+        "//daml-script/test:retrointerface.dar",
+        "//daml-script/test:template.dar",
     ] + [
         ":script-test-v{}.dar".format(major)
         for major in LF_MAJOR_VERSIONS

sdk/daml-script/test/daml/MultiTest.daml

@@ -9,25 +9,8 @@ import DA.List
 import DA.Optional(fromSome)
 import DA.Time
 import Daml.Script
-
-template T
-  with
-    p1 : Party
-    p2 : Party
-  where
-    signatory p1, p2
-
-template TProposal
-  with
-    p1 : Party
-    p2 : Party
-  where
-    signatory p1
-    observer p2
-    choice Accept : (ContractId T, Int)
-      controller p2
-      do cid <- create T { p1, p2 }
-         pure (cid, 42)
+import Template
+import qualified RetroInterface as Retro
 
 multiTest : Script Int
 multiTest = do
@@ -83,30 +66,6 @@ waitForCid tries p cid
       Some _ -> pure ()
   where delay = seconds 1
 
-
-template Box with
-    s: Party
-    content: Text
-  where
-    signatory s
-    key s: Party
-    maintainer key
-    nonconsuming choice Open: Text with
-        c: Party
-      controller c
-      do
-        pure content
-
-template Helper with
-    s: Party
-  where
-    signatory s
-    nonconsuming choice FailWith: () with
-        msg: Text
-      controller s
-      do
-        error msg
-
 disclosuresTest : Script Text
 disclosuresTest = do
   -- init
@@ -160,3 +119,14 @@ inactiveDisclosureDoesNotFailDuringSubmission = do
       pure ()
 
     pure ()
+
+retroactiveExercise: Script Int
+retroactiveExercise = do
+    alice <- allocatePartyOn "alice" (ParticipantName "participant0")
+    bob <- allocatePartyOn "bob" (ParticipantName "participant1")
+    proposalCid <- alice `submit` createCmd (TProposal alice bob)
+    waitForCid tries bob proposalCid
+    (cid, _) <- bob `submit` exerciseCmd proposalCid (Accept)
+    waitForCid tries alice cid
+    alice `submit` exerciseCmd (toInterfaceContractId @Retro.I cid) (Retro.Noop alice)
+    pure 43

sdk/daml-script/test/daml/NO_AUTO_COPYRIGHT

@@ -0,0 +1 @@
+RetroInterface.daml and Template.daml were backported and created in 2024

sdk/daml-script/test/daml/RetroInterface.daml

@@ -0,0 +1,20 @@
+-- Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module RetroInterface where
+
+import Template
+
+data IView = IView {}
+
+interface I where
+  viewtype IView
+
+  nonconsuming choice Noop : Int with
+      a : Party
+    controller a
+    do pure 42
+
+  interface instance I for T where
+    view = IView {}
+

sdk/daml-script/test/daml/Template.daml

@@ -0,0 +1,46 @@
+-- Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module Template where
+
+template T
+  with
+    p1 : Party
+    p2 : Party
+  where
+    signatory p1, p2
+
+template TProposal
+  with
+    p1 : Party
+    p2 : Party
+  where
+    signatory p1
+    observer p2
+    choice Accept : (ContractId T, Int)
+      controller p2
+      do cid <- create T { p1, p2 }
+         pure (cid, 42)
+
+template Box with
+    s: Party
+    content: Text
+  where
+    signatory s
+    key s: Party
+    maintainer key
+    nonconsuming choice Open: Text with
+        c: Party
+      controller c
+      do
+        pure content
+
+template Helper with
+    s: Party
+  where
+    signatory s
+    nonconsuming choice FailWith: () with
+        msg: Text
+      controller s
+      do
+        error msg

sdk/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/Daml2ScriptTestRunner.scala

@@ -23,6 +23,7 @@ class Daml2ScriptTestRunner extends DamlScriptTestRunner {
           |MultiTest:listKnownPartiesTest SUCCESS
           |MultiTest:multiTest SUCCESS
           |MultiTest:partyIdHintTest SUCCESS
+          |MultiTest:retroactiveExercise SUCCESS
           |ScriptExample:allocateParties SUCCESS
           |ScriptExample:initializeFixed SUCCESS
           |ScriptExample:initializeUser SUCCESS

sdk/daml-script/test/src/com/digitalasset/daml/lf/engine/script/test/MultiParticipantIT.scala

@@ -4,6 +4,7 @@
 package com.daml.lf.engine.script
 package test
 
+import com.daml.bazeltools.BazelRunfiles.rlocation
 import com.daml.lf.data.FrontStack
 import com.daml.lf.data.Ref._
 import com.daml.lf.engine.script.ScriptTimeMode
@@ -13,6 +14,7 @@ import org.scalatest.Inside
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AsyncWordSpec
 
+import java.nio.file.Paths
 import scala.util.{Failure, Success}
 
 class MultiParticipantITV1 extends MultiParticipantIT(LanguageMajorVersion.V1)
@@ -30,8 +32,10 @@ class MultiParticipantIT(override val majorLanguageVersion: LanguageMajorVersion
   final override protected lazy val nParticipants = 2
   final override protected lazy val timeMode = ScriptTimeMode.WallClock
 
-  // TODO(#17366): Delete once 2.0 is introduced and Canton supports LF v2 in non-dev mode.
-  final override protected lazy val devMode = (majorLanguageVersion == LanguageMajorVersion.V2)
+  final override protected lazy val darFiles = List(
+    rlocation(Paths.get(s"daml-script/test/template.dar")),
+    rlocation(Paths.get(s"daml-script/test/retrointerface.dar")),
+  )
 
   "Multi-participant Daml Script" can {
     "multiTest" should {
@@ -90,7 +94,6 @@ class MultiParticipantIT(override val majorLanguageVersion: LanguageMajorVersion
           )
           assert(vals.get(1) == second)
         }
-
       }
     }
 
@@ -129,5 +132,22 @@ class MultiParticipantIT(override val majorLanguageVersion: LanguageMajorVersion
         } yield error.getMessage should include regex """Unhandled Daml exception\: DA\.Exception\.GeneralError\:GeneralError\@[a-f0-9]{8}\{ message \= \"Here\" \}"""
       }
     }
+
+    "exercise retroactive instance" in {
+      // Regression test for https://github.com/DACH-NY/canton/issues/20645
+      // To reproduce the bug in 2.8.10 or 2.9.4 the package retrointerface.dar
+      // should not be loaded in the participant1 before starting the test.
+      // Unfortunately there is n simple way to ensure that, so here are some guardrail:
+      // - do not write in this file other tests that use retroactiveExercise
+      // - do not load the script in the ledger, but only the template code.
+      for {
+        clients <- scriptClients()
+        r <- run(
+          clients,
+          QualifiedName.assertFromString("MultiTest:retroactiveExercise"),
+          dar = dar,
+        )
+      } yield assert(r == SInt64(43))
+    }
   }
 }