Merge remote-tracking branch 'origin/version-1.4.x' into node-2348-ap…

…i-liquid-balances

.github/ISSUE_TEMPLATE/feature_request.md

@@ -3,7 +3,7 @@ name: Feature request
 about: Suggest an idea for this project
 title: ''
 labels: feature
-assignees: pivoo81
+assignees: phearnot
 
 ---
 

grpc-server/src/test/scala/com/wavesplatform/events/BlockchainUpdatesSpec.scala

@@ -212,9 +212,7 @@ class BlockchainUpdatesSpec extends FreeSpec with WithBUDomain with ScalaFutures
       }
     }
 
-    "should survive invalid rollback" in withDomainAndRepo(
-      SettingsFromDefaultConfig.copy(dbSettings = SettingsFromDefaultConfig.dbSettings.copy(maxRollbackDepth = 0))
-    ) { (d, repo) =>
+    "should survive invalid rollback" in withDomainAndRepo(RideV6.copy(dbSettings = dbSettings.copy(maxRollbackDepth = 0))) { (d, repo) =>
       for (_ <- 1 to 10) d.appendBlock()
       intercept[RuntimeException](d.rollbackTo(1)) // Should fail
       d.appendBlock()

lang/js/src/main/scala/JsAPI.scala → lang/js/src/main/scala/com/wavesplatform/JsAPI.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 import JsApiUtils.*
 import com.wavesplatform.DocSource
 import com.wavesplatform.lang.*

lang/js/src/main/scala/JsApiUtils.scala → lang/js/src/main/scala/com/wavesplatform/JsApiUtils.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 import com.wavesplatform.lang.contract.DApp
 import com.wavesplatform.lang.v1.FunctionHeader.{Native, User}
 import com.wavesplatform.lang.v1.compiler.CompilationError
@@ -14,7 +16,7 @@ import scala.scalajs.js.JSConverters._
 object JsApiUtils {
 
   def serPart[T](f: T => js.Any)(part: PART[T]): js.Object = {
-    val partValue = Expressions.PART.toOption(part).fold(null:Any)(f)
+    val partValue = Expressions.PART.toOption(part).fold(null: Any)(f)
     jObj.applyDynamic("apply")(
       "value"    -> partValue,
       "posStart" -> part.position.start,
@@ -55,11 +57,11 @@ object JsApiUtils {
     }
 
     jObj.applyDynamic("apply")(
-      "type"         -> "DAPP",
-      "posStart"     -> ast.position.start,
-      "posEnd"       -> ast.position.end,
-      "decList"      -> ast.decs.map(serDec).toJSArray,
-      "annFuncList"  -> ast.fs.map(serAnnFunc).toJSArray
+      "type"        -> "DAPP",
+      "posStart"    -> ast.position.start,
+      "posEnd"      -> ast.position.end,
+      "decList"     -> ast.decs.map(serDec).toJSArray,
+      "annFuncList" -> ast.fs.map(serAnnFunc).toJSArray
     )
   }
 
@@ -146,8 +148,8 @@ object JsApiUtils {
 
       case Expressions.FOLD(_, limit, value, acc, func, _, _) =>
         val additionalDataObj = jObj.applyDynamic("apply")(
-          "name"  -> s"FOLD<$limit>",
-          "args"  -> js.Array(serExpr(value), serExpr(acc), func.key.toString: js.Any)
+          "name" -> s"FOLD<$limit>",
+          "args" -> js.Array(serExpr(value), serExpr(acc), func.key.toString: js.Any)
         )
         mergeJSObjects(commonDataObj, additionalDataObj)
 
@@ -173,7 +175,9 @@ object JsApiUtils {
   }
 
   def serMatchCase(c: Expressions.MATCH_CASE, simpleCtx: Map[String, Pos]): js.Object = {
-    val vars = c.pattern.subpatterns.collect { case (Expressions.TypedVar(Some(newVarName), caseType), _) => (serPartStr(newVarName), serType(caseType)) }
+    val vars = c.pattern.subpatterns.collect { case (Expressions.TypedVar(Some(newVarName), caseType), _) =>
+      (serPartStr(newVarName), serType(caseType))
+    }
     jObj.applyDynamic("apply")(
       "type"       -> "MATCH_CASE",
       "posStart"   -> c.position.start,
@@ -200,7 +204,7 @@ object JsApiUtils {
     t match {
       case Expressions.AnyType(_) =>
         jObj.applyDynamic("apply")(
-          "typeName"  -> "Any"
+          "typeName" -> "Any"
         )
       case Expressions.Single(name, parameter) =>
         jObj.applyDynamic("apply")(
@@ -255,21 +259,25 @@ object JsApiUtils {
   def toJs(ast: EXPR): js.Object = {
     def r(expr: EXPR): js.Object = {
       expr match {
-        case CONST_LONG(t)      => jObj.applyDynamic("apply")("type" -> "LONG", "value"    -> t.toDouble)
-        case GETTER(ref, field) => jObj.applyDynamic("apply")("type" -> "GETTER", "ref"    -> r(ref), "field" -> field)
+        case CONST_LONG(t)      => jObj.applyDynamic("apply")("type" -> "LONG", "value" -> t.toDouble)
+        case GETTER(ref, field) => jObj.applyDynamic("apply")("type" -> "GETTER", "ref" -> r(ref), "field" -> field)
         case CONST_BYTESTR(bs)  => jObj.applyDynamic("apply")("type" -> "BYTESTR", "value" -> bs.arr.toJSArray)
-        case CONST_STRING(s)    => jObj.applyDynamic("apply")("type" -> "STRING", "value"  -> s)
+        case CONST_STRING(s)    => jObj.applyDynamic("apply")("type" -> "STRING", "value" -> s)
         case LET_BLOCK(let, body) =>
           jObj.applyDynamic("apply")("type" -> "BLOCK", "let" -> jObj("name" -> let.name, "value" -> r(let.value)), "body" -> r(body))
         case IF(cond, ifTrue, ifFalse) =>
           jObj.applyDynamic("apply")("type" -> "IF", "condition" -> r(cond), "true" -> r(ifTrue), "false" -> r(ifFalse))
-        case REF(key)         => jObj.applyDynamic("apply")("type" -> "REF", "key"    -> key)
+        case REF(key)         => jObj.applyDynamic("apply")("type" -> "REF", "key" -> key)
         case CONST_BOOLEAN(b) => jObj.applyDynamic("apply")("type" -> "BOOL", "value" -> b)
         case FUNCTION_CALL(function, args) =>
-          jObj.applyDynamic("apply")("type" -> "CALL", "name" -> (function match {
-            case Native(name)          => name.toString()
-            case User(internalName, _) => internalName
-          }), "args" -> args.map(r).toJSArray)
+          jObj.applyDynamic("apply")(
+            "type" -> "CALL",
+            "name" -> (function match {
+              case Native(name)          => name.toString()
+              case User(internalName, _) => internalName
+            }),
+            "args" -> args.map(r).toJSArray
+          )
         case t => jObj.applyDynamic("apply")("[not_supported]stringRepr" -> t.toString)
       }
     }

lang/tests-js/src/test/scala/DAppComplexities.scala → lang/tests-js/src/test/scala/com/wavesplatform/DAppComplexities.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 case class DAppComplexities(
     complexity: Int,
     verifierComplexity: Int,

lang/tests-js/src/test/scala/Global.scala → lang/tests-js/src/test/scala/com/wavesplatform/Global.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 import scala.scalajs.js.annotation.JSExportTopLevel
 import scala.scalajs.js.typedarray.ArrayBuffer
 

lang/tests-js/src/test/scala/JsAPITest.scala → lang/tests-js/src/test/scala/com/wavesplatform/JsAPITest.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 import com.wavesplatform.lang.directives.values.{V5, V6}
 import utest.*
 

lang/tests-js/src/test/scala/JsTestBase.scala → lang/tests-js/src/test/scala/com/wavesplatform/JsTestBase.scala

@@ -1,3 +1,5 @@
+package com.wavesplatform
+
 import com.wavesplatform.lang.directives.values.{StdLibVersion, V6}
 import utest.{TestSuite, assert}
 

node/src/main/scala/com/wavesplatform/Application.scala

@@ -215,7 +215,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con
         .map {
           case Right(discardedBlocks) =>
             allChannels.broadcast(LocalScoreChanged(blockchainUpdater.score))
-            if (returnTxsToUtx) utxStorage.addAndCleanup(discardedBlocks.view.flatMap(_._1.transactionData))
+            if (returnTxsToUtx) utxStorage.addAndScheduleCleanup(discardedBlocks.view.flatMap(_._1.transactionData))
             Right(discardedBlocks)
           case Left(error) => Left(error)
         }

node/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala

@@ -102,7 +102,7 @@ object ExtensionAppender extends ScorexLogging {
 
                       val newTransactions = newBlocks.view.flatMap(_.transactionData).toSet
                       utxStorage.removeAll(newTransactions)
-                      utxStorage.addAndCleanup(droppedBlocks.flatMap(_._1.transactionData).filterNot(newTransactions))
+                      utxStorage.addAndScheduleCleanup(droppedBlocks.flatMap(_._1.transactionData).filterNot(newTransactions))
                       Right(Some(blockchainUpdater.score))
                   }
                 }

node/src/main/scala/com/wavesplatform/state/appender/MicroblockAppender.scala

@@ -30,11 +30,14 @@ object MicroblockAppender extends ScorexLogging {
       blockchainUpdater
         .processMicroBlock(microBlock, verify)
         .map { totalBlockId =>
-          if (microBlock.transactionData.nonEmpty) log.trace {
-            s"Removing mined txs from ${microBlock.stringRepr(totalBlockId)}: ${microBlock.transactionData.map(_.id()).mkString(", ")}"
+          if (microBlock.transactionData.nonEmpty) {
+            utxStorage.removeAll(microBlock.transactionData)
+            log.trace(
+              s"Removing txs of ${microBlock.stringRepr(totalBlockId)} ${microBlock.transactionData.map(_.id()).mkString("(", ", ", ")")} from UTX pool"
+            )
           }
-          utxStorage.removeAll(microBlock.transactionData)
-          utxStorage.cleanUnconfirmed()
+
+          utxStorage.scheduleCleanup()
           totalBlockId
         }
     }).executeOn(scheduler)

node/src/main/scala/com/wavesplatform/state/appender/package.scala

@@ -39,7 +39,7 @@ package object appender {
           .map { discardedDiffs =>
             utx.removeAll(block.transactionData)
             utx.setPriorityDiffs(discardedDiffs)
-            utx.runCleanup()
+            utx.scheduleCleanup()
             Some(blockchainUpdater.height)
           }
       }

node/src/main/scala/com/wavesplatform/utx/UtxPoolImpl.scala

@@ -156,11 +156,10 @@ class UtxPoolImpl(
         log.trace(s"putIfNew(${tx.id()}) succeeded, isNew = $isNew")
       case Left(err) =>
         log.debug(s"putIfNew(${tx.id()}) failed with ${extractErrorMessage(err)}")
-        val errMsg = err match {
+        traceLogger.trace(err match {
           case w: WithLog => w.toStringWithLog(maxTxErrorLogSize)
           case err        => err.toString
-        }
-        traceLogger.trace(errMsg)
+        })
     }
     tracedIsNew
   }
@@ -528,12 +527,12 @@ class UtxPoolImpl(
   }
 
   /** DOES NOT verify transactions */
-  def addAndCleanup(transactions: Iterable[Transaction]): Unit = {
+  def addAndScheduleCleanup(transactions: Iterable[Transaction]): Unit = {
     transactions.foreach(addTransaction(_, verify = false))
     TxCleanup.runCleanupAsync()
   }
 
-  def runCleanup(): Unit = {
+  def scheduleCleanup(): Unit = {
     TxCleanup.runCleanupAsync()
   }
 

node/src/test/scala/com/wavesplatform/api/common/CommonAccountApiSpec.scala

@@ -10,9 +10,10 @@ import com.wavesplatform.lang.v1.compiler.TestCompiler
 import com.wavesplatform.lang.v1.traits.domain.{Lease, Recipient}
 import com.wavesplatform.settings.TestFunctionalitySettings
 import com.wavesplatform.state.{DataEntry, Diff, EmptyDataEntry, StringDataEntry, diffs}
-import com.wavesplatform.test.DomainPresets.RideV6
+import com.wavesplatform.test.DomainPresets.{RideV4, RideV6}
 import com.wavesplatform.test.FreeSpec
-import com.wavesplatform.transaction.TxHelpers.secondAddress
+import com.wavesplatform.transaction.TxHelpers.{data, secondAddress}
+import com.wavesplatform.transaction.TxVersion.V2
 import com.wavesplatform.transaction.{DataTransaction, GenesisTransaction, TxHelpers}
 import com.wavesplatform.{BlocksTransactionsHelpers, history}
 import monix.execution.Scheduler.Implicits.global
@@ -25,46 +26,33 @@ class CommonAccountApiSpec extends FreeSpec with WithDomain with BlocksTransacti
       val entry2 = StringDataEntry("test1", "test")
       val entry3 = StringDataEntry("test2", "test")
 
-      val preconditions = for {
-        acc <- accountGen
-        ts = System.currentTimeMillis()
-        fee <- smallFeeGen
-        genesis        = GenesisTransaction.create(acc.toAddress, diffs.ENOUGH_AMT, ts).explicitGet()
-        data1          = DataTransaction.selfSigned(1.toByte, acc, Seq(entry1), fee, ts).explicitGet()
-        data2          = DataTransaction.selfSigned(1.toByte, acc, Seq(entry2), fee, ts).explicitGet()
-        data3          = DataTransaction.selfSigned(1.toByte, acc, Seq(entry3), fee, ts).explicitGet()
-        data4          = DataTransaction.selfSigned(2.toByte, acc, Seq(EmptyDataEntry("test"), EmptyDataEntry("test1")), fee, ts).explicitGet()
-        data5          = DataTransaction.selfSigned(2.toByte, acc, Seq(EmptyDataEntry("test2"), entry1, entry2), fee, ts).explicitGet()
-        (block1, mbs1) = UnsafeBlocks.unsafeChainBaseAndMicro(history.randomSig, Seq(genesis), Seq(Seq(data1)), acc, 3, ts)
-        (block2, mbs2) = UnsafeBlocks.unsafeChainBaseAndMicro(mbs1.last.totalResBlockSig, Seq(data2), Seq(Seq(data3)), acc, 3, ts)
-        (block3, mbs3) = UnsafeBlocks.unsafeChainBaseAndMicro(mbs2.last.totalResBlockSig, Seq(data4), Seq(Seq(data5)), acc, 3, ts)
-      } yield (acc, block1, mbs1.head, block2, mbs2.head, block3, mbs3.head)
+      val acc     = accountGen.sample.get
+      val genesis = TxHelpers.genesis(acc.toAddress)
+      val data1   = data(acc, Seq(entry1))
+      val data2   = data(acc, Seq(entry2))
+      val data3   = data(acc, Seq(entry3))
+      val data4   = data(acc, Seq(EmptyDataEntry("test"), EmptyDataEntry("test1")), version = V2)
+      val data5   = data(acc, Seq(EmptyDataEntry("test2"), entry1, entry2), version = V2)
+
 
-      forAll(preconditions) { case (acc, block1, mb1, block2, mb2, block3, mb3) =>
         withDomain(
-          domainSettingsWithFS(
-            TestFunctionalitySettings.withFeatures(
-              BlockchainFeatures.NG,
-              BlockchainFeatures.DataTransaction,
-              BlockchainFeatures.BlockV5
-            )
-          )
+          RideV4
         ) { d =>
           val commonAccountsApi = CommonAccountsApi(() => d.blockchainUpdater.bestLiquidDiff.getOrElse(Diff.empty), d.db, () => d.blockchainUpdater)
           def dataList(): Set[DataEntry[_]] = commonAccountsApi.dataStream(acc.toAddress, None).toListL.runSyncUnsafe().toSet
 
-          d.appendBlock(block1)
-          d.appendMicroBlock(mb1)
+          d.appendBlock(genesis)
+          d.appendMicroBlock(data1)
           dataList() shouldBe Set(entry1)
-          d.appendBlock(block2)
+          d.appendBlock(data2)
           dataList() shouldBe Set(entry1, entry2)
-          d.appendMicroBlock(mb2)
+          d.appendMicroBlock(data3)
           dataList() shouldBe Set(entry1, entry2, entry3)
-          d.appendBlock(block3)
+          d.appendBlock(data4)
           dataList() shouldBe Set(entry3)
-          d.appendMicroBlock(mb3)
+          d.appendMicroBlock(data5)
           dataList() shouldBe Set(entry1, entry2)
-        }
+
       }
     }
 

node/src/test/scala/com/wavesplatform/history/BlockchainUpdaterSponsoredFeeBlockTest.scala

@@ -1,22 +1,22 @@
 package com.wavesplatform.history
 
-import com.wavesplatform.account.KeyPair
 import com.wavesplatform.common.state.ByteStr
 import com.wavesplatform.common.utils.EitherExt2
-import com.wavesplatform.crypto._
 import com.wavesplatform.features.BlockchainFeatures
 import com.wavesplatform.history.Domain.BlockchainUpdaterExt
 import com.wavesplatform.settings.{BlockchainSettings, WavesSettings}
-import com.wavesplatform.state._
-import com.wavesplatform.state.diffs._
-import com.wavesplatform.test._
+import com.wavesplatform.state.*
+import com.wavesplatform.state.diffs.*
+import com.wavesplatform.test.*
 import com.wavesplatform.transaction.Asset.Waves
 import com.wavesplatform.transaction.assets.{IssueTransaction, SponsorFeeTransaction}
-import com.wavesplatform.transaction.transfer._
+import com.wavesplatform.transaction.transfer.*
 import com.wavesplatform.transaction.{Asset, GenesisTransaction}
 import org.scalacheck.Gen
 
 class BlockchainUpdaterSponsoredFeeBlockTest extends PropSpec with DomainScenarioDrivenPropertyCheck {
+  private val time = new TestTime
+  private def ts   = time.getTimestamp()
 
   private val amtTx = 100000
 
@@ -26,7 +26,6 @@ class BlockchainUpdaterSponsoredFeeBlockTest extends PropSpec with DomainScenari
   val sponsorPreconditions: Gen[Setup] = for {
 
     master                      <- accountGen
-    ts                          <- timestampGen
     transferAssetWavesFee       <- smallFeeGen
     _                           <- accountGen
     alice                       <- accountGen
@@ -90,33 +89,29 @@ class BlockchainUpdaterSponsoredFeeBlockTest extends PropSpec with DomainScenari
 
   val SponsoredFeeActivatedAt0BlockchainSettings: BlockchainSettings = DefaultBlockchainSettings.copy(
     functionalitySettings = DefaultBlockchainSettings.functionalitySettings
-      .copy(featureCheckBlocksPeriod = 1, blocksForFeatureActivation = 1, preActivatedFeatures = Map(
+      .copy(
+        featureCheckBlocksPeriod = 1,
+        blocksForFeatureActivation = 1,
+        preActivatedFeatures = Map(
           BlockchainFeatures.FeeSponsorship.id -> 0,
           BlockchainFeatures.NG.id             -> 0,
           BlockchainFeatures.BlockV5.id        -> 0
-        ))
+        )
+      )
   )
 
   val SponsoredActivatedAt0WavesSettings: WavesSettings = settings.copy(blockchainSettings = SponsoredFeeActivatedAt0BlockchainSettings)
 
   property("not enough waves to sponsor sponsored tx") {
     scenario(sponsorPreconditions, SponsoredActivatedAt0WavesSettings) {
-      case (domain, (genesis, masterToAlice, feeAsset, sponsor, aliceToBob, bobToMaster, bobToMaster2)) =>
-        val (block0, microBlocks) = chainBaseAndMicro(randomSig, genesis, Seq(masterToAlice, feeAsset, sponsor).map(Seq(_)))
-        val block1 =
-          customBuildBlockOfTxs(microBlocks.last.totalResBlockSig, Seq.empty, KeyPair(Array.fill(KeyLength)(1: Byte)), 3: Byte, sponsor.timestamp + 1)
-        val block2 = customBuildBlockOfTxs(block1.id(), Seq.empty, KeyPair(Array.fill(KeyLength)(1: Byte)), 3: Byte, sponsor.timestamp + 1)
-        val block3 = buildBlockOfTxs(block2.id(), Seq(aliceToBob, bobToMaster))
-        val block4 = buildBlockOfTxs(block3.id(), Seq(bobToMaster2))
-
-        domain.blockchainUpdater.processBlock(block0) should beRight
-        domain.blockchainUpdater.processMicroBlock(microBlocks(0)) should beRight
-        domain.blockchainUpdater.processMicroBlock(microBlocks(1)) should beRight
-        domain.blockchainUpdater.processMicroBlock(microBlocks(2)) should beRight
-        domain.blockchainUpdater.processBlock(block1) should beRight
-        domain.blockchainUpdater.processBlock(block2) should beRight
-        domain.blockchainUpdater.processBlock(block3) should beRight
-        domain.blockchainUpdater.processBlock(block4) should produce("negative waves balance" /*"unavailable funds"*/ )
+      case (d, (genesis, masterToAlice, feeAsset, sponsor, aliceToBob, bobToMaster, bobToMaster2)) =>
+        d.appendBlock(genesis)
+        d.appendBlock()
+        d.appendMicroBlock(masterToAlice)
+        d.appendMicroBlock(feeAsset)
+        d.appendMicroBlock(sponsor)
+        d.appendBlock(aliceToBob, bobToMaster)
+        d.appendBlockE(bobToMaster2) should produce("negative waves balance" /*"unavailable funds"*/ )
     }
   }