From b8ad3fdbe96e95a554439e53691a9cce97f04aa9 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:26:53 +1100 Subject: [PATCH 01/16] chore: bump dependencies, including nim-ethers with chronos v4 support Bumps the following dependencies: - nim-ethers to commit 507ac6a4cc71cec9be7693fa393db4a49b52baf9 which contains a pinned nim-eth version. This is to be replaced by a versioned library, so it will be pinned to a particular version. There is a crucial fix in this version of ethers that fixes nonce management which is causing issues in the Codex testnet. - nim-json-rpc to v0.4.4 - nim-json-serialization to v0.2.8 - nim-serde to v1.2.2 - nim-serialization to v0.2.4 Currently, one of the integration tests is failing. --- codex/contracts/market.nim | 2 +- codex/contracts/marketplace.nim | 18 +++++++++--------- tests/contracts/testDeployment.nim | 2 +- tests/contracts/time.nim | 1 + vendor/nim-ethers | 2 +- vendor/nim-json-rpc | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-serde | 2 +- vendor/nim-serialization | 2 +- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 6dd7e2b3c..693830658 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -176,7 +176,7 @@ method fillSlot(market: OnChainMarket, method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = convertEthersError: - var freeSlot: Future[?TransactionResponse] + var freeSlot: Future[Confirmable] if rewardRecipient =? market.rewardRecipient: # If --reward-recipient specified, use it as the reward recipient, and use # the SP's address as the collateral recipient diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 6425bfa0c..f3bac79da 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -23,12 +23,12 @@ proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.} proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.} proc minCollateralThreshold*(marketplace: Marketplace): UInt256 {.contract, view.} -proc requestStorage*(marketplace: Marketplace, request: StorageRequest): ?TransactionResponse {.contract.} -proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): ?TransactionResponse {.contract.} -proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): ?TransactionResponse {.contract.} -proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): ?TransactionResponse {.contract.} -proc freeSlot*(marketplace: Marketplace, id: SlotId): ?TransactionResponse {.contract.} -proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): ?TransactionResponse {.contract.} +proc requestStorage*(marketplace: Marketplace, request: StorageRequest): Confirmable {.contract.} +proc fillSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256, proof: Groth16Proof): Confirmable {.contract.} +proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId): Confirmable {.contract.} +proc withdrawFunds*(marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address): Confirmable {.contract.} +proc freeSlot*(marketplace: Marketplace, id: SlotId): Confirmable {.contract.} +proc freeSlot*(marketplace: Marketplace, id: SlotId, rewardRecipient: Address, collateralRecipient: Address): Confirmable {.contract.} proc getRequest*(marketplace: Marketplace, id: RequestId): StorageRequest {.contract, view.} proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.} proc getActiveSlot*(marketplace: Marketplace, id: SlotId): Slot {.contract, view.} @@ -49,8 +49,8 @@ proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.} proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.} -proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): ?TransactionResponse {.contract.} -proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): ?TransactionResponse {.contract.} +proc submitProof*(marketplace: Marketplace, id: SlotId, proof: Groth16Proof): Confirmable {.contract.} +proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256): Confirmable {.contract.} -proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): ?TransactionResponse {.contract.} +proc reserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): Confirmable {.contract.} proc canReserveSlot*(marketplace: Marketplace, requestId: RequestId, slotIndex: UInt256): bool {.contract, view.} diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim index 4101b71a6..f89e28a8c 100644 --- a/tests/contracts/testDeployment.nim +++ b/tests/contracts/testDeployment.nim @@ -9,7 +9,7 @@ import ../checktest type MockProvider = ref object of Provider chainId*: UInt256 -method getChainId*(provider: MockProvider): Future[UInt256] {.async.} = +method getChainId*(provider: MockProvider): Future[UInt256] {.async: (raises:[ProviderError]).} = return provider.chainId proc configFactory(): CodexConf = diff --git a/tests/contracts/time.nim b/tests/contracts/time.nim index cd6aac1b8..11bd1b6d1 100644 --- a/tests/contracts/time.nim +++ b/tests/contracts/time.nim @@ -1,4 +1,5 @@ import pkg/ethers +import pkg/serde proc currentTime*(provider: Provider): Future[UInt256] {.async.} = return (!await provider.getBlock(BlockTag.pending)).timestamp diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 5b170adcb..507ac6a4c 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 5b170adcb1ffb1dbb273d1b7679bf3d9a08adb76 +Subproject commit 507ac6a4cc71cec9be7693fa393db4a49b52baf9 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 0bf2bcbe7..0408795be 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 0bf2bcbe74a18a3c7a709d57108bb7b51e748a92 +Subproject commit 0408795be95c00d75e96eaef6eae8a9c734014f5 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index bb53d49ca..5127b26ee 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit bb53d49caf2a6c6cf1df365ba84af93cdcfa7aa3 +Subproject commit 5127b26ee58076e9369e7c126c196793c2b12e73 diff --git a/vendor/nim-serde b/vendor/nim-serde index b1e5e5d39..83e4a2ccf 160000 --- a/vendor/nim-serde +++ b/vendor/nim-serde @@ -1 +1 @@ -Subproject commit b1e5e5d39a99ea56b750f6d9272dd319f4ad4291 +Subproject commit 83e4a2ccf621d3040c6e7e0267393ca2d205988e diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 384eb2561..f709bd9e1 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 384eb2561ee755446cff512a8e057325848b86a7 +Subproject commit f709bd9e16b1b6870fe3e4401196479e014a2ef6 From be0d7102ab18ecede59e5a053ef131153c494f35 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:13:25 +1100 Subject: [PATCH 02/16] fix integration test - When a state's run was cancelled, it was being caught as an error due to catching all CatchableErrors. This caused a state transition to SaleErrored, however cancellation of run was not actually an error. Handling this correctly fixed the issue. - Stopping of the clock was moved to after `HostInteractions` (sales) which avoided an assertion around getting time when the clock was not started. --- codex/node.nim | 6 ++--- codex/utils/asyncstatemachine.nim | 45 +++++++++++++++---------------- vendor/nim-ethers | 2 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/codex/node.nim b/codex/node.nim index 956acd49a..f180fd629 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -757,15 +757,15 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.discovery.isNil: await self.discovery.stop() - if not self.clock.isNil: - await self.clock.stop() - if clientContracts =? self.contracts.client: await clientContracts.stop() if hostContracts =? self.contracts.host: await hostContracts.stop() + if not self.clock.isNil: + await self.clock.stop() + if validatorContracts =? self.contracts.validator: await validatorContracts.stop() diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 3f10cfbed..8cd3390a4 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -67,23 +67,21 @@ proc run(machine: Machine, state: State) {.async.} = proc scheduler(machine: Machine) {.async.} = var running: Future[void] - try: - while machine.started: - let event = await machine.scheduled.get().track(machine) - if next =? event(machine.state): - if not running.isNil and not running.finished: - await running.cancelAndWait() - let fromState = if machine.state.isNil: "" else: $machine.state - machine.state = next - debug "enter state", state = fromState & " => " & $machine.state - running = machine.run(machine.state) - running - .track(machine) - .catch((err: ref CatchableError) => - machine.schedule(machine.onError(err)) - ) - except CancelledError: - discard + while machine.started: + let event = await machine.scheduled.get().track(machine) + if next =? event(machine.state): + if not running.isNil and not running.finished: + trace "cancelling current state", state = $machine.state + await running.cancelAndWait() + let fromState = if machine.state.isNil: "" else: $machine.state + machine.state = next + debug "enter state", state = fromState & " => " & $machine.state + try: + running = machine.run(machine.state).track(machine) + except CancelledError as e: + trace("run cancelled in state, swallowing", state = $machine.state) + except CatchableError as e: + machine.schedule(machine.onError(e)) proc start*(machine: Machine, initialState: State) = if machine.started: @@ -93,12 +91,13 @@ proc start*(machine: Machine, initialState: State) = machine.scheduled = newAsyncQueue[Event]() machine.started = true - machine.scheduler() - .track(machine) - .catch((err: ref CatchableError) => - error("Error in scheduler", error = err.msg) - ) - machine.schedule(Event.transition(machine.state, initialState)) + try: + discard machine.scheduler().track(machine) + machine.schedule(Event.transition(machine.state, initialState)) + except CancelledError as e: + discard + except CatchableError as e: + error("Error in scheduler", error = e.msg) proc stop*(machine: Machine) {.async.} = if not machine.started: diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 507ac6a4c..d11c3cb1e 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 507ac6a4cc71cec9be7693fa393db4a49b52baf9 +Subproject commit d11c3cb1e69c9cce3ddb5c026594ac37ebed373b From 2b7e4c7f0b6bf1d9d5d8be834b1ecd80852352a6 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:08:43 +1100 Subject: [PATCH 03/16] bump ethers to include nonce fix and filter not found fix --- vendor/nim-ethers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-ethers b/vendor/nim-ethers index d11c3cb1e..765379a66 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit d11c3cb1e69c9cce3ddb5c026594ac37ebed373b +Subproject commit 765379a662e93edf5217b8656c0af4dcf6865fd2 From b61605f6ffa4da4678ce6a2dd05da07ee5a6434d Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:19:58 +1100 Subject: [PATCH 04/16] bump ethers: fixes missing symbol not exported in ethers --- tests/contracts/time.nim | 2 +- vendor/nim-ethers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/contracts/time.nim b/tests/contracts/time.nim index 11bd1b6d1..ae4487895 100644 --- a/tests/contracts/time.nim +++ b/tests/contracts/time.nim @@ -1,5 +1,5 @@ import pkg/ethers -import pkg/serde +import pkg/serde/json proc currentTime*(provider: Provider): Future[UInt256] {.async.} = return (!await provider.getBlock(BlockTag.pending)).timestamp diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 765379a66..5f64bbc05 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 765379a662e93edf5217b8656c0af4dcf6865fd2 +Subproject commit 5f64bbc05bb7ee126c33c84dd16a15bc53232ca7 From 30213b474bc7853d49ee6e4e85894ce261e515f9 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:36:22 +1100 Subject: [PATCH 05/16] Fix cirdl test imports/exports --- tests/tools/cirdl/testcirdl.nim | 11 ++++++----- vendor/nim-ethers | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/tools/cirdl/testcirdl.nim b/tests/tools/cirdl/testcirdl.nim index d8e71e791..b639be282 100644 --- a/tests/tools/cirdl/testcirdl.nim +++ b/tests/tools/cirdl/testcirdl.nim @@ -2,10 +2,11 @@ import std/os import std/osproc import std/options import pkg/chronos -import codex/contracts -import ../../integration/marketplacesuite +import pkg/codex/contracts +import ../../asynctest +import ../../contracts/deployment -marketplacesuite "tools/cirdl": +suite "tools/cirdl": const cirdl = "build" / "cirdl" workdir = "." @@ -14,11 +15,11 @@ marketplacesuite "tools/cirdl": let circuitPath = "testcircuitpath" rpcEndpoint = "ws://localhost:8545" - marketplaceAddress = $marketplace.address + marketplaceAddress = Marketplace.address discard existsOrCreateDir(circuitPath) - let args = [circuitPath, rpcEndpoint, marketplaceAddress] + let args = [circuitPath, rpcEndpoint, $marketplaceAddress] let process = osproc.startProcess( cirdl, diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 5f64bbc05..fcd9bbb9b 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 5f64bbc05bb7ee126c33c84dd16a15bc53232ca7 +Subproject commit fcd9bbb9b927f4cb69cb806cbd4c23e52fc6b699 From 1019d976333a79bb13bd6c3777a91140f97e3578 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:36:39 +1100 Subject: [PATCH 06/16] Debugging in ci --- tests/codex/utils/testasyncstatemachine.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/codex/utils/testasyncstatemachine.nim b/tests/codex/utils/testasyncstatemachine.nim index a114f7c93..f65bb7ea6 100644 --- a/tests/codex/utils/testasyncstatemachine.nim +++ b/tests/codex/utils/testasyncstatemachine.nim @@ -25,6 +25,7 @@ method run(state: State1, machine: Machine): Future[?State] {.async.} = method run(state: State2, machine: Machine): Future[?State] {.async.} = inc runs[1] + echo "State2 run, runs: ", $runs try: await sleepAsync(1.hours) except CancelledError: @@ -36,6 +37,7 @@ method run(state: State3, machine: Machine): Future[?State] {.async.} = method run(state: State4, machine: Machine): Future[?State] {.async.} = inc runs[3] + echo "State4 run, runs: ", $runs raise newException(ValueError, "failed") method onMoveToNextStateEvent*(state: State): ?State {.base, upraises:[].} = @@ -58,6 +60,7 @@ method onError(state: State3, error: ref CatchableError): ?State = method onError(state: State4, error: ref CatchableError): ?State = inc errors[3] + echo "State4 onError, errors: ", $errors some State(State2.new()) asyncchecksuite "async state machines": From 4a4880f8b7d42933864954bc4ebeb3342099673c Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:54:05 +1100 Subject: [PATCH 07/16] Handle CancelledErrors for state.run in one place only --- codex/utils/asyncstatemachine.nim | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 8cd3390a4..3f15af319 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -59,11 +59,8 @@ proc onError(machine: Machine, error: ref CatchableError): Event = state.onError(error) proc run(machine: Machine, state: State) {.async.} = - try: - if next =? await state.run(machine): - machine.schedule(Event.transition(state, next)) - except CancelledError: - discard + if next =? await state.run(machine): + machine.schedule(Event.transition(state, next)) proc scheduler(machine: Machine) {.async.} = var running: Future[void] @@ -76,12 +73,14 @@ proc scheduler(machine: Machine) {.async.} = let fromState = if machine.state.isNil: "" else: $machine.state machine.state = next debug "enter state", state = fromState & " => " & $machine.state - try: - running = machine.run(machine.state).track(machine) - except CancelledError as e: - trace("run cancelled in state, swallowing", state = $machine.state) - except CatchableError as e: - machine.schedule(machine.onError(e)) + running = machine.run(machine.state) + running + .track(machine) + .cancelled(proc() = trace "state.run cancelled, swallowing", state = $machine.state) + .catch(proc(err: ref CatchableError) = + trace "error caught in state.run, calling state.onError", state = $machine.state + machine.schedule(machine.onError(err)) + ) proc start*(machine: Machine, initialState: State) = if machine.started: From f9171002680074979112bea77eaf5b0c12233edd Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:49:09 +1100 Subject: [PATCH 08/16] Rename `config` to `configuration` There was a symbol clash preventing compilation and it was easiest to rename `config` to `configuration` in the contracts. Not even remotely ideal, but it was the only way. --- codex/contracts/market.nim | 8 ++++---- codex/contracts/marketplace.nim | 2 +- tests/integration/marketplacesuite.nim | 4 ++-- vendor/codex-contracts-eth | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 693830658..b8d1da686 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -56,7 +56,7 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} = discard await token.increaseAllowance(market.contract.address(), amount).confirm(0) method getZkeyHash*(market: OnChainMarket): Future[?string] {.async.} = - let config = await market.contract.config() + let config = await market.contract.configuration() return some config.proofs.zkeyHash method getSigner*(market: OnChainMarket): Future[Address] {.async.} = @@ -65,18 +65,18 @@ method getSigner*(market: OnChainMarket): Future[Address] {.async.} = method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() let period = config.proofs.period return Periodicity(seconds: period) method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() return config.proofs.timeout method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} = convertEthersError: - let config = await market.contract.config() + let config = await market.contract.configuration() return config.proofs.downtime method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} = diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index f3bac79da..020f501eb 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -17,7 +17,7 @@ export requests type Marketplace* = ref object of Contract -proc config*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} +proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} proc token*(marketplace: Marketplace): Address {.contract, view.} proc slashMisses*(marketplace: Marketplace): UInt256 {.contract, view.} proc slashPercentage*(marketplace: Marketplace): UInt256 {.contract, view.} diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 2b81bdd8f..01bfd712c 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -1,5 +1,5 @@ import pkg/chronos -import pkg/ethers/erc20 +import pkg/ethers from pkg/libp2p import Cid import pkg/codex/contracts/marketplace as mp import pkg/codex/periods @@ -85,7 +85,7 @@ template marketplacesuite*(name: string, body: untyped) = marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner()) let tokenAddress = await marketplace.token() token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let config = await mp.config(marketplace) + let config = await marketplace.configuration() period = config.proofs.period.truncate(uint64) periodicity = Periodicity(seconds: period.u256) diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 997696a20..058642598 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 997696a20e0976011cdbc2f0ff3a844672056ba2 +Subproject commit 0586425980ae56dd438aaa8963acd0ec185c2e83 From 70143f4f7252c05912a76cb1b36ac8c665d4d9ee Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:49:42 +1100 Subject: [PATCH 09/16] bump ethers to latest Prevents an issue were `JsonNode.items` symbol could not be found --- vendor/nim-ethers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-ethers b/vendor/nim-ethers index fcd9bbb9b..6523e70ea 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit fcd9bbb9b927f4cb69cb806cbd4c23e52fc6b699 +Subproject commit 6523e70eafecc3b9728a491058818e685bd384fb From 4260cdd514997641dae5b8610ead3594f1fcc75d Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:13:18 +1100 Subject: [PATCH 10/16] More changes to support `config` > `configuration` --- tests/contracts/testContracts.nim | 2 +- tests/contracts/testMarket.nim | 6 +++--- tools/cirdl/cirdl.nim | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index b098b80f4..bbbf41aaf 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -38,7 +38,7 @@ ethersuite "Marketplace contracts": let tokenAddress = await marketplace.token() token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let config = await marketplace.config() + let config = await marketplace.configuration() periodicity = Periodicity(seconds: config.proofs.period) request = StorageRequest.example diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 66088b714..a836628c3 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -34,7 +34,7 @@ ethersuite "On-Chain Market": setup: let address = Marketplace.address(dummyVerifier = true) marketplace = Marketplace.new(address, ethProvider.getSigner()) - let config = await marketplace.config() + let config = await marketplace.configuration() hostRewardRecipient = accounts[2] market = OnChainMarket.new(marketplace) @@ -76,13 +76,13 @@ ethersuite "On-Chain Market": test "can retrieve proof periodicity": let periodicity = await market.periodicity() - let config = await marketplace.config() + let config = await marketplace.configuration() let periodLength = config.proofs.period check periodicity.seconds == periodLength test "can retrieve proof timeout": let proofTimeout = await market.proofTimeout() - let config = await marketplace.config() + let config = await marketplace.configuration() check proofTimeout == config.proofs.timeout test "supports marketplace requests": diff --git a/tools/cirdl/cirdl.nim b/tools/cirdl/cirdl.nim index d1533ceb0..110516723 100644 --- a/tools/cirdl/cirdl.nim +++ b/tools/cirdl/cirdl.nim @@ -34,7 +34,7 @@ proc getCircuitHash(rpcEndpoint: string, marketplaceAddress: string): Future[?!s return failure("Invalid address: " & marketplaceAddress) let marketplace = Marketplace.new(address, provider) - let config = await marketplace.config() + let config = await marketplace.configuration() return success config.proofs.zkeyHash proc formatUrl(hash: string): string = From 8806166a26a133708dd26ba43ca8ad80cc363af9 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:53:33 +1100 Subject: [PATCH 11/16] cleanup --- codex/utils/asyncstatemachine.nim | 14 +++++++------- tests/codex/utils/testasyncstatemachine.nim | 3 --- tests/integration/marketplacesuite.nim | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 3f15af319..2d02110d6 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -90,13 +90,13 @@ proc start*(machine: Machine, initialState: State) = machine.scheduled = newAsyncQueue[Event]() machine.started = true - try: - discard machine.scheduler().track(machine) - machine.schedule(Event.transition(machine.state, initialState)) - except CancelledError as e: - discard - except CatchableError as e: - error("Error in scheduler", error = e.msg) + machine.scheduler() + .track(machine) + .cancelled(proc() = trace("machine.scheduler cancelled, swallowing")) + .catch((err: ref CatchableError) => + error("Error in scheduler", error = err.msg) + ) + machine.schedule(Event.transition(machine.state, initialState)) proc stop*(machine: Machine) {.async.} = if not machine.started: diff --git a/tests/codex/utils/testasyncstatemachine.nim b/tests/codex/utils/testasyncstatemachine.nim index f65bb7ea6..a114f7c93 100644 --- a/tests/codex/utils/testasyncstatemachine.nim +++ b/tests/codex/utils/testasyncstatemachine.nim @@ -25,7 +25,6 @@ method run(state: State1, machine: Machine): Future[?State] {.async.} = method run(state: State2, machine: Machine): Future[?State] {.async.} = inc runs[1] - echo "State2 run, runs: ", $runs try: await sleepAsync(1.hours) except CancelledError: @@ -37,7 +36,6 @@ method run(state: State3, machine: Machine): Future[?State] {.async.} = method run(state: State4, machine: Machine): Future[?State] {.async.} = inc runs[3] - echo "State4 run, runs: ", $runs raise newException(ValueError, "failed") method onMoveToNextStateEvent*(state: State): ?State {.base, upraises:[].} = @@ -60,7 +58,6 @@ method onError(state: State3, error: ref CatchableError): ?State = method onError(state: State4, error: ref CatchableError): ?State = inc errors[3] - echo "State4 onError, errors: ", $errors some State(State2.new()) asyncchecksuite "async state machines": diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 01bfd712c..d3b1ef577 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -1,5 +1,5 @@ import pkg/chronos -import pkg/ethers +import pkg/ethers/erc20 from pkg/libp2p import Cid import pkg/codex/contracts/marketplace as mp import pkg/codex/periods From a180815ab3500fa3635fd32e52d771db072b546d Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:20:50 +1100 Subject: [PATCH 12/16] testing to see if this fixes failure in ci --- codex/utils/asyncstatemachine.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 2d02110d6..3f15af319 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -90,13 +90,13 @@ proc start*(machine: Machine, initialState: State) = machine.scheduled = newAsyncQueue[Event]() machine.started = true - machine.scheduler() - .track(machine) - .cancelled(proc() = trace("machine.scheduler cancelled, swallowing")) - .catch((err: ref CatchableError) => - error("Error in scheduler", error = err.msg) - ) - machine.schedule(Event.transition(machine.state, initialState)) + try: + discard machine.scheduler().track(machine) + machine.schedule(Event.transition(machine.state, initialState)) + except CancelledError as e: + discard + except CatchableError as e: + error("Error in scheduler", error = e.msg) proc stop*(machine: Machine) {.async.} = if not machine.started: From 9204eef75f22c755e010bd5a9ad8aebd4f741907 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:47:33 +1100 Subject: [PATCH 13/16] Add integration test that checks for "nonce too high" in logs --- tests/integration/codexprocess.nim | 13 +++- tests/integration/hardhatprocess.nim | 19 ++++++ tests/integration/multinodes.nim | 20 ++++++ tests/integration/nodeconfig.nim | 34 ---------- tests/integration/nodeprocess.nim | 3 + tests/integration/testslotreservations.nim | 79 ++++++++++++++++++++++ tests/testIntegration.nim | 1 + 7 files changed, 134 insertions(+), 35 deletions(-) delete mode 100644 tests/integration/nodeconfig.nim create mode 100644 tests/integration/testslotreservations.nim diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index ce6334340..4e707549a 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -5,6 +5,7 @@ import pkg/chronicles import pkg/chronos/asyncproc import pkg/ethers import pkg/libp2p +import pkg/stew/io2 import std/os import std/strutils import codex/conf @@ -40,6 +41,16 @@ method outputLineEndings(node: CodexProcess): string = method onOutputLineCaptured(node: CodexProcess, line: string) = discard +method logFileContains*(node: CodexProcess, text: string): bool = + let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) + without logFile =? config.logFile.?string: + raiseAssert "codex node does have a --log-file option set (use .withLogFile())" + + without logContents =? logFile.readAllChars: + raiseAssert "failed to open codex log file, aborting" + + return logContents.contains(text) + proc dataDir(node: CodexProcess): string = let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) return config.dataDir.string @@ -73,4 +84,4 @@ method stop*(node: CodexProcess) {.async.} = node.client = none CodexClient method removeDataDir*(node: CodexProcess) = - removeDir(node.dataDir) + os.removeDir(node.dataDir) diff --git a/tests/integration/hardhatprocess.nim b/tests/integration/hardhatprocess.nim index 0e88aa786..84cf4132b 100644 --- a/tests/integration/hardhatprocess.nim +++ b/tests/integration/hardhatprocess.nim @@ -40,6 +40,25 @@ method processOptions(node: HardhatProcess): set[AsyncProcessOption] = method outputLineEndings(node: HardhatProcess): string = return "\n" +method logFileContains*(hardhat: HardhatProcess, text: string): bool = + without fileHandle =? hardhat.logFile: + raiseAssert "failed to open hardhat log file, aborting" + + without fileSize =? fileHandle.getFileSize: + raiseAssert "failed to get current hardhat log file size, aborting" + + if checkFileSize(fileSize).isErr: + raiseAssert "file size too big for nim indexing" + + var data = "" + data.setLen(fileSize) + + without bytesRead =? readFile(fileHandle, + data.toOpenArray(0, len(data) - 1)): + raiseAssert "unable to read hardhat log, aborting" + + return data.contains(text) + proc openLogFile(node: HardhatProcess, logFilePath: string): IoHandle = let logFileHandle = openFile( logFilePath, diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 1ad16a384..97277beb4 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -222,6 +222,26 @@ template multinodesuite*(name: string, body: untyped) = return await newCodexProcess(validatorIdx, config, Role.Validator) + + proc searchLogs(role: Role, text: string): seq[bool] = + var hits: seq[bool] = @[] + if role == Role.Hardhat: + return @[hardhat().logFileContains(text)] + elif role == Role.Client: + for client in clients(): + hits.add client.logFileContains(text) + else: + for provider in providers(): + hits.add provider.logFileContains(text) + + return hits + + proc logsContain(role: Role, text: string): bool = + return searchLogs(role, text).allIt(it) + + proc logsDoNotContain(role: Role, text: string): bool = + return searchLogs(role, text).allIt(not it) + proc teardownImpl() {.async.} = for nodes in @[validators(), clients(), providers()]: for node in nodes: diff --git a/tests/integration/nodeconfig.nim b/tests/integration/nodeconfig.nim deleted file mode 100644 index d6adb80fa..000000000 --- a/tests/integration/nodeconfig.nim +++ /dev/null @@ -1,34 +0,0 @@ -import pkg/chronicles -import pkg/questionable - -export chronicles - -type - NodeConfig* = ref object of RootObj - logFile*: bool - logLevel*: ?LogLevel - debugEnabled*: bool - -proc debug*[T: NodeConfig](config: T, enabled = true): T = - ## output log in stdout - var startConfig = config - startConfig.debugEnabled = enabled - return startConfig - -proc withLogFile*[T: NodeConfig]( - config: T, - logToFile: bool = true -): T = - - var startConfig = config - startConfig.logFile = logToFile - return startConfig - -proc withLogLevel*[T: NodeConfig]( - config: NodeConfig, - level: LogLevel -): T = - - var startConfig = config - startConfig.logLevel = some level - return startConfig diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index 61947c20c..d8367c14f 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -44,6 +44,9 @@ method outputLineEndings(node: NodeProcess): string {.base.} = method onOutputLineCaptured(node: NodeProcess, line: string) {.base.} = raiseAssert "not implemented" +method logFileContains*(hardhat: NodeProcess): bool {.base.} = + raiseAssert "not implemented" + method start*(node: NodeProcess) {.base, async.} = logScope: nodeName = node.name diff --git a/tests/integration/testslotreservations.nim b/tests/integration/testslotreservations.nim new file mode 100644 index 000000000..9fb54f2f1 --- /dev/null +++ b/tests/integration/testslotreservations.nim @@ -0,0 +1,79 @@ +import pkg/stew/byteutils +import pkg/codex/units +import ../examples +import ../contracts/time +import ../contracts/deployment +import ./marketplacesuite +import ./nodeconfigs +import ./hardhatconfig + +marketplacesuite "Slot reservations": + + test "nonce does not go too high when reserving slots", + NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: HardhatConfig() + .withLogFile() + .some, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output.debug() + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("node", "erasure", "marketplace") + .some, + + providers: + CodexConfigs.init(nodes=6) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("node", "marketplace", "sales", "reservations", "proving", "ethers", "statemachine") + .some, + ): + let reward = 400.u256 + let duration = 50.periods + let collateral = 200.u256 + let expiry = 30.periods + let data = await RandomChunker.example(blocks=8) + let client = clients()[0] + let clientApi = client.client + + # provider makes storage available + for i in 0.. Date: Tue, 29 Oct 2024 19:10:14 +1100 Subject: [PATCH 14/16] Do not attempt to start hardhat node since CI already started one --- tests/integration/testslotreservations.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/testslotreservations.nim b/tests/integration/testslotreservations.nim index 9fb54f2f1..924a46205 100644 --- a/tests/integration/testslotreservations.nim +++ b/tests/integration/testslotreservations.nim @@ -12,9 +12,7 @@ marketplacesuite "Slot reservations": test "nonce does not go too high when reserving slots", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig() - .withLogFile() - .some, + hardhat: HardhatConfig.none, clients: CodexConfigs.init(nodes=1) From 7b015a9805d59452a2a4efb1663826981fc314ea Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:56:01 +1100 Subject: [PATCH 15/16] debugging in ci --- tests/integration/codexprocess.nim | 4 ++-- tests/integration/testslotreservations.nim | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index 4e707549a..e9b55f823 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -46,8 +46,8 @@ method logFileContains*(node: CodexProcess, text: string): bool = without logFile =? config.logFile.?string: raiseAssert "codex node does have a --log-file option set (use .withLogFile())" - without logContents =? logFile.readAllChars: - raiseAssert "failed to open codex log file, aborting" + without logContents =? logFile.readAllChars, error: + raiseAssert "failed to open codex log file, aborting, error: " & error.msg return logContents.contains(text) diff --git a/tests/integration/testslotreservations.nim b/tests/integration/testslotreservations.nim index 924a46205..44a01733e 100644 --- a/tests/integration/testslotreservations.nim +++ b/tests/integration/testslotreservations.nim @@ -51,10 +51,7 @@ marketplacesuite "Slot reservations": var slotIdxFilled: seq[UInt256] = @[] proc onSlotFilled(event: SlotFilled) = slotIdxFilled.add event.slotIndex - proc onSlotReservationsFull(event: SlotReservationsFull) = - echo "Slot reservations full for slot ", event.slotIndex - let subscriptionFull = await marketplace.subscribe(SlotReservationsFull, onSlotReservationsFull) let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) # client requests storage but requires multiple slots to host the content @@ -74,4 +71,3 @@ marketplacesuite "Slot reservations": check logsDoNotContain(Role.Provider, "Nonce too high") await subscription.unsubscribe() - await subscriptionFull.unsubscribe() From bab9c1f6946c90f0d040da3a4803916c5f9b14ee Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:59:18 +1100 Subject: [PATCH 16/16] debugging on ci --- tests/integration/codexprocess.nim | 8 ++++++-- tests/integration/testslotreservations.nim | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index e9b55f823..e8f404ccf 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -46,8 +46,12 @@ method logFileContains*(node: CodexProcess, text: string): bool = without logFile =? config.logFile.?string: raiseAssert "codex node does have a --log-file option set (use .withLogFile())" - without logContents =? logFile.readAllChars, error: - raiseAssert "failed to open codex log file, aborting, error: " & error.msg + let resLogContents = logFile.readAllChars + if resLogContents.isErr: + # without logContents =? logFile.readAllChars: + raiseAssert "failed to open codex log file, aborting (log path: " & logFile & ")" + + let logContents = resLogContents.value return logContents.contains(text) diff --git a/tests/integration/testslotreservations.nim b/tests/integration/testslotreservations.nim index 44a01733e..f8c29beef 100644 --- a/tests/integration/testslotreservations.nim +++ b/tests/integration/testslotreservations.nim @@ -68,6 +68,7 @@ marketplacesuite "Slot reservations": # wait until all slots filled check eventually(slotIdxFilled.len == 5, timeout=expiry.int * 1000) - check logsDoNotContain(Role.Provider, "Nonce too high") + teardown: + check logsDoNotContain(Role.Provider, "Nonce too high") await subscription.unsubscribe()