Merge branch 'master' into feature/ceremony-files

# Conflicts:
#	vendor/codex-contracts-eth

Author: benbierens

codex/contracts/market.nim

@@ -126,6 +126,11 @@ method getRequestEnd*(market: OnChainMarket,
   convertEthersError:
     return await market.contract.requestEnd(id)
 
+method requestExpiresAt*(market: OnChainMarket,
+                      id: RequestId): Future[SecondsSince1970] {.async.} =
+  convertEthersError:
+    return await market.contract.requestExpiry(id)
+
 method getHost(market: OnChainMarket,
                requestId: RequestId,
                slotIndex: UInt256): Future[?Address] {.async.} =

codex/contracts/marketplace.nim

@@ -55,6 +55,7 @@ proc mySlots*(marketplace: Marketplace): seq[SlotId] {.contract, view.}
 proc requestState*(marketplace: Marketplace, requestId: RequestId): RequestState {.contract, view.}
 proc slotState*(marketplace: Marketplace, slotId: SlotId): SlotState {.contract, view.}
 proc requestEnd*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
+proc requestExpiry*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
 
 proc proofTimeout*(marketplace: Marketplace): UInt256 {.contract, view.}
 

codex/market.nim

@@ -84,6 +84,10 @@ method getRequestEnd*(market: Market,
                       id: RequestId): Future[SecondsSince1970] {.base, async.} =
   raiseAssert("not implemented")
 
+method requestExpiresAt*(market: Market,
+                      id: RequestId): Future[SecondsSince1970] {.base, async.} =
+  raiseAssert("not implemented")
+
 method getHost*(market: Market,
                 requestId: RequestId,
                 slotIndex: UInt256): Future[?Address] {.base, async.} =

codex/purchasing.nim

@@ -18,18 +18,15 @@ type
     clock: Clock
     purchases: Table[PurchaseId, Purchase]
     proofProbability*: UInt256
-    requestExpiryInterval*: UInt256
   PurchaseTimeout* = Timeout
 
 const DefaultProofProbability = 100.u256
-const DefaultRequestExpiryInterval = (10 * 60).u256
 
 proc new*(_: type Purchasing, market: Market, clock: Clock): Purchasing =
   Purchasing(
     market: market,
     clock: clock,
     proofProbability: DefaultProofProbability,
-    requestExpiryInterval: DefaultRequestExpiryInterval,
   )
 
 proc load*(purchasing: Purchasing) {.async.} =
@@ -52,8 +49,6 @@ proc populate*(purchasing: Purchasing,
   result = request
   if result.ask.proofProbability == 0.u256:
     result.ask.proofProbability = purchasing.proofProbability
-  if result.expiry == 0.u256:
-    result.expiry = (purchasing.clock.now().u256 + purchasing.requestExpiryInterval)
   if result.nonce == Nonce.default:
     var id = result.nonce.toArray
     doAssert randomBytes(id) == 32

codex/purchasing/states/submitted.nim

@@ -34,7 +34,7 @@ method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.
     await subscription.unsubscribe()
 
   proc withTimeout(future: Future[void]) {.async.} =
-    let expiry = request.expiry.truncate(int64) + 1
+    let expiry = (await market.requestExpiresAt(request.id)) + 1
     trace "waiting for request fulfillment or expiry", expiry
     await future.withTimeout(clock, expiry)
 

codex/rest/api.nim

@@ -399,7 +399,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
       ## duration         - the duration of the request in seconds
       ## proofProbability - how often storage proofs are required
       ## reward           - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
-      ## expiry           - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
+      ## expiry           - specifies threshold in seconds from now when the request expires if the Request does not find requested amount of nodes to host the data
       ## nodes            - number of nodes the content should be stored on
       ## tolerance        - allowed number of nodes that can be lost before content is lost
       ## colateral        - requested collateral from hosts when they fill slot
@@ -425,15 +425,8 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
         without expiry =? params.expiry:
           return RestApiResponse.error(Http400, "Expiry required")
 
-        if node.clock.isNil:
-          return RestApiResponse.error(Http500)
-
-        if expiry <= node.clock.now.u256:
-          return RestApiResponse.error(Http400, "Expiry needs to be in future. Now: " & $node.clock.now)
-
-        let expiryLimit = node.clock.now.u256 + params.duration
-        if expiry > expiryLimit:
-          return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration). Limit: " & $expiryLimit)
+        if expiry <= 0 or expiry >= params.duration:
+          return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration")
 
         without purchaseId =? await node.requestStorage(
           cid,
@@ -494,7 +487,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
 
 proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
   ## various node management api's
-  ## 
+  ##
   router.api(
     MethodGet,
     "/api/codex/v1/spr") do () -> RestApiResponse:

codex/sales/salesagent.nim

@@ -72,8 +72,11 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
     without request =? data.request:
       return
 
+    let market = agent.context.market
+    let expiry = await market.requestExpiresAt(data.requestId)
+
     while true:
-      let deadline = max(clock.now, request.expiry.truncate(int64)) + 1
+      let deadline = max(clock.now, expiry) + 1
       trace "Waiting for request to be cancelled", now=clock.now, expiry=deadline
       await clock.waitUntil(deadline)
 

codex/validation.nim

@@ -34,6 +34,12 @@ proc new*(
 proc slots*(validation: Validation): seq[SlotId] =
   validation.slots.toSeq
 
+proc iterateSlots(validation: Validation, action: proc(s: SlotId): Future[void] {.async.}) {.async.} =
+  # Copy of hashSet, for iteration.
+  let slots = validation.slots
+  for slotId in slots:
+    await action(slotId)
+
 proc getCurrentPeriod(validation: Validation): UInt256 =
   return validation.periodicity.periodOf(validation.clock.now().u256)
 
@@ -55,11 +61,12 @@ proc subscribeSlotFilled(validation: Validation) {.async.} =
 
 proc removeSlotsThatHaveEnded(validation: Validation) {.async.} =
   var ended: HashSet[SlotId]
-  for slotId in validation.slots:
+  proc onSlot(slotId: SlotId) {.async.} =
     let state = await validation.market.slotState(slotId)
     if state != SlotState.Filled:
       trace "Removing slot", slotId
       ended.incl(slotId)
+  await validation.iterateSlots(onSlot)
   validation.slots.excl(ended)
 
 proc markProofAsMissing(validation: Validation,
@@ -81,9 +88,10 @@ proc markProofAsMissing(validation: Validation,
     error "Marking proof as missing failed", msg = e.msg
 
 proc markProofsAsMissing(validation: Validation) {.async.} =
-  for slotId in validation.slots:
+  proc onSlot(slotId: SlotId) {.async.} =
     let previousPeriod = validation.getCurrentPeriod() - 1
     await validation.markProofAsMissing(slotId, previousPeriod)
+  await validation.iterateSlots(onSlot)
 
 proc run(validation: Validation) {.async.} =
   trace "Validation started"

openapi.yaml

@@ -213,8 +213,7 @@ components:
           description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
         expiry:
           type: string
-          description: Number as decimal string that represents expiry time of the request (in unix timestamp)
-
+          description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
     StorageAsk:
       type: object
       required:

tests/codex/helpers/mockmarket.nim

@@ -20,6 +20,7 @@ type
     activeSlots*: Table[Address, seq[SlotId]]
     requested*: seq[StorageRequest]
     requestEnds*: Table[RequestId, SecondsSince1970]
+    requestExpiry*: Table[RequestId, SecondsSince1970]
     requestState*: Table[RequestId, RequestState]
     slotState*: Table[SlotId, SlotState]
     fulfilled*: seq[Fulfillment]
@@ -165,6 +166,10 @@ method getRequestEnd*(market: MockMarket,
                       id: RequestId): Future[SecondsSince1970] {.async.} =
   return market.requestEnds[id]
 
+method requestExpiresAt*(market: MockMarket,
+                      id: RequestId): Future[SecondsSince1970] {.async.} =
+  return market.requestExpiry[id]
+
 method getHost*(market: MockMarket,
                requestId: RequestId,
                slotIndex: UInt256): Future[?Address] {.async.} =

tests/codex/sales/testsales.nim

@@ -473,6 +473,9 @@ asyncchecksuite "Sales":
     check eventually (await reservations.all(Availability)).get == @[availability]
 
   test "makes storage available again when request expires":
+    let expiry = getTime().toUnix() + 10
+    market.requestExpiry[request.id] = expiry
+
     let origSize = availability.freeSize
     sales.onStore = proc(request: StorageRequest,
                          slot: UInt256,
@@ -486,11 +489,14 @@ asyncchecksuite "Sales":
     # would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation.
     await sleepAsync(chronos.milliseconds(100))
     market.requestState[request.id]=RequestState.Cancelled
-    clock.set(request.expiry.truncate(int64)+1)
+    clock.set(expiry + 1)
     check eventually (await reservations.all(Availability)).get == @[availability]
     check getAvailability().freeSize == origSize
 
   test "verifies that request is indeed expired from onchain before firing onCancelled":
+    let expiry = getTime().toUnix() + 10
+    market.requestExpiry[request.id] = expiry
+
     let origSize = availability.freeSize
     sales.onStore = proc(request: StorageRequest,
                          slot: UInt256,
@@ -504,7 +510,7 @@ asyncchecksuite "Sales":
     # If we would not await, then the `clock.set` would run "too fast" as the `subscribeCancellation()`
     # would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation.
     await sleepAsync(chronos.milliseconds(100))
-    clock.set(request.expiry.truncate(int64)+1)
+    clock.set(expiry + 1)
     check getAvailability().freeSize == 0
 
     market.requestState[request.id]=RequestState.Cancelled # Now "on-chain" is also expired

tests/codex/sales/testsalesagent.nim

@@ -41,19 +41,7 @@ method run*(state: MockErrorState, machine: Machine): Future[?State] {.async.} =
   raise newException(ValueError, "failure")
 
 asyncchecksuite "Sales agent":
-  var request = StorageRequest(
-    ask: StorageAsk(
-      slots: 4,
-      slotSize: 100.u256,
-      duration: 60.u256,
-      reward: 10.u256,
-    ),
-    content: StorageContent(
-      cid: "some cid"
-    ),
-    expiry: (getTime() + initDuration(hours=1)).toUnix.u256
-  )
-
+  let request = StorageRequest.example
   var agent: SalesAgent
   var context: SalesContext
   var slotIndex: UInt256
@@ -62,6 +50,7 @@ asyncchecksuite "Sales agent":
 
   setup:
     market = MockMarket.new()
+    market.requestExpiry[request.id] = getTime().toUnix() + request.expiry.truncate(int64)
     clock = MockClock.new()
     context = SalesContext(market: market, clock: clock)
     slotIndex = 0.u256
@@ -109,15 +98,15 @@ asyncchecksuite "Sales agent":
     agent.start(MockState.new())
     await agent.subscribe()
     market.requestState[request.id] = RequestState.Cancelled
-    clock.set(request.expiry.truncate(int64) + 1)
+    clock.set(market.requestExpiry[request.id] + 1)
     check eventually onCancelCalled
 
   for requestState in {RequestState.New, Started, Finished, Failed}:
     test "onCancelled is not called when request state is " & $requestState:
       agent.start(MockState.new())
       await agent.subscribe()
       market.requestState[request.id] = requestState
-      clock.set(request.expiry.truncate(int64) + 1)
+      clock.set(market.requestExpiry[request.id] + 1)
       await sleepAsync(100.millis)
       check not onCancelCalled
 
@@ -126,7 +115,7 @@ asyncchecksuite "Sales agent":
       agent.start(MockState.new())
       await agent.subscribe()
       market.requestState[request.id] = requestState
-      clock.set(request.expiry.truncate(int64) + 1)
+      clock.set(market.requestExpiry[request.id] + 1)
       check eventually agent.data.cancelled.finished
 
   test "cancelled future is finished (cancelled) when onFulfilled called":

tests/codex/testpurchasing.nim

@@ -19,7 +19,7 @@ asyncchecksuite "Purchasing":
   var purchasing: Purchasing
   var market: MockMarket
   var clock: MockClock
-  var request: StorageRequest
+  var request, populatedRequest: StorageRequest
 
   setup:
     market = MockMarket.new()
@@ -34,6 +34,12 @@ asyncchecksuite "Purchasing":
       )
     )
 
+    # We need request which has stable ID during the whole Purchasing pipeline
+    # for some tests (related to expiry). Because of Purchasing.populate() we need
+    # to do the steps bellow.
+    populatedRequest = StorageRequest.example
+    populatedRequest.client = await market.getSigner()
+
   test "submits a storage request when asked":
     discard await purchasing.purchase(request)
     check eventually market.requested.len > 0
@@ -63,23 +69,6 @@ asyncchecksuite "Purchasing":
     check eventually market.requested.len > 0
     check market.requested[0].ask.proofProbability == 42.u256
 
-  test "has a default value for request expiration interval":
-    check purchasing.requestExpiryInterval != 0.u256
-
-  test "can change default value for request expiration interval":
-    purchasing.requestExpiryInterval = 42.u256
-    let start = getTime().toUnix()
-    discard await purchasing.purchase(request)
-    check eventually market.requested.len > 0
-    check market.requested[0].expiry == (start + 42).u256
-
-  test "can override expiry time per request":
-    let expiry = (getTime().toUnix() + 42).u256
-    request.expiry = expiry
-    discard await purchasing.purchase(request)
-    check eventually market.requested.len > 0
-    check market.requested[0].expiry == expiry
-
   test "includes a random nonce in every storage request":
     discard await purchasing.purchase(request)
     discard await purchasing.purchase(request)
@@ -92,29 +81,37 @@ asyncchecksuite "Purchasing":
     check market.requested[0].client == await market.getSigner()
 
   test "succeeds when request is finished":
-    let purchase = await purchasing.purchase(request)
+    market.requestExpiry[populatedRequest.id] = getTime().toUnix() + 10
+    let purchase = await purchasing.purchase(populatedRequest)
+
     check eventually market.requested.len > 0
     let request = market.requested[0]
     let requestEnd = getTime().toUnix() + 42
     market.requestEnds[request.id] = requestEnd
+
     market.emitRequestFulfilled(request.id)
     clock.set(requestEnd + 1)
     await purchase.wait()
     check purchase.error.isNone
 
   test "fails when request times out":
-    let purchase = await purchasing.purchase(request)
+    let expiry = getTime().toUnix() + 10
+    market.requestExpiry[populatedRequest.id] = expiry
+    let purchase = await purchasing.purchase(populatedRequest)
     check eventually market.requested.len > 0
     let request = market.requested[0]
-    clock.set(request.expiry.truncate(int64) + 1)
+
+    clock.set(expiry + 1)
     expect PurchaseTimeout:
       await purchase.wait()
 
   test "checks that funds were withdrawn when purchase times out":
-    let purchase = await purchasing.purchase(request)
+    let expiry = getTime().toUnix() + 10
+    market.requestExpiry[populatedRequest.id] = expiry
+    let purchase = await purchasing.purchase(populatedRequest)
     check eventually market.requested.len > 0
     let request = market.requested[0]
-    clock.set(request.expiry.truncate(int64) + 1)
+    clock.set(expiry + 1)
     expect PurchaseTimeout:
       await purchase.wait()
     check market.withdrawn == @[request.id]

tests/contracts/testContracts.nim

@@ -85,7 +85,8 @@ ethersuite "Marketplace contracts":
     check endBalance == (startBalance + request.ask.duration * request.ask.reward + request.ask.collateral)
 
   test "cannot mark proofs missing for cancelled request":
-    await ethProvider.advanceTimeTo(request.expiry + 1)
+    let expiry = await marketplace.requestExpiry(request.id)
+    await ethProvider.advanceTimeTo((expiry + 1).u256)
     switchAccount(client)
     let missingPeriod = periodicity.periodOf(await ethProvider.currentTime())
     await ethProvider.advanceTime(periodicity.seconds)