Skip to content

Commit 15ff87a

Browse files
veaceslavdoinaAuHaumarkspanbroekemizzletbekas
authored
Merge latest master into release (#842)
* fix: createReservation lock (#825) * fix: createReservation lock * fix: additional locking places * fix: acquire lock * chore: feedback Co-authored-by: markspanbroek <mark@spanbroek.net> Signed-off-by: Adam Uhlíř <adam@uhlir.dev> * feat: withLock template and fixed tests * fix: use proc for MockReservations constructor * chore: feedback Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> Signed-off-by: Adam Uhlíř <adam@uhlir.dev> * chore: feedback implementation --------- Signed-off-by: Adam Uhlíř <adam@uhlir.dev> Co-authored-by: markspanbroek <mark@spanbroek.net> Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> * Block deletion with ref count & repostore refactor (#631) * Fix StoreStream so it doesn't return parity bytes (#838) * fix storestream so it doesn\'t return parity bits for protected/verifiable manifests * use Cid.example instead of creating a mock manually * Fix verifiable manifest initialization (#839) * fix verifiable manifest initialization * fix linearstrategy, use verifiableStrategy to select blocks for slots * check for both strategies in attribute inheritance test * ci: add verify_circuit=true to the releases (#840) * provisional fix so EC errors do not crash the node on download (#841) --------- Signed-off-by: Adam Uhlíř <adam@uhlir.dev> Co-authored-by: Adam Uhlíř <adam@uhlir.dev> Co-authored-by: markspanbroek <mark@spanbroek.net> Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> Co-authored-by: Tomasz Bekas <tomasz.bekas@gmail.com> Co-authored-by: Giuliano Mega <giuliano.mega@gmail.com>
1 parent 6e9bdf1 commit 15ff87a

34 files changed

Lines changed: 1410 additions & 986 deletions

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ env:
1111
nim_version: v1.6.14
1212
rust_version: 1.78.0
1313
binary_base: codex
14+
nim_flags: '-d:verify_circuit=true'
1415
upload_to_codex: false
1516

1617
jobs:
@@ -74,7 +75,7 @@ jobs:
7475
7576
- name: Release - Build
7677
run: |
77-
make NIMFLAGS="--out:${{ env.binary }}"
78+
make NIMFLAGS="--out:${{ env.binary }} ${{ env.nim_flags }}"
7879
7980
- name: Release - Upload binaries
8081
uses: actions/upload-artifact@v4

‎codex/codex.nim‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ proc new*(
258258
repoDs = repoData,
259259
metaDs = LevelDbDatastore.new(config.dataDir / CodexMetaNamespace)
260260
.expect("Should create metadata store!"),
261-
quotaMaxBytes = config.storageQuota.uint,
261+
quotaMaxBytes = config.storageQuota,
262262
blockTtl = config.blockTtl)
263263

264264
maintenance = BlockMaintainer.new(

‎codex/indexingstrategy.nim‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ func getLinearIndicies(
4949
self.checkIteration(iteration)
5050

5151
let
52-
first = self.firstIndex + iteration * (self.step + 1)
53-
last = min(first + self.step, self.lastIndex)
52+
first = self.firstIndex + iteration * self.step
53+
last = min(first + self.step - 1, self.lastIndex)
5454

5555
getIter(first, last, 1)
5656

@@ -94,4 +94,4 @@ func init*(
9494
firstIndex: firstIndex,
9595
lastIndex: lastIndex,
9696
iterations: iterations,
97-
step: divUp((lastIndex - firstIndex), iterations))
97+
step: divUp((lastIndex - firstIndex + 1), iterations))

‎codex/manifest/manifest.nim‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,6 @@ func isManifest*(mc: MultiCodec): ?!bool =
135135
# Various sizes and verification
136136
############################################################
137137

138-
func bytes*(self: Manifest, pad = true): NBytes =
139-
## Compute how many bytes corresponding StoreStream(Manifest, pad) will return
140-
if pad or self.protected:
141-
self.blocksCount.NBytes * self.blockSize
142-
else:
143-
self.datasetSize
144-
145138
func rounded*(self: Manifest): int =
146139
## Number of data blocks in *protected* manifest including padding at the end
147140
roundUp(self.originalBlocksCount, self.ecK)
@@ -238,7 +231,7 @@ func new*(
238231
treeCid: Cid,
239232
datasetSize: NBytes,
240233
ecK, ecM: int,
241-
strategy: StrategyType): Manifest =
234+
strategy = SteppedStrategy): Manifest =
242235
## Create an erasure protected dataset from an
243236
## unprotected one
244237
##
@@ -284,7 +277,7 @@ func new*(
284277
ecM: int,
285278
originalTreeCid: Cid,
286279
originalDatasetSize: NBytes,
287-
strategy: StrategyType): Manifest =
280+
strategy = SteppedStrategy): Manifest =
288281

289282
Manifest(
290283
treeCid: treeCid,
@@ -306,7 +299,7 @@ func new*(
306299
verifyRoot: Cid,
307300
slotRoots: openArray[Cid],
308301
cellSize = DefaultCellSize,
309-
strategy = SteppedStrategy): ?!Manifest =
302+
strategy = LinearStrategy): ?!Manifest =
310303
## Create a verifiable dataset from an
311304
## protected one
312305
##
@@ -331,6 +324,7 @@ func new*(
331324
ecM: manifest.ecM,
332325
originalTreeCid: manifest.treeCid,
333326
originalDatasetSize: manifest.originalDatasetSize,
327+
protectedStrategy: manifest.protectedStrategy,
334328
verifiable: true,
335329
verifyRoot: verifyRoot,
336330
slotRoots: @slotRoots,

‎codex/merkletree/codex/coders.nim‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ push: {.upraises: [].}
1414
import pkg/libp2p
1515
import pkg/questionable
1616
import pkg/questionable/results
17+
import pkg/stew/byteutils
18+
import pkg/serde/json
1719

1820
import ../../units
1921
import ../../errors
@@ -100,3 +102,18 @@ proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof =
100102
nodes.add node
101103

102104
CodexProof.init(mcodec, index.int, nleaves.int, nodes)
105+
106+
proc fromJson*(
107+
_: type CodexProof,
108+
json: JsonNode
109+
): ?!CodexProof =
110+
expectJsonKind(Cid, JString, json)
111+
var bytes: seq[byte]
112+
try:
113+
bytes = hexToSeqByte(json.str)
114+
except ValueError as err:
115+
return failure(err)
116+
117+
CodexProof.decode(bytes)
118+
119+
func `%`*(proof: CodexProof): JsonNode = % byteutils.toHex(proof.encode())

‎codex/node.nim‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -240,14 +240,14 @@ proc streamEntireDataset(
240240
self: CodexNodeRef,
241241
manifest: Manifest,
242242
manifestCid: Cid,
243-
): ?!LPStream =
243+
): Future[?!LPStream] {.async.} =
244244
## Streams the contents of the entire dataset described by the manifest.
245245
##
246246
trace "Retrieving blocks from manifest", manifestCid
247247

248248
if manifest.protected:
249249
# Retrieve, decode and save to the local store all EС groups
250-
proc erasureJob(): Future[void] {.async.} =
250+
proc erasureJob(): Future[?!void] {.async.} =
251251
try:
252252
# Spawn an erasure decoding job
253253
let
@@ -258,10 +258,16 @@ proc streamEntireDataset(
258258
self.taskpool)
259259
without _ =? (await erasure.decode(manifest)), error:
260260
trace "Unable to erasure decode manifest", manifestCid, exc = error.msg
261-
except CatchableError as exc:
261+
# --------------------------------------------------------------------------
262+
# FIXME this is a HACK so that the node does not crash during the workshop.
263+
# We should NOT catch Defect.
264+
except Exception as exc:
262265
trace "Exception decoding manifest", manifestCid, exc = exc.msg
266+
return failure(exc.msg)
267+
# --------------------------------------------------------------------------
263268

264-
asyncSpawn erasureJob()
269+
if err =? (await erasureJob()).errorOption:
270+
return failure(err)
265271

266272
# Retrieve all blocks of the dataset sequentially from the local store or network
267273
trace "Creating store stream for manifest", manifestCid
@@ -283,7 +289,7 @@ proc retrieve*(
283289

284290
return await self.streamSingleBlock(cid)
285291

286-
self.streamEntireDataset(manifest, cid)
292+
await self.streamEntireDataset(manifest, cid)
287293

288294
proc store*(
289295
self: CodexNodeRef,
@@ -534,7 +540,9 @@ proc onStore(
534540
trace "Unable to fetch manifest for cid", cid, err = err.msg
535541
return failure(err)
536542

537-
without builder =? Poseidon2Builder.new(self.networkStore, manifest), err:
543+
without builder =? Poseidon2Builder.new(
544+
self.networkStore, manifest, manifest.verifiableStrategy
545+
), err:
538546
trace "Unable to create slots builder", err = err.msg
539547
return failure(err)
540548

@@ -559,8 +567,8 @@ proc onStore(
559567

560568
return success()
561569

562-
without indexer =? manifest.protectedStrategy.init(
563-
0, manifest.numSlotBlocks() - 1, manifest.numSlots).catch, err:
570+
without indexer =? manifest.verifiableStrategy.init(
571+
0, manifest.blocksCount - 1, manifest.numSlots).catch, err:
564572
trace "Unable to create indexing strategy from protected manifest", err = err.msg
565573
return failure(err)
566574

‎codex/rest/json.nim‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import ../sales
77
import ../purchasing
88
import ../utils/json
99
import ../manifest
10+
import ../units
1011

1112
export json
1213

@@ -65,10 +66,10 @@ type
6566
id*: NodeId
6667

6768
RestRepoStore* = object
68-
totalBlocks* {.serialize.}: uint
69-
quotaMaxBytes* {.serialize.}: uint
70-
quotaUsedBytes* {.serialize.}: uint
71-
quotaReservedBytes* {.serialize.}: uint
69+
totalBlocks* {.serialize.}: Natural
70+
quotaMaxBytes* {.serialize.}: NBytes
71+
quotaUsedBytes* {.serialize.}: NBytes
72+
quotaReservedBytes* {.serialize.}: NBytes
7273

7374
proc init*(_: type RestContentList, content: seq[RestContent]): RestContentList =
7475
RestContentList(

0 commit comments

Comments
 (0)