Feature/dag unit tests

private/erasure_code/erasure_code.py

@@ -0,0 +1,117 @@
+import math
+from itertools import combinations
+
+from reedsolo import ReedSolomonError, RSCodec
+
+
+def missing_shards_idx(n, k):
+    return [list(combo) for combo in combinations(range(n), k)]
+
+
+def split_into_blocks(encoded: bytes, shard_size: int):
+    blocks = []
+    for offset in range(0, len(encoded), shard_size):
+        block = encoded[offset : offset + shard_size]
+        if len(block) < shard_size:
+            block = block.ljust(shard_size, b"\x00")
+        blocks.append(block)
+    return blocks
+
+
+class ErasureCode:
+    def __init__(self, data_blocks: int, parity_blocks: int):
+        if data_blocks <= 0 or parity_blocks <= 0:
+            raise ValueError("Data and parity shards must be > 0")
+        self.data_blocks = data_blocks
+        self.parity_blocks = parity_blocks
+        self.total_shards = data_blocks + parity_blocks
+
+    @classmethod
+    def new(cls, data_blocks: int, parity_blocks: int):
+        return cls(data_blocks, parity_blocks)
+
+    def encode(self, data: bytes) -> bytes:
+        total_size = len(data)
+        shard_size = math.ceil(total_size / self.data_blocks)
+        padded_data = data.ljust(self.data_blocks * shard_size, b"\x00")
+
+        shards = [bytearray(padded_data[i * shard_size : (i + 1) * shard_size]) for i in range(self.data_blocks)]
+        parity_shards = [bytearray(shard_size) for _ in range(self.parity_blocks)]
+        rsc = RSCodec(self.parity_blocks)
+
+        for j in range(shard_size):
+            message = bytearray(self.data_blocks)
+            for i in range(self.data_blocks):
+                message[i] = shards[i][j]
+
+            encoded_msg = rsc.encode(message)
+            parity_bytes = encoded_msg[self.data_blocks :]
+
+            for i in range(self.parity_blocks):
+                parity_shards[i][j] = parity_bytes[i]
+
+        all_shards = shards + parity_shards
+        return b"".join(all_shards)
+
+    def extract_data(self, encoded: bytes, original_data_size: int, erase_pos=None) -> bytes:
+        shard_size = len(encoded) // self.total_shards
+
+        shards = [bytearray(encoded[i * shard_size : (i + 1) * shard_size]) for i in range(self.total_shards)]
+
+        rsc = RSCodec(self.parity_blocks)
+        decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)]
+
+        erased_shard_indices = set()
+        if erase_pos:
+            for pos in erase_pos:
+                erased_shard_indices.add(pos // shard_size)
+
+        erased_shard_indices_list = list(erased_shard_indices)
+
+        for j in range(shard_size):
+            message = bytearray(self.total_shards)
+            for i in range(self.total_shards):
+                message[i] = shards[i][j]
+
+            try:
+                decoded_msg, _, _ = rsc.decode(message, erase_pos=erased_shard_indices_list)
+                for i in range(self.data_blocks):
+                    decoded_shards[i][j] = decoded_msg[i]
+            except ReedSolomonError as e:
+                raise ValueError(f"Decoding error at byte {j}: {str(e)}")
+
+        full_data = b"".join(decoded_shards)
+        return full_data[:original_data_size]
+
+    def extract_data_blocks(self, blocks, original_data_size: int) -> bytes:
+        if not blocks:
+            raise ValueError("No blocks provided")
+
+        valid_block = next((b for b in blocks if b is not None), None)
+        if valid_block is None:
+            raise ValueError("All blocks are missing")
+
+        shard_size = len(valid_block)
+        if len(blocks) != self.total_shards:
+            raise ValueError(f"Expected {self.total_shards} blocks, got {len(blocks)}")
+
+        erase_pos = [i for i, b in enumerate(blocks) if b is None]
+        shards = [bytearray(b) if b is not None else bytearray(shard_size) for b in blocks]
+
+        rsc = RSCodec(self.parity_blocks)
+        decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)]
+
+        for j in range(shard_size):
+            stripe = bytearray(self.total_shards)
+            for i in range(self.total_shards):
+                stripe[i] = shards[i][j]
+
+            try:
+                decoded_stripe, _, _ = rsc.decode(stripe, erase_pos=erase_pos)
+                for i in range(self.data_blocks):
+                    decoded_shards[i][j] = decoded_stripe[i]
+            except ReedSolomonError as e:
+                raise ValueError("Decoding error: " + str(e))
+
+        full_data = b"".join(decoded_shards)
+        return full_data[:original_data_size]

sdk/config.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from typing import List, Optional
 
+from private.erasure_code.erasure_code import ErasureCode
 from private.memory.memory import Size
 
 BLOCK_SIZE = 1 * Size.MB
@@ -34,15 +35,23 @@ def __init__(
         private_key: str,
         storage_contract_address: str,
         access_contract_address: Optional[str] = None,
+        policy_factory_contract_address: Optional[str] = None,
     ):
         self.dial_uri = dial_uri
         self.private_key = private_key
         self.storage_contract_address = storage_contract_address
         self.access_contract_address = access_contract_address
+        self.policy_factory_contract_address = policy_factory_contract_address or ""
 
     @staticmethod
     def default():
-        return Config(dial_uri="", private_key="", storage_contract_address="", access_contract_address="")
+        return Config(
+            dial_uri="",
+            private_key="",
+            storage_contract_address="",
+            access_contract_address="",
+            policy_factory_contract_address="",
+        )
 
 
 ## [SDK Error Class]
@@ -55,8 +64,8 @@ class SDKError(Exception):
 ## [Validation Functions]
 
 
-# Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum
 def validate_hex_string(hex_string: str) -> bool:
+    """Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum"""
     if not hex_string.startswith("0x"):
         return False
     if len(hex_string) < 10:
@@ -66,24 +75,23 @@ def validate_hex_string(hex_string: str) -> bool:
 
 ## [Test Configurations]
 
+
 DEFAULT_CONFIG_TEST_STREAMING_CONN = {
     "AKAVE_SDK_NODE": "connect.akave.ai:5000",
     "ENCRYPTION_KEY": "",
 }
 
 DEFAULT_CONFIG_TEST_SDK_CONN = {
-    "AKAVE_SDK_NODE": "connect.akave.ai:5000",  # For streaming operations
-    "AKAVE_IPC_NODE": "connect.akave.ai:5500",  # For IPC operations
+    "AKAVE_SDK_NODE": "connect.akave.ai:5000",
+    "AKAVE_IPC_NODE": "connect.akave.ai:5500",
     "ETHEREUM_NODE_URL": "https://n3-us.akave.ai/ext/bc/2JMWNmZbYvWcJRPPy1siaDBZaDGTDAaqXoY5UBKh4YrhNFzEce/rpc",
-    "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF",  # Will be obtained from node
-    "ACCESS_CONTRACT_ADDRESS": "",  # Will be obtained from node
+    "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF",
+    "ACCESS_CONTRACT_ADDRESS": "",
 }
 
 
-## [Error Handling Functions]
+## [Known Error Strings]
 
-# List of known error strings from the smart contracts
-# Replace these with the actual error strings from your contracts
 
 KNOWN_ERROR_STRINGS: List[str] = [
     "Storage: bucket doesn't exist",
@@ -92,16 +100,15 @@ def validate_hex_string(hex_string: str) -> bool:
     "Storage: file exists",
     "AccessManager: caller is not the owner",
     "AccessManager: caller is not authorized",
-    # Add all other known error strings here...
 ]
 
 
 @dataclass
 class SDKConfig:
     address: str
-    max_concurrency: int
-    block_part_size: int
-    use_connection_pool: bool
+    max_concurrency: int = 10
+    block_part_size: int = 1024 * 1024
+    use_connection_pool: bool = True
     parity_blocks_count: int = 0
     chunk_buffer: int = 10
     encryption_key: Optional[bytes] = None
@@ -111,3 +118,4 @@ class SDKConfig:
     max_retries: Optional[int] = 3
     backoff_delay: Optional[int] = 1
     ipc_address: Optional[str] = None
+    erasure_code: Optional[ErasureCode] = None

sdk/sdk_ipc.py

@@ -8,7 +8,6 @@
 import threading
 import time
 from datetime import datetime
-from hashlib import sha256
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import grpc
@@ -772,9 +771,7 @@ def _calculate_file_id(self, bucket_id: bytes, file_name: str) -> bytes:
             from Crypto.Hash import keccak
 
             combined = bucket_id + file_name.encode()
-            hash_obj = keccak.new(digest_bits=256)
-            hash_obj.update(combined)
-            return hash_obj.digest()
+            return keccak(combined)
         except ImportError:
             raise SDKError("Failed to import required modules for file ID calculation")
         except Exception as e:
@@ -1361,7 +1358,11 @@ def download_chunk_blocks(
                     except Exception as e:
                         raise SDKError(f"failed to download block: {str(e)}")
 
-            data = b"".join([b for b in blocks if b is not None])
+            if hasattr(self, "erasure_code") and self.erasure_code is not None:
+
+                data = self.erasure_code.extract_data_blocks(blocks, chunk_download.size)
+            else:
+                data = b"".join([b for b in blocks if b is not None])
 
             if file_encryption_key:
                 from private.encryption import decrypt

setup.py

@@ -23,4 +23,4 @@
     ],
     python_requires=">=3.9",
     install_requires=requirements,
-)
+)