diff --git a/private/erasure_code/__init__.py b/private/erasure_code/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/private/erasure_code/erasure_code.py b/private/erasure_code/erasure_code.py new file mode 100644 index 0000000..c054478 --- /dev/null +++ b/private/erasure_code/erasure_code.py @@ -0,0 +1,117 @@ +import math +from itertools import combinations + +from reedsolo import ReedSolomonError, RSCodec + + +def missing_shards_idx(n, k): + return [list(combo) for combo in combinations(range(n), k)] + + +def split_into_blocks(encoded: bytes, shard_size: int): + blocks = [] + for offset in range(0, len(encoded), shard_size): + block = encoded[offset : offset + shard_size] + if len(block) < shard_size: + block = block.ljust(shard_size, b"\x00") + blocks.append(block) + return blocks + + +class ErasureCode: + def __init__(self, data_blocks: int, parity_blocks: int): + if data_blocks <= 0 or parity_blocks <= 0: + raise ValueError("Data and parity shards must be > 0") + self.data_blocks = data_blocks + self.parity_blocks = parity_blocks + self.total_shards = data_blocks + parity_blocks + + @classmethod + def new(cls, data_blocks: int, parity_blocks: int): + return cls(data_blocks, parity_blocks) + + def encode(self, data: bytes) -> bytes: + total_size = len(data) + shard_size = math.ceil(total_size / self.data_blocks) + padded_data = data.ljust(self.data_blocks * shard_size, b"\x00") + + shards = [bytearray(padded_data[i * shard_size : (i + 1) * shard_size]) for i in range(self.data_blocks)] + parity_shards = [bytearray(shard_size) for _ in range(self.parity_blocks)] + rsc = RSCodec(self.parity_blocks) + + for j in range(shard_size): + message = bytearray(self.data_blocks) + for i in range(self.data_blocks): + message[i] = shards[i][j] + + encoded_msg = rsc.encode(message) + parity_bytes = encoded_msg[self.data_blocks :] + + for i in range(self.parity_blocks): + parity_shards[i][j] = parity_bytes[i] + + all_shards = shards + parity_shards + return b"".join(all_shards) + + def extract_data(self, encoded: bytes, original_data_size: int, erase_pos=None) -> bytes: + shard_size = len(encoded) // self.total_shards + + shards = [bytearray(encoded[i * shard_size : (i + 1) * shard_size]) for i in range(self.total_shards)] + + rsc = RSCodec(self.parity_blocks) + decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)] + + erased_shard_indices = set() + if erase_pos: + for pos in erase_pos: + erased_shard_indices.add(pos // shard_size) + + erased_shard_indices_list = list(erased_shard_indices) + + for j in range(shard_size): + message = bytearray(self.total_shards) + for i in range(self.total_shards): + message[i] = shards[i][j] + + try: + decoded_msg, _, _ = rsc.decode(message, erase_pos=erased_shard_indices_list) + for i in range(self.data_blocks): + decoded_shards[i][j] = decoded_msg[i] + except ReedSolomonError as e: + raise ValueError(f"Decoding error at byte {j}: {str(e)}") + + full_data = b"".join(decoded_shards) + return full_data[:original_data_size] + + def extract_data_blocks(self, blocks, original_data_size: int) -> bytes: + if not blocks: + raise ValueError("No blocks provided") + + valid_block = next((b for b in blocks if b is not None), None) + if valid_block is None: + raise ValueError("All blocks are missing") + + shard_size = len(valid_block) + if len(blocks) != self.total_shards: + raise ValueError(f"Expected {self.total_shards} blocks, got {len(blocks)}") + + erase_pos = [i for i, b in enumerate(blocks) if b is None] + shards = [bytearray(b) if b is not None else bytearray(shard_size) for b in blocks] + + rsc = RSCodec(self.parity_blocks) + decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)] + + for j in range(shard_size): + stripe = bytearray(self.total_shards) + for i in range(self.total_shards): + stripe[i] = shards[i][j] + + try: + decoded_stripe, _, _ = rsc.decode(stripe, erase_pos=erase_pos) + for i in range(self.data_blocks): + decoded_shards[i][j] = decoded_stripe[i] + except ReedSolomonError as e: + raise ValueError("Decoding error: " + str(e)) + + full_data = b"".join(decoded_shards) + return full_data[:original_data_size] diff --git a/sdk/config.py b/sdk/config.py index 254bb0e..d7832ac 100644 --- a/sdk/config.py +++ b/sdk/config.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List, Optional +from private.erasure_code.erasure_code import ErasureCode from private.memory.memory import Size BLOCK_SIZE = 1 * Size.MB @@ -34,15 +35,23 @@ def __init__( private_key: str, storage_contract_address: str, access_contract_address: Optional[str] = None, + policy_factory_contract_address: Optional[str] = None, ): self.dial_uri = dial_uri self.private_key = private_key self.storage_contract_address = storage_contract_address self.access_contract_address = access_contract_address + self.policy_factory_contract_address = policy_factory_contract_address or "" @staticmethod def default(): - return Config(dial_uri="", private_key="", storage_contract_address="", access_contract_address="") + return Config( + dial_uri="", + private_key="", + storage_contract_address="", + access_contract_address="", + policy_factory_contract_address="", + ) ## [SDK Error Class] @@ -55,8 +64,8 @@ class SDKError(Exception): ## [Validation Functions] -# Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum def validate_hex_string(hex_string: str) -> bool: + """Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum""" if not hex_string.startswith("0x"): return False if len(hex_string) < 10: @@ -66,24 +75,23 @@ def validate_hex_string(hex_string: str) -> bool: ## [Test Configurations] + DEFAULT_CONFIG_TEST_STREAMING_CONN = { "AKAVE_SDK_NODE": "connect.akave.ai:5000", "ENCRYPTION_KEY": "", } DEFAULT_CONFIG_TEST_SDK_CONN = { - "AKAVE_SDK_NODE": "connect.akave.ai:5000", # For streaming operations - "AKAVE_IPC_NODE": "connect.akave.ai:5500", # For IPC operations + "AKAVE_SDK_NODE": "connect.akave.ai:5000", + "AKAVE_IPC_NODE": "connect.akave.ai:5500", "ETHEREUM_NODE_URL": "https://n3-us.akave.ai/ext/bc/2JMWNmZbYvWcJRPPy1siaDBZaDGTDAaqXoY5UBKh4YrhNFzEce/rpc", - "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF", # Will be obtained from node - "ACCESS_CONTRACT_ADDRESS": "", # Will be obtained from node + "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF", + "ACCESS_CONTRACT_ADDRESS": "", } -## [Error Handling Functions] +## [Known Error Strings] -# List of known error strings from the smart contracts -# Replace these with the actual error strings from your contracts KNOWN_ERROR_STRINGS: List[str] = [ "Storage: bucket doesn't exist", @@ -92,16 +100,15 @@ def validate_hex_string(hex_string: str) -> bool: "Storage: file exists", "AccessManager: caller is not the owner", "AccessManager: caller is not authorized", - # Add all other known error strings here... ] @dataclass class SDKConfig: address: str - max_concurrency: int - block_part_size: int - use_connection_pool: bool + max_concurrency: int = 10 + block_part_size: int = 1024 * 1024 + use_connection_pool: bool = True parity_blocks_count: int = 0 chunk_buffer: int = 10 encryption_key: Optional[bytes] = None @@ -111,3 +118,4 @@ class SDKConfig: max_retries: Optional[int] = 3 backoff_delay: Optional[int] = 1 ipc_address: Optional[str] = None + erasure_code: Optional[ErasureCode] = None diff --git a/sdk/sdk_ipc.py b/sdk/sdk_ipc.py index 12d1d8b..b0ea9ae 100644 --- a/sdk/sdk_ipc.py +++ b/sdk/sdk_ipc.py @@ -8,7 +8,6 @@ import threading import time from datetime import datetime -from hashlib import sha256 from typing import Any, Callable, Dict, List, Optional, Tuple, Union import grpc @@ -772,9 +771,7 @@ def _calculate_file_id(self, bucket_id: bytes, file_name: str) -> bytes: from Crypto.Hash import keccak combined = bucket_id + file_name.encode() - hash_obj = keccak.new(digest_bits=256) - hash_obj.update(combined) - return hash_obj.digest() + return keccak(combined) except ImportError: raise SDKError("Failed to import required modules for file ID calculation") except Exception as e: @@ -1361,7 +1358,11 @@ def download_chunk_blocks( except Exception as e: raise SDKError(f"failed to download block: {str(e)}") - data = b"".join([b for b in blocks if b is not None]) + if hasattr(self, "erasure_code") and self.erasure_code is not None: + + data = self.erasure_code.extract_data_blocks(blocks, chunk_download.size) + else: + data = b"".join([b for b in blocks if b is not None]) if file_encryption_key: from private.encryption import decrypt diff --git a/setup.py b/setup.py index 7d2e2cc..289e920 100644 --- a/setup.py +++ b/setup.py @@ -23,4 +23,4 @@ ], python_requires=">=3.9", install_requires=requirements, -) \ No newline at end of file +) diff --git a/test_ipc_upload_integration.py b/test_ipc_upload_integration.py new file mode 100755 index 0000000..c850d03 --- /dev/null +++ b/test_ipc_upload_integration.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +import os +import time +from pathlib import Path + +from akavesdk import SDK, SDKConfig + + +def main(): + NODE_ADDRESS = "connect.akave.ai:5500" + PRIVATE_KEY = "a5c223e956644f1ba11f0dcc6f3df4992184ff3c919223744d0cf1db33dab4d6" + BUCKET_NAME = "finalfr" + + script_dir = Path(__file__).parent + test_file_path = script_dir / "random_3mb_file.bin" + + if not test_file_path.exists(): + print(f"❌ Error: Test file not found: {test_file_path}") + print(f" Please ensure random_3mb_file.bin exists in {script_dir}") + return 1 + + file_size = os.path.getsize(test_file_path) + # Use unique filename with timestamp to avoid conflicts + timestamp = int(time.time()) + file_name = f"test_{timestamp}.bin" + + print(f"\n{'='*70}") + print(f"πŸš€ Akave IPC Upload Integration Test") + print(f"{'='*70}") + print(f"πŸ“‘ Node Address: {NODE_ADDRESS}") + print(f"πŸ“¦ Bucket Name: {BUCKET_NAME}") + print(f"πŸ“„ File Name: {file_name}") + print(f"πŸ“‚ Source File: {test_file_path}") + print(f"πŸ“Š File Size: {file_size:,} bytes ({file_size / (1024*1024):.2f} MB)") + print(f"{'='*70}\n") + + try: + print("πŸ”§ Step 1: Initializing SDK...") + config = SDKConfig( + address=NODE_ADDRESS, + private_key=PRIVATE_KEY, + max_concurrency=5, + block_part_size=128 * 1024, + use_connection_pool=True, + chunk_buffer=10, + ) + sdk = SDK(config) + print("βœ… SDK initialized successfully\n") + + print("πŸ”§ Step 2: Creating IPC instance...") + ipc = sdk.ipc() + print("βœ… IPC instance created\n") + + print(f"πŸ”§ Step 3: Checking/Creating bucket '{BUCKET_NAME}'...") + existing_bucket = ipc.view_bucket(None, BUCKET_NAME) + + if existing_bucket is None: + print(f" Bucket doesn't exist, creating...") + result = ipc.create_bucket(None, BUCKET_NAME) + print(f"βœ… Bucket created successfully") + print(f" Bucket ID: {result.id}") + print(f" Bucket Name: {result.name}") + print(f" Created At: {result.created_at}") + time.sleep(2) + else: + print(f"βœ… Bucket already exists") + print(f" Bucket ID: {existing_bucket.id}") + print(f" Bucket Name: {existing_bucket.name}\n") + + print(f"πŸ”§ Step 4: Uploading file...") + print(f" File: {file_name}") + print(f" This may take a while for a {file_size / (1024*1024):.2f} MB file...") + print(f" Note: upload() will create file upload and handle all transactions\n") + + start_time = time.time() + + with open(test_file_path, "rb") as f: + file_meta = ipc.upload(None, BUCKET_NAME, file_name, f) + + upload_duration = time.time() - start_time + upload_speed = (file_size / (1024 * 1024)) / upload_duration if upload_duration > 0 else 0 + + print(f"\nβœ… File uploaded successfully!") + print(f" Root CID: {file_meta.root_cid}") + print(f" File Name: {file_meta.name}") + print(f" File Size: {file_meta.size:,} bytes") + print(f" Encoded Size: {file_meta.encoded_size:,} bytes") + print(f" Upload Duration: {upload_duration:.2f} seconds") + print(f" Upload Speed: {upload_speed:.2f} MB/s\n") + + print(f"πŸ”§ Step 5: Verifying file metadata...") + try: + retrieved_meta = ipc.file_info(None, BUCKET_NAME, file_name) + + if retrieved_meta is None: + print(f"⚠️ Could not retrieve file metadata (but upload succeeded!)") + else: + print(f"βœ… File metadata verified") + print(f" Name: {retrieved_meta.name}") + print(f" Bucket: {retrieved_meta.bucket_name}") + print(f" Root CID: {retrieved_meta.root_cid}") + print(f" Size: {retrieved_meta.encoded_size:,} bytes\n") + except Exception as e: + print(f"⚠️ File verification skipped (upload succeeded!)") + print(f" Note: {str(e)}") + print(f" The file was successfully uploaded and committed\n") + retrieved_meta = None + + if retrieved_meta and retrieved_meta.root_cid != file_meta.root_cid: + print(f"⚠️ Warning: Root CID mismatch!") + print(f" Upload CID: {file_meta.root_cid}") + print(f" Retrieved CID: {retrieved_meta.root_cid}") + elif retrieved_meta: + print(f"βœ… Root CID matches upload!\n") + + print(f"πŸ”§ Step 6: Listing files in bucket...") + files = ipc.list_files(None, BUCKET_NAME) + print(f"βœ… Found {len(files)} file(s) in bucket '{BUCKET_NAME}'") + + uploaded_file = next((f for f in files if f.name == file_name), None) + if uploaded_file: + print(f"βœ… Uploaded file found in bucket listing") + else: + print(f"⚠️ Warning: Uploaded file not found in listing") + + print(f"\n{'='*70}") + print(f"βœ… Upload Test Completed Successfully!") + print(f"{'='*70}") + print(f"\nπŸ“‹ Summary:") + print(f" β€’ File: {file_name}") + print(f" β€’ Size: {file_size:,} bytes ({file_size / (1024*1024):.2f} MB)") + print(f" β€’ Root CID: {file_meta.root_cid}") + print(f" β€’ Bucket: {BUCKET_NAME}") + print(f" β€’ Upload Time: {upload_duration:.2f}s") + print(f" β€’ Upload Speed: {upload_speed:.2f} MB/s") + print(f"{'='*70}\n") + + sdk.close() + return 0 + + except Exception as e: + print(f"\n❌ Error during upload test:") + print(f" {type(e).__name__}: {str(e)}") + import traceback + + print(f"\nπŸ“‹ Full traceback:") + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + exit_code = main() + exit(exit_code) diff --git a/tests/unit/test_dag.py b/tests/unit/test_dag.py index 408d129..86beb8c 100644 --- a/tests/unit/test_dag.py +++ b/tests/unit/test_dag.py @@ -1,6 +1,545 @@ +"""Unit tests for sdk/dag.py - DAG creation, CID operations, and block handling.""" + +import hashlib +import io +from unittest.mock import Mock, patch, MagicMock + import pytest +from sdk.dag import ( + DAGRoot, + ChunkDAG, + DAGError, + build_dag, + _create_unixfs_file_node, + _create_chunk_dag_root_node, + node_sizes, + _extract_unixfs_data_size, + _encode_varint, + _decode_varint, + extract_block_data, + _extract_unixfs_data_fallback, + _extract_unixfs_data, + bytes_to_node, + get_node_links, + block_by_cid, +) +from sdk.model import FileBlockUpload + + +class TestDAGRoot: + """Test DAGRoot class functionality.""" + + def test_new(self): + """Test DAGRoot.new() creates a new instance.""" + dag_root = DAGRoot.new() + assert isinstance(dag_root, DAGRoot) + assert dag_root.node is None + assert dag_root.fs_node_data == b"" + assert dag_root.links == [] + assert dag_root.total_file_size == 0 + + def test_add_link_with_string_cid(self): + """Test adding a link with string CID.""" + dag_root = DAGRoot.new() + cid_str = "bafybeigtest123" + raw_data_size = 1024 + proto_node_size = 512 + + dag_root.add_link(cid_str, raw_data_size, proto_node_size) + + assert len(dag_root.links) == 1 + assert dag_root.links[0]["cid_str"] == cid_str + assert dag_root.links[0]["size"] == proto_node_size + assert dag_root.total_file_size == raw_data_size + + def test_add_link_with_cid_object(self): + """Test adding a link with CID object.""" + dag_root = DAGRoot.new() + mock_cid = Mock() + mock_cid.string.return_value = "bafybeigtest456" + raw_data_size = 2048 + proto_node_size = 1024 + + dag_root.add_link(mock_cid, raw_data_size, proto_node_size) + + assert len(dag_root.links) == 1 + assert dag_root.total_file_size == raw_data_size + + def test_add_multiple_links(self): + """Test adding multiple links accumulates size.""" + dag_root = DAGRoot.new() + + dag_root.add_link("cid1", 1024, 512) + dag_root.add_link("cid2", 2048, 1024) + dag_root.add_link("cid3", 4096, 2048) + + assert len(dag_root.links) == 3 + assert dag_root.total_file_size == 1024 + 2048 + 4096 + + def test_build_no_links_raises_error(self): + """Test build() raises error when no links added.""" + dag_root = DAGRoot.new() + + with pytest.raises(DAGError, match="no chunks added"): + dag_root.build() + + def test_build_single_link_returns_cid(self): + """Test build() with single link returns that link's CID.""" + dag_root = DAGRoot.new() + mock_cid = Mock() + mock_cid.string.return_value = "bafybeigsinglelinktest" + + dag_root.add_link(mock_cid, 1024, 512) + result = dag_root.build() + + assert result == "bafybeigsinglelinktest" + + def test_encode_varint_single_byte(self): + """Test _encode_varint for values fitting in single byte.""" + dag_root = DAGRoot.new() + + assert dag_root._encode_varint(0) == b"\x00" + assert dag_root._encode_varint(1) == b"\x01" + assert dag_root._encode_varint(127) == b"\x7f" + + def test_encode_varint_multi_byte(self): + """Test _encode_varint for values requiring multiple bytes.""" + dag_root = DAGRoot.new() + + # 128 requires 2 bytes + assert dag_root._encode_varint(128) == b"\x80\x01" + assert dag_root._encode_varint(255) == b"\xff\x01" + assert dag_root._encode_varint(16384) == b"\x80\x80\x01" + + def test_create_unixfs_file_data(self): + """Test _create_unixfs_file_data creates proper UnixFS data.""" + dag_root = DAGRoot.new() + dag_root.total_file_size = 1024 + + result = dag_root._create_unixfs_file_data() + + # Should start with type = File (0x08, 0x02) + assert result[:2] == b"\x08\x02" + # Should contain size field and encoded varint + assert len(result) > 2 + + def test_create_unixfs_file_data_zero_size(self): + """Test _create_unixfs_file_data with zero size.""" + dag_root = DAGRoot.new() + dag_root.total_file_size = 0 + + result = dag_root._create_unixfs_file_data() + + # Should only have type field, no size + assert result == b"\x08\x02" + + +class TestVarintEncoding: + """Test varint encoding and decoding functions.""" + + def test_encode_varint_zero(self): + """Test encoding zero.""" + assert _encode_varint(0) == b"\x00" + + def test_encode_varint_small_values(self): + """Test encoding small values.""" + assert _encode_varint(1) == b"\x01" + assert _encode_varint(127) == b"\x7f" + + def test_encode_varint_medium_values(self): + """Test encoding medium values requiring multi-byte.""" + assert _encode_varint(128) == b"\x80\x01" + assert _encode_varint(300) == b"\xac\x02" + + def test_encode_varint_large_values(self): + """Test encoding large values.""" + assert _encode_varint(16384) == b"\x80\x80\x01" + + def test_decode_varint_single_byte(self): + """Test decoding single-byte varints.""" + value, bytes_read = _decode_varint(b"\x00") + assert value == 0 + assert bytes_read == 1 + + value, bytes_read = _decode_varint(b"\x7f") + assert value == 127 + assert bytes_read == 1 + + def test_decode_varint_multi_byte(self): + """Test decoding multi-byte varints.""" + value, bytes_read = _decode_varint(b"\x80\x01") + assert value == 128 + assert bytes_read == 2 + + value, bytes_read = _decode_varint(b"\xac\x02") + assert value == 300 + assert bytes_read == 2 + + def test_decode_varint_with_extra_data(self): + """Test decoding varint when followed by extra data.""" + value, bytes_read = _decode_varint(b"\x7fextra_data") + assert value == 127 + assert bytes_read == 1 + + def test_decode_varint_too_long_raises_error(self): + """Test decoding overlong varint raises ValueError.""" + long_varint = b"\x80" * 20 + with pytest.raises(ValueError, match="varint too long"): + _decode_varint(long_varint) + + def test_encode_decode_roundtrip(self): + """Test encode-decode roundtrip for various values.""" + test_values = [0, 1, 127, 128, 255, 256, 16384, 1000000] + + for value in test_values: + encoded = _encode_varint(value) + decoded, bytes_read = _decode_varint(encoded) + assert decoded == value, f"Roundtrip failed for {value}" + assert bytes_read == len(encoded) + + +class TestUnixFSDataExtraction: + """Test UnixFS data extraction functions.""" + + def test_extract_unixfs_data_size_simple(self): + """Test extracting size from simple UnixFS data.""" + # Field 3 (fileSize) with value 1024 + unixfs_data = bytes([0x18, 0x80, 0x08]) # 3<<3|0, then varint for 1024 + size = _extract_unixfs_data_size(unixfs_data) + assert size == 1024 + + def test_extract_unixfs_data_size_zero(self): + """Test extracting zero size.""" + # Empty UnixFS data + unixfs_data = b"" + size = _extract_unixfs_data_size(unixfs_data) + assert size == 0 + + def test_extract_unixfs_data_with_field_4(self): + """Test extracting from field 4 (data length).""" + # Field 4 (data) with length-delimited wire type + unixfs_data = bytes([0x22, 0x04]) + b"test" # field 4, length 4, then "test" + size = _extract_unixfs_data_size(unixfs_data) + assert size == 4 + + def test_extract_unixfs_data_fallback_simple(self): + """Test fallback extraction with simple data.""" + # Field 1 (data field) with UnixFS inside + inner_data = bytes([0x08, 0x02]) # Type=File + node_data = bytes([0x0a, len(inner_data)]) + inner_data + + result = _extract_unixfs_data_fallback(node_data) + # Should return the node data or inner data + assert isinstance(result, bytes) + + def test_extract_unixfs_data_empty_inner(self): + """Test extract with empty inner data.""" + result = _extract_unixfs_data(b"") + assert result == b"" + + def test_extract_unixfs_data_with_field_4(self): + """Test extracting actual data from field 4.""" + # Field 4 with "hello world" + data_content = b"hello world" + unixfs_data = bytes([0x22, len(data_content)]) + data_content + + result = _extract_unixfs_data(unixfs_data) + assert result == data_content + + +class TestChunkDAG: + """Test ChunkDAG dataclass.""" + + def test_chunk_dag_creation(self): + """Test ChunkDAG instantiation.""" + blocks = [FileBlockUpload(cid="cid1", data=b"data1")] + chunk_dag = ChunkDAG( + cid="root_cid", raw_data_size=100, encoded_size=200, blocks=blocks + ) + + assert chunk_dag.cid == "root_cid" + assert chunk_dag.raw_data_size == 100 + assert chunk_dag.encoded_size == 200 + assert chunk_dag.blocks == blocks + + def test_chunk_dag_empty_blocks(self): + """Test ChunkDAG with empty blocks.""" + chunk_dag = ChunkDAG( + cid="root_cid", raw_data_size=0, encoded_size=0, blocks=[] + ) + + assert chunk_dag.blocks == [] + + +class TestBuildDAG: + """Test build_dag function.""" + + def test_build_dag_small_data_no_ipld(self): + """Test build_dag with small data and no IPLD library.""" + data = b"small data" + reader = io.BytesIO(data) + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = build_dag(None, reader, block_size=1024) + + assert isinstance(result, ChunkDAG) + assert result.raw_data_size == len(data) + assert len(result.blocks) == 1 + assert result.blocks[0].data == data + + def test_build_dag_empty_data_raises_error(self): + """Test build_dag with empty data raises error.""" + reader = io.BytesIO(b"") + + with pytest.raises(DAGError, match="empty data"): + build_dag(None, reader, block_size=1024) + + def test_build_dag_exact_block_size(self): + """Test build_dag when data exactly matches block size.""" + data = b"x" * 1024 + reader = io.BytesIO(data) + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = build_dag(None, reader, block_size=1024) + + assert result.raw_data_size == 1024 + assert len(result.blocks) == 1 + + def test_build_dag_multiple_blocks(self): + """Test build_dag with data spanning multiple blocks.""" + data = b"x" * 3000 # 3x1024 bytes + reader = io.BytesIO(data) + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = build_dag(None, reader, block_size=1024) + + assert result.raw_data_size == 3000 + assert len(result.blocks) == 3 + + def test_build_dag_partial_last_block(self): + """Test build_dag with partial last block.""" + data = b"x" * 2500 # 2.5 blocks of 1024 + reader = io.BytesIO(data) + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = build_dag(None, reader, block_size=1024) + + assert result.raw_data_size == 2500 + assert len(result.blocks) == 3 + + +class TestCreateUnixFSFileNode: + """Test _create_unixfs_file_node function.""" + + def test_create_unixfs_file_node_fallback(self): + """Test node creation with IPLD unavailable.""" + data = b"test data" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + cid, encoded = _create_unixfs_file_node(data) + + assert isinstance(cid, str) + assert cid.startswith("bafybeig") + assert encoded == data + + def test_create_unixfs_file_node_empty_data_fallback(self): + """Test empty data node creation fallback.""" + data = b"" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + cid, encoded = _create_unixfs_file_node(data) + + assert isinstance(cid, str) + assert cid.startswith("bafybeig") + + +class TestCreateChunkDAGRootNode: + """Test _create_chunk_dag_root_node function.""" + + def test_create_chunk_dag_root_no_ipld(self): + """Test chunk DAG root creation without IPLD.""" + blocks = [ + FileBlockUpload(cid="cid1", data=b"block1"), + FileBlockUpload(cid="cid2", data=b"block2"), + ] + + with patch("sdk.dag.IPLD_AVAILABLE", False): + cid, size = _create_chunk_dag_root_node(blocks) + + assert isinstance(cid, str) + assert cid.startswith("bafybeig") + assert size == len(b"block1block2") + + def test_create_chunk_dag_root_empty_blocks(self): + """Test chunk DAG root with empty blocks list.""" + blocks = [] + + with patch("sdk.dag.IPLD_AVAILABLE", False): + cid, size = _create_chunk_dag_root_node(blocks) + + assert isinstance(cid, str) + assert size == 0 + + +class TestNodeSizes: + """Test node_sizes function.""" + + def test_node_sizes_simple_data_no_ipld(self): + """Test node size calculation without IPLD.""" + node_data = b"simple node data" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + raw_size, encoded_size = node_sizes(node_data) + + assert raw_size == len(node_data) + assert encoded_size == len(node_data) + + def test_node_sizes_empty_node_no_ipld(self): + """Test node size for empty node.""" + node_data = b"" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + raw_size, encoded_size = node_sizes(node_data) + + assert raw_size == 0 + assert encoded_size == 0 + + +class TestExtractBlockData: + """Test extract_block_data function.""" + + def test_extract_block_data_raw_codec_no_ipld(self): + """Test extracting block data in raw codec without IPLD.""" + cid_str = "bafkreigtest" + data = b"raw block data" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = extract_block_data(cid_str, data) + + # Should return fallback extraction + assert isinstance(result, bytes) + + def test_extract_block_data_empty_data(self): + """Test extracting from empty data.""" + cid_str = "bafybeigtest" + data = b"" + + with patch("sdk.dag.IPLD_AVAILABLE", False): + result = extract_block_data(cid_str, data) + + assert result == b"" + + +class TestBytesToNode: + """Test bytes_to_node function.""" + + def test_bytes_to_node_invalid_data_raises_error(self): + """Test bytes_to_node with invalid data raises error.""" + invalid_data = b"\xff\xff\xff" + + with patch("sdk.dag.DAG_PB_AVAILABLE", False): + with pytest.raises(DAGError, match="failed to decode"): + bytes_to_node(invalid_data) + + +class TestGetNodeLinks: + """Test get_node_links function.""" + + def test_get_node_links_empty_node(self): + """Test getting links from node with no links.""" + # This test would need actual PBNode or mocking + with patch("sdk.dag.DAG_PB_AVAILABLE", False): + with pytest.raises(DAGError): + get_node_links(b"invalid") + + def test_get_node_links_with_mock_node(self): + """Test getting links from mocked node.""" + mock_link = Mock() + mock_link.hash = "test_cid" + mock_link.name = "test" + mock_link.size = 1024 + + mock_node = Mock() + mock_node.links = [mock_link] + + with patch("sdk.dag.bytes_to_node", return_value=mock_node): + result = get_node_links(b"dummy_data") + + assert len(result) == 1 + assert result[0]["cid"] == "test_cid" + assert result[0]["name"] == "test" + assert result[0]["size"] == 1024 + + +class TestBlockByCID: + """Test block_by_cid function.""" + + def test_block_by_cid_found(self): + """Test finding block by CID.""" + blocks = [ + FileBlockUpload(cid="cid1", data=b"data1"), + FileBlockUpload(cid="cid2", data=b"data2"), + ] + + block, found = block_by_cid(blocks, "cid1") + + assert found is True + assert block.cid == "cid1" + assert block.data == b"data1" + + def test_block_by_cid_not_found(self): + """Test block not found returns false.""" + blocks = [ + FileBlockUpload(cid="cid1", data=b"data1"), + ] + + block, found = block_by_cid(blocks, "cid999") + + assert found is False + assert block.cid == "" + assert block.data == b"" + + def test_block_by_cid_empty_list(self): + """Test searching empty block list.""" + block, found = block_by_cid([], "cid1") + + assert found is False + + +class TestDAGIntegration: + """Integration tests for DAG functionality.""" + + def test_dag_root_build_workflow_fallback(self): + """Test complete DAGRoot workflow with fallback (no IPLD).""" + with patch("sdk.dag.IPLD_AVAILABLE", False): + dag_root = DAGRoot.new() + + dag_root.add_link("cid1", 1024, 512) + dag_root.add_link("cid2", 1024, 512) + + result = dag_root.build() + + assert isinstance(result, str) + assert result.startswith("bafybeig") + + def test_build_dag_workflow_small_data(self): + """Test complete build_dag workflow with small data.""" + data = b"Hello, World!" + reader = io.BytesIO(data) + + with patch("sdk.dag.IPLD_AVAILABLE", False): + chunk_dag = build_dag(None, reader, block_size=1024) + + assert chunk_dag.raw_data_size == len(data) + assert len(chunk_dag.blocks) >= 1 + assert chunk_dag.blocks[0].data == data + + def test_varint_roundtrip_with_sizes(self): + """Test varint encoding/decoding with file sizes.""" + sizes = [1024, 65536, 1048576, 1024 * 1024 * 100] -def test_placeholder(): - """Placeholder test to keep file valid.""" - pass + for size in sizes: + encoded = _encode_varint(size) + decoded, _ = _decode_varint(encoded) + assert decoded == size diff --git a/tests/unit/test_erasure_code.py b/tests/unit/test_erasure_code.py new file mode 100644 index 0000000..7d68e32 --- /dev/null +++ b/tests/unit/test_erasure_code.py @@ -0,0 +1,373 @@ +import math +from unittest.mock import Mock, patch + +import pytest + +from private.erasure_code.erasure_code import ErasureCode, missing_shards_idx, split_into_blocks + + +class TestMissingShardsIdx: + + def test_missing_shards_idx_basic(self): + result = missing_shards_idx(5, 2) + assert isinstance(result, list) + assert len(result) == 10 + assert [0, 1] in result + assert [4, 3] in result or [3, 4] in result + + def test_missing_shards_idx_single(self): + result = missing_shards_idx(3, 1) + assert len(result) == 3 + assert [0] in result + assert [1] in result + assert [2] in result + + def test_missing_shards_idx_none(self): + result = missing_shards_idx(5, 0) + assert len(result) == 1 + assert result[0] == [] + + +class TestSplitIntoBlocks: + + def test_split_into_blocks_exact(self): + data = b"123456789012" + blocks = split_into_blocks(data, 4) + assert len(blocks) == 3 + assert blocks[0] == b"1234" + assert blocks[1] == b"5678" + assert blocks[2] == b"9012" + + def test_split_into_blocks_with_padding(self): + data = b"12345" + blocks = split_into_blocks(data, 3) + assert len(blocks) == 2 + assert blocks[0] == b"123" + assert blocks[1] == b"45\x00" + + def test_split_into_blocks_empty(self): + data = b"" + blocks = split_into_blocks(data, 4) + assert len(blocks) == 0 + + def test_split_into_blocks_single(self): + data = b"ab" + blocks = split_into_blocks(data, 5) + assert len(blocks) == 1 + assert blocks[0] == b"ab\x00\x00\x00" + + +class TestErasureCodeInit: + + def test_init_valid(self): + ec = ErasureCode(4, 2) + assert ec.data_blocks == 4 + assert ec.parity_blocks == 2 + assert ec.total_shards == 6 + + def test_init_invalid_data_blocks_zero(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(0, 2) + + def test_init_invalid_data_blocks_negative(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(-1, 2) + + def test_init_invalid_parity_blocks_zero(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(4, 0) + + def test_init_invalid_parity_blocks_negative(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(4, -2) + + def test_new_classmethod(self): + ec = ErasureCode.new(3, 2) + assert isinstance(ec, ErasureCode) + assert ec.data_blocks == 3 + assert ec.parity_blocks == 2 + + +class TestErasureCodeEncode: + + def test_encode_basic(self): + ec = ErasureCode(4, 2) + data = b"Hello, World! This is test data." + + encoded = ec.encode(data) + + assert isinstance(encoded, bytes) + assert len(encoded) >= len(data) + expected_shard_size = math.ceil(len(data) / 4) + expected_total = 6 * expected_shard_size + assert len(encoded) == expected_total + + def test_encode_empty_data(self): + ec = ErasureCode(2, 1) + data = b"" + + encoded = ec.encode(data) + + assert isinstance(encoded, bytes) + + def test_encode_small_data(self): + ec = ErasureCode(2, 1) + data = b"Hi" + + encoded = ec.encode(data) + + assert len(encoded) >= len(data) + expected_shard_size = math.ceil(len(data) / 2) + assert len(encoded) == 3 * expected_shard_size + + def test_encode_large_data(self): + ec = ErasureCode(8, 4) + data = b"x" * 10000 + + encoded = ec.encode(data) + + expected_shard_size = math.ceil(len(data) / 8) + expected_total = 12 * expected_shard_size + assert len(encoded) == expected_total + + +class TestErasureCodeExtractData: + + def test_extract_data_no_errors(self): + ec = ErasureCode(4, 2) + data = b"Test data for erasure coding" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_extract_data_with_erase_pos(self): + ec = ErasureCode(4, 2) + data = b"Test data with erasure" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + + erase_pos = list(range(0, shard_size)) + + corrupted = bytearray(encoded) + for pos in erase_pos: + corrupted[pos] = 0 + + decoded = ec.extract_data(bytes(corrupted), len(data), erase_pos=erase_pos) + + assert decoded == data + + def test_extract_data_too_many_errors(self): + ec = ErasureCode(3, 1) + data = b"Test" + + encoded = ec.encode(data) + shard_size = len(encoded) // 4 + + corrupted = bytearray(encoded) + for i in range(0, shard_size * 2): + corrupted[i] = 0xFF + + with pytest.raises(ValueError, match="Decoding error"): + ec.extract_data(bytes(corrupted), len(data)) + + def test_extract_data_empty(self): + ec = ErasureCode(2, 1) + data = b"" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, 0) + + assert decoded == b"" + + +class TestErasureCodeExtractDataBlocks: + + def test_extract_data_blocks_all_present(self): + ec = ErasureCode(4, 2) + data = b"Hello, erasure coding blocks!" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + blocks = split_into_blocks(encoded, shard_size) + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + def test_extract_data_blocks_with_missing(self): + ec = ErasureCode(4, 2) + data = b"Test missing blocks" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + blocks = split_into_blocks(encoded, shard_size) + + blocks[0] = None + blocks[2] = None + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + def test_extract_data_blocks_no_blocks(self): + ec = ErasureCode(3, 2) + + with pytest.raises(ValueError, match="No blocks provided"): + ec.extract_data_blocks([], 100) + + def test_extract_data_blocks_all_missing(self): + ec = ErasureCode(3, 2) + blocks = [None, None, None, None, None] + + with pytest.raises(ValueError, match="All blocks are missing"): + ec.extract_data_blocks(blocks, 100) + + def test_extract_data_blocks_wrong_count(self): + ec = ErasureCode(4, 2) + blocks = [b"block1", b"block2"] + + with pytest.raises(ValueError, match="Expected 6 blocks"): + ec.extract_data_blocks(blocks, 100) + + def test_extract_data_blocks_partial_missing(self): + ec = ErasureCode(3, 1) + data = b"Partial missing test" + + encoded = ec.encode(data) + shard_size = len(encoded) // 4 + blocks = split_into_blocks(encoded, shard_size) + + blocks[1] = None + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + +class TestErasureCodeRoundtrip: + + def test_roundtrip_simple(self): + ec = ErasureCode(5, 3) + data = b"Simple roundtrip test" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_roundtrip_with_unicode(self): + ec = ErasureCode(4, 2) + data = "Hello δΈ–η•Œ! 🌍".encode("utf-8") + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + assert decoded.decode("utf-8") == "Hello δΈ–η•Œ! 🌍" + + def test_roundtrip_binary_data(self): + ec = ErasureCode(6, 2) + data = bytes(range(256)) + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_roundtrip_large_file(self): + ec = ErasureCode(10, 4) + data = b"x" * 50000 + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + assert len(decoded) == 50000 + + +class TestErasureCodeEdgeCases: + + def test_single_byte_data(self): + ec = ErasureCode(2, 1) + data = b"x" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_high_redundancy(self): + ec = ErasureCode(2, 10) + data = b"High redundancy test" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_many_data_blocks(self): + ec = ErasureCode(20, 5) + data = b"y" * 1000 + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_equal_data_parity(self): + ec = ErasureCode(5, 5) + data = b"Equal data and parity" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + +@pytest.mark.integration +class TestErasureCodeIntegration: + + def test_full_workflow(self): + ec = ErasureCode(4, 2) + original_data = b"Integration test data for erasure coding" + + encoded_data = ec.encode(original_data) + + shard_size = len(encoded_data) // 6 + blocks = split_into_blocks(encoded_data, shard_size) + + assert len(blocks) == 6 + + blocks[1] = None + + recovered_data = ec.extract_data_blocks(blocks, len(original_data)) + + assert recovered_data == original_data + + def test_multiple_missing_blocks(self): + ec = ErasureCode(6, 3) + data = b"Test recovery with multiple missing blocks" + + encoded = ec.encode(data) + shard_size = len(encoded) // 9 + blocks = split_into_blocks(encoded, shard_size) + + blocks[0] = None + blocks[4] = None + blocks[7] = None + + recovered = ec.extract_data_blocks(blocks, len(data)) + + assert recovered == data + + def test_stress_test(self): + ec = ErasureCode(8, 4) + + for size in [10, 100, 1000, 5000]: + data = b"z" * size + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + assert decoded == data diff --git a/tests/unit/test_sdk_ipc.py b/tests/unit/test_sdk_ipc.py index e46bd75..9ca1618 100644 --- a/tests/unit/test_sdk_ipc.py +++ b/tests/unit/test_sdk_ipc.py @@ -10,6 +10,148 @@ class TestCreateBucket: """Test create bucket functionality.""" + +import io +from datetime import datetime +from unittest.mock import MagicMock, Mock, call, patch + +import pytest + +from sdk.config import SDKConfig, SDKError +from sdk.model import ( + Chunk, + FileBlockUpload, + IPCBucket, + IPCBucketCreateResult, + IPCFileDownload, + IPCFileMeta, + IPCFileUpload, +) +from sdk.sdk_ipc import IPC, TxWaitSignal, encryption_key, maybe_encrypt_metadata, to_ipc_proto_chunk + + +class TestTxWaitSignal: + + def test_init(self): + chunk = Mock() + tx = "0x123456" + signal = TxWaitSignal(chunk, tx) + + assert signal.FileUploadChunk == chunk + assert signal.Transaction == tx + + +class TestEncryptionKey: + + def test_encryption_key_empty_parent(self): + result = encryption_key(b"", "bucket", "file") + assert result == b"" + + @patch("sdk.sdk_ipc.derive_key") + def test_encryption_key_with_data(self, mock_derive): + parent = b"parent_key_32bytes_test123456789" + mock_derive.return_value = b"derived" + + result = encryption_key(parent, "bucket", "file") + + assert result == b"derived" + mock_derive.assert_called_once_with(parent, b"bucket/file") + + @patch("sdk.sdk_ipc.derive_key") + def test_encryption_key_multiple_info(self, mock_derive): + parent = b"key" + mock_derive.return_value = b"result" + + result = encryption_key(parent, "a", "b", "c") + + mock_derive.assert_called_once_with(parent, b"a/b/c") + + +class TestMaybeEncryptMetadata: + + def test_maybe_encrypt_metadata_no_key(self): + result = maybe_encrypt_metadata("plain_value", "path", b"") + assert result == "plain_value" + + @patch("sdk.sdk_ipc.derive_key") + @patch("sdk.sdk_ipc.encrypt") + def test_maybe_encrypt_metadata_with_key(self, mock_encrypt, mock_derive): + key = b"encryption_key_32bytes_test12345" + mock_derive.return_value = b"file_key" + mock_encrypt.return_value = b"\x01\x02\x03" + + result = maybe_encrypt_metadata("value", "path/to/file", key) + + assert result == "010203" + mock_derive.assert_called_once_with(key, b"path/to/file") + mock_encrypt.assert_called_once_with(b"file_key", b"value", b"metadata") + + @patch("sdk.sdk_ipc.derive_key") + @patch("sdk.sdk_ipc.encrypt") + def test_maybe_encrypt_metadata_error(self, mock_encrypt, mock_derive): + key = b"encryption_key_32bytes_test12345" + mock_derive.side_effect = Exception("Derive failed") + + with pytest.raises(SDKError, match="failed to encrypt metadata"): + maybe_encrypt_metadata("value", "path", key) + + +class TestToIPCProtoChunk: + + @patch("sdk.sdk_ipc.CID") + @patch("sdk.sdk_ipc.ipcnodeapi_pb2") + def test_to_ipc_proto_chunk_basic(self, mock_pb2, mock_cid): + mock_cid.decode.return_value = b"bytes" + mock_block_class = Mock() + mock_pb2.IPCChunk.Block = mock_block_class + mock_pb2.IPCChunk = Mock() + + blocks = [ + FileBlockUpload(cid="cid1", data=b"data1"), + FileBlockUpload(cid="cid2", data=b"data2"), + ] + + cids, sizes, proto_chunk, err = to_ipc_proto_chunk("chunk_cid", 0, 100, blocks) + + assert err is None + assert isinstance(cids, list) + assert isinstance(sizes, list) + assert len(sizes) == 2 + + def test_to_ipc_proto_chunk_empty_blocks(self): + cids, sizes, proto_chunk, err = to_ipc_proto_chunk("cid", 0, 100, []) + + assert err is None + assert cids == [] + assert sizes == [] + + +class TestIPCInit: + + def test_ipc_init(self): + mock_client = Mock() + mock_conn = Mock() + mock_ipc_instance = Mock() + config = SDKConfig( + address="test:5500", + max_concurrency=5, + block_part_size=128 * 1024, + use_connection_pool=True, + streaming_max_blocks_in_chunk=10, + ) + + ipc = IPC(mock_client, mock_conn, mock_ipc_instance, config) + + assert ipc.client == mock_client + assert ipc.conn == mock_conn + assert ipc.ipc == mock_ipc_instance + assert ipc.max_concurrency == 5 + assert ipc.block_part_size == 128 * 1024 + assert ipc.max_blocks_in_chunk == 10 + + +class TestCreateBucket: + def setup_method(self): self.mock_client = Mock() self.mock_conn = Mock() @@ -26,8 +168,14 @@ def setup_method(self): ) self.ipc = IPC(self.mock_client, self.mock_conn, self.mock_ipc, self.config) + self.config = SDKConfig(address="test:5500") + self.ipc = IPC(self.mock_client, self.mock_conn, self.mock_ipc, self.config) + + def test_create_bucket_invalid_name(self): + with pytest.raises(SDKError, match="invalid bucket name"): + self.ipc.create_bucket(None, "ab") + def test_create_bucket_success(self): - """Test successful bucket creation.""" mock_receipt = Mock() mock_receipt.status = 1 mock_receipt.blockNumber = 100