diff --git a/private/erasure_code/__init__.py b/private/erasure_code/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/private/erasure_code/erasure_code.py b/private/erasure_code/erasure_code.py new file mode 100644 index 0000000..c054478 --- /dev/null +++ b/private/erasure_code/erasure_code.py @@ -0,0 +1,117 @@ +import math +from itertools import combinations + +from reedsolo import ReedSolomonError, RSCodec + + +def missing_shards_idx(n, k): + return [list(combo) for combo in combinations(range(n), k)] + + +def split_into_blocks(encoded: bytes, shard_size: int): + blocks = [] + for offset in range(0, len(encoded), shard_size): + block = encoded[offset : offset + shard_size] + if len(block) < shard_size: + block = block.ljust(shard_size, b"\x00") + blocks.append(block) + return blocks + + +class ErasureCode: + def __init__(self, data_blocks: int, parity_blocks: int): + if data_blocks <= 0 or parity_blocks <= 0: + raise ValueError("Data and parity shards must be > 0") + self.data_blocks = data_blocks + self.parity_blocks = parity_blocks + self.total_shards = data_blocks + parity_blocks + + @classmethod + def new(cls, data_blocks: int, parity_blocks: int): + return cls(data_blocks, parity_blocks) + + def encode(self, data: bytes) -> bytes: + total_size = len(data) + shard_size = math.ceil(total_size / self.data_blocks) + padded_data = data.ljust(self.data_blocks * shard_size, b"\x00") + + shards = [bytearray(padded_data[i * shard_size : (i + 1) * shard_size]) for i in range(self.data_blocks)] + parity_shards = [bytearray(shard_size) for _ in range(self.parity_blocks)] + rsc = RSCodec(self.parity_blocks) + + for j in range(shard_size): + message = bytearray(self.data_blocks) + for i in range(self.data_blocks): + message[i] = shards[i][j] + + encoded_msg = rsc.encode(message) + parity_bytes = encoded_msg[self.data_blocks :] + + for i in range(self.parity_blocks): + parity_shards[i][j] = parity_bytes[i] + + all_shards = shards + parity_shards + return b"".join(all_shards) + + def extract_data(self, encoded: bytes, original_data_size: int, erase_pos=None) -> bytes: + shard_size = len(encoded) // self.total_shards + + shards = [bytearray(encoded[i * shard_size : (i + 1) * shard_size]) for i in range(self.total_shards)] + + rsc = RSCodec(self.parity_blocks) + decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)] + + erased_shard_indices = set() + if erase_pos: + for pos in erase_pos: + erased_shard_indices.add(pos // shard_size) + + erased_shard_indices_list = list(erased_shard_indices) + + for j in range(shard_size): + message = bytearray(self.total_shards) + for i in range(self.total_shards): + message[i] = shards[i][j] + + try: + decoded_msg, _, _ = rsc.decode(message, erase_pos=erased_shard_indices_list) + for i in range(self.data_blocks): + decoded_shards[i][j] = decoded_msg[i] + except ReedSolomonError as e: + raise ValueError(f"Decoding error at byte {j}: {str(e)}") + + full_data = b"".join(decoded_shards) + return full_data[:original_data_size] + + def extract_data_blocks(self, blocks, original_data_size: int) -> bytes: + if not blocks: + raise ValueError("No blocks provided") + + valid_block = next((b for b in blocks if b is not None), None) + if valid_block is None: + raise ValueError("All blocks are missing") + + shard_size = len(valid_block) + if len(blocks) != self.total_shards: + raise ValueError(f"Expected {self.total_shards} blocks, got {len(blocks)}") + + erase_pos = [i for i, b in enumerate(blocks) if b is None] + shards = [bytearray(b) if b is not None else bytearray(shard_size) for b in blocks] + + rsc = RSCodec(self.parity_blocks) + decoded_shards = [bytearray(shard_size) for _ in range(self.data_blocks)] + + for j in range(shard_size): + stripe = bytearray(self.total_shards) + for i in range(self.total_shards): + stripe[i] = shards[i][j] + + try: + decoded_stripe, _, _ = rsc.decode(stripe, erase_pos=erase_pos) + for i in range(self.data_blocks): + decoded_shards[i][j] = decoded_stripe[i] + except ReedSolomonError as e: + raise ValueError("Decoding error: " + str(e)) + + full_data = b"".join(decoded_shards) + return full_data[:original_data_size] diff --git a/private/httpext/__init__.py b/private/httpext/__init__.py index c91bb3b..e4b89f5 100644 --- a/private/httpext/__init__.py +++ b/private/httpext/__init__.py @@ -1,6 +1,3 @@ -# HTTP-related utility functions for internal Akave SDK components. +from .httpext import HTTPExtClient, RangeDownloadResult, HTTPExtError, RangeNotSatisfiableError - -from .httpext import range_download - -__all__ = ["range_download"] +__all__ = ["HTTPExtClient", "RangeDownloadResult", "HTTPExtError", "RangeNotSatisfiableError"] diff --git a/private/httpext/httpext.py b/private/httpext/httpext.py index 3e10222..2325e92 100644 --- a/private/httpext/httpext.py +++ b/private/httpext/httpext.py @@ -1,53 +1,332 @@ -import logging -from typing import Optional - +""" +HTTP Extension module for range-based downloads. +This module provides HTTP Range header support for partial content downloads, +enabling efficient retrieval of specific byte ranges from remote resources. +""" +from dataclasses import dataclass +from typing import Optional, Tuple, BinaryIO import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry -def range_download( - client: requests.Session, - url: str, - offset: int, - length: int, - timeout: Optional[float] = 10.0, -) -> bytes: - """ - Download a specific byte range from *url* using the given HTTP client. +class HTTPExtError(Exception): + """Base exception for HTTP extension errors.""" + pass + + +class RangeNotSatisfiableError(HTTPExtError): + """Raised when the requested range cannot be satisfied (HTTP 416).""" + def __init__(self, message: str, content_length: Optional[int] = None): + super().__init__(message) + self.content_length = content_length + + +class NetworkError(HTTPExtError): + """Raised when a network-related error occurs.""" + pass + + +class InvalidRangeError(HTTPExtError): + """Raised when an invalid range is specified.""" + pass - The function raises ``ValueError`` if the range is invalid and a generic - ``Exception`` for network or HTTP errors. + +@dataclass +class RangeDownloadResult: + """Result of a range download operation.""" + data: bytes + start: int + end: int + total_size: Optional[int] + content_length: int + + @property + def is_partial(self) -> bool: + """Returns True if this is a partial content response.""" + return self.total_size is not None and self.content_length < self.total_size + + +class HTTPExtClient: """ - if length <= 0 or offset < 0: - raise ValueError("length must be positive and offset must be non-negative") + HTTP client with support for range-based downloads. + This client handles HTTP Range requests for partial content retrieval, + with proper handling of various response codes and error conditions. + """ + DEFAULT_TIMEOUT = 30 + DEFAULT_RETRIES = 3 + DEFAULT_BACKOFF_FACTOR = 0.3 - end = offset + length - 1 - headers = {"Range": f"bytes={offset}-{end}"} + def __init__( + self, + timeout: int = DEFAULT_TIMEOUT, + retries: int = DEFAULT_RETRIES, + backoff_factor: float = DEFAULT_BACKOFF_FACTOR, + ): + """ + Initialize the HTTP extension client. + Args: + timeout: Request timeout in seconds. + retries: Number of retry attempts for failed requests. + backoff_factor: Backoff factor for retry delays. + """ + self.timeout = timeout + self.session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=[500, 502, 503, 504], + allowed_methods=["GET", "HEAD"], + ) + adapter = HTTPAdapter(max_retries=retry_strategy) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) - try: - response = client.get(url, headers=headers, timeout=timeout) - except requests.RequestException as exc: - raise Exception(f"request failed: {exc}") from exc + def close(self) -> None: + """Close the HTTP session and release resources.""" + self.session.close() - try: - # Some CDNs may return 200 OK for range requests. - if response.status_code not in (requests.codes.partial_content, requests.codes.ok): - try: - body = response.content - except Exception as body_exc: # pragma: no cover - extremely rare - logging.warning("failed to read error response body: %s", body_exc) - body = b"" + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + return False + + def range_download( + self, + url: str, + start: int, + end: Optional[int] = None, + headers: Optional[dict] = None, + ) -> RangeDownloadResult: + """ + Download a specific byte range from a URL. + Implements HTTP Range requests as per RFC 7233. Supports both + bounded ranges (start-end) and suffix ranges (start-). + Args: + url: The URL to download from. + start: The starting byte position (0-indexed). + end: The ending byte position (inclusive). If None, downloads to end of file. + headers: Additional headers to include in the request. + Returns: + RangeDownloadResult containing the downloaded data and metadata. + Raises: + InvalidRangeError: If the range parameters are invalid. + RangeNotSatisfiableError: If the server returns 416 (Range Not Satisfiable). + NetworkError: If a network error occurs. + HTTPExtError: For other HTTP errors. + """ + self._validate_range(start, end) + range_header = self._build_range_header(start, end) + request_headers = {"Range": range_header} + if headers: + request_headers.update(headers) - body_text = body.decode(errors="replace") - raise Exception(f"download failed with status {response.status_code}: {body_text}") + try: + response = self.session.get( + url, + headers=request_headers, + timeout=self.timeout, + ) + return self._handle_response(response, start, end) + except requests.exceptions.Timeout as e: + raise NetworkError(f"Request timed out: {e}") from e + except requests.exceptions.ConnectionError as e: + raise NetworkError(f"Connection error: {e}") from e + except requests.exceptions.RequestException as e: + raise HTTPExtError(f"Request failed: {e}") from e + + def range_download_to_file( + self, + url: str, + start: int, + end: Optional[int], + writer: BinaryIO, + headers: Optional[dict] = None, + chunk_size: int = 8192, + ) -> RangeDownloadResult: + """ + Download a specific byte range from a URL directly to a file. + This method streams the response to avoid loading large ranges into memory. + Args: + url: The URL to download from. + start: The starting byte position (0-indexed). + end: The ending byte position (inclusive). If None, downloads to end of file. + writer: A binary file-like object to write the data to. + headers: Additional headers to include in the request. + chunk_size: Size of chunks to read/write at a time. + Returns: + RangeDownloadResult containing metadata (data field will be empty). + Raises: + InvalidRangeError: If the range parameters are invalid. + RangeNotSatisfiableError: If the server returns 416 (Range Not Satisfiable). + NetworkError: If a network error occurs. + HTTPExtError: For other HTTP errors. + """ + self._validate_range(start, end) + range_header = self._build_range_header(start, end) + request_headers = {"Range": range_header} + if headers: + request_headers.update(headers) + + try: + response = self.session.get( + url, + headers=request_headers, + timeout=self.timeout, + stream=True, + ) + # Handle error responses before streaming + if response.status_code == 416: + content_length = self._parse_content_length_from_416(response) + raise RangeNotSatisfiableError( + f"Range not satisfiable: {start}-{end}", + content_length=content_length, + ) + if response.status_code not in (200, 206): + raise HTTPExtError( + f"Unexpected status code: {response.status_code}" + ) + # Stream content to writer + bytes_written = 0 + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + writer.write(chunk) + bytes_written += len(chunk) + + # Parse content range + actual_start, actual_end, total_size = self._parse_content_range( + response.headers.get("Content-Range") + ) + + return RangeDownloadResult( + data=b"", + start=actual_start if actual_start is not None else start, + end=actual_end if actual_end is not None else start + bytes_written - 1, + total_size=total_size, + content_length=bytes_written, + ) + except requests.exceptions.Timeout as e: + raise NetworkError(f"Request timed out: {e}") from e + except requests.exceptions.ConnectionError as e: + raise NetworkError(f"Connection error: {e}") from e + except requests.exceptions.RequestException as e: + raise HTTPExtError(f"Request failed: {e}") from e + + def get_content_length(self, url: str, headers: Optional[dict] = None) -> Optional[int]: + """ + Get the content length of a resource using a HEAD request. + Args: + url: The URL to query. + headers: Additional headers to include in the request. + Returns: + The content length in bytes, or None if not available. + Raises: + NetworkError: If a network error occurs. + HTTPExtError: For other HTTP errors. + """ try: - data = response.content - except requests.RequestException as exc: - raise Exception(f"failed to read response body: {exc}") from exc + response = self.session.head(url, headers=headers, timeout=self.timeout) + response.raise_for_status() + content_length = response.headers.get("Content-Length") + if content_length: + return int(content_length) + return None + except requests.exceptions.Timeout as e: + raise NetworkError(f"Request timed out: {e}") from e + except requests.exceptions.ConnectionError as e: + raise NetworkError(f"Connection error: {e}") from e + except requests.exceptions.RequestException as e: + raise HTTPExtError(f"Request failed: {e}") from e + + def _validate_range(self, start: int, end: Optional[int]) -> None: + """Validate range parameters.""" + if start < 0: + raise InvalidRangeError("Start position cannot be negative") + if end is not None: + if end < 0: + raise InvalidRangeError("End position cannot be negative") + if end < start: + raise InvalidRangeError("End position cannot be less than start position") + + def _build_range_header(self, start: int, end: Optional[int]) -> str: + """Build the HTTP Range header value.""" + if end is not None: + return f"bytes={start}-{end}" + return f"bytes={start}-" + + def _handle_response( + self, response: requests.Response, start: int, end: Optional[int] + ) -> RangeDownloadResult: + """Handle the HTTP response and build the result.""" + if response.status_code == 416: + content_length = self._parse_content_length_from_416(response) + raise RangeNotSatisfiableError( + f"Range not satisfiable: {start}-{end}", + content_length=content_length, + ) + + if response.status_code == 206: + # Partial content - parse Content-Range header + actual_start, actual_end, total_size = self._parse_content_range( + response.headers.get("Content-Range") + ) + return RangeDownloadResult( + data=response.content, + start=actual_start if actual_start is not None else start, + end=actual_end if actual_end is not None else start + len(response.content) - 1, + total_size=total_size, + content_length=len(response.content), + ) + + if response.status_code == 200: + # Server doesn't support range requests, returned full content + return RangeDownloadResult( + data=response.content, + start=0, + end=len(response.content) - 1, + total_size=len(response.content), + content_length=len(response.content), + ) + + raise HTTPExtError(f"Unexpected status code: {response.status_code}") + + def _parse_content_range( + self, content_range: Optional[str] + ) -> Tuple[Optional[int], Optional[int], Optional[int]]: + """ + Parse the Content-Range header. + Format: bytes start-end/total or bytes start-end/* + Returns: + Tuple of (start, end, total_size). total_size may be None if unknown. + """ + if not content_range: + return None, None, None - return data - finally: try: - response.close() - except Exception as close_exc: # pragma: no cover - defensive - logging.debug("error closing HTTP response: %s", close_exc) + # Format: "bytes start-end/total" + if not content_range.startswith("bytes "): + return None, None, None + range_part = content_range[6:] # Remove "bytes " + range_spec, size_spec = range_part.split("/") + start_str, end_str = range_spec.split("-") + start = int(start_str) + end = int(end_str) + total_size = None if size_spec == "*" else int(size_spec) + return start, end, total_size + except (ValueError, IndexError): + return None, None, None + + def _parse_content_length_from_416(self, response: requests.Response) -> Optional[int]: + """Extract content length from a 416 response if available.""" + content_range = response.headers.get("Content-Range") + if content_range: + # Format might be "bytes */total" + try: + if content_range.startswith("bytes */"): + return int(content_range[8:]) + except ValueError: + pass + return None diff --git a/sdk/config.py b/sdk/config.py index 254bb0e..d7832ac 100644 --- a/sdk/config.py +++ b/sdk/config.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List, Optional +from private.erasure_code.erasure_code import ErasureCode from private.memory.memory import Size BLOCK_SIZE = 1 * Size.MB @@ -34,15 +35,23 @@ def __init__( private_key: str, storage_contract_address: str, access_contract_address: Optional[str] = None, + policy_factory_contract_address: Optional[str] = None, ): self.dial_uri = dial_uri self.private_key = private_key self.storage_contract_address = storage_contract_address self.access_contract_address = access_contract_address + self.policy_factory_contract_address = policy_factory_contract_address or "" @staticmethod def default(): - return Config(dial_uri="", private_key="", storage_contract_address="", access_contract_address="") + return Config( + dial_uri="", + private_key="", + storage_contract_address="", + access_contract_address="", + policy_factory_contract_address="", + ) ## [SDK Error Class] @@ -55,8 +64,8 @@ class SDKError(Exception): ## [Validation Functions] -# Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum def validate_hex_string(hex_string: str) -> bool: + """Basic validation: expect hex string like '0x' + 8 hex chars (4 bytes) minimum""" if not hex_string.startswith("0x"): return False if len(hex_string) < 10: @@ -66,24 +75,23 @@ def validate_hex_string(hex_string: str) -> bool: ## [Test Configurations] + DEFAULT_CONFIG_TEST_STREAMING_CONN = { "AKAVE_SDK_NODE": "connect.akave.ai:5000", "ENCRYPTION_KEY": "", } DEFAULT_CONFIG_TEST_SDK_CONN = { - "AKAVE_SDK_NODE": "connect.akave.ai:5000", # For streaming operations - "AKAVE_IPC_NODE": "connect.akave.ai:5500", # For IPC operations + "AKAVE_SDK_NODE": "connect.akave.ai:5000", + "AKAVE_IPC_NODE": "connect.akave.ai:5500", "ETHEREUM_NODE_URL": "https://n3-us.akave.ai/ext/bc/2JMWNmZbYvWcJRPPy1siaDBZaDGTDAaqXoY5UBKh4YrhNFzEce/rpc", - "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF", # Will be obtained from node - "ACCESS_CONTRACT_ADDRESS": "", # Will be obtained from node + "STORAGE_CONTRACT_ADDRESS": "0x9Aa8ff1604280d66577ecB5051a3833a983Ca3aF", + "ACCESS_CONTRACT_ADDRESS": "", } -## [Error Handling Functions] +## [Known Error Strings] -# List of known error strings from the smart contracts -# Replace these with the actual error strings from your contracts KNOWN_ERROR_STRINGS: List[str] = [ "Storage: bucket doesn't exist", @@ -92,16 +100,15 @@ def validate_hex_string(hex_string: str) -> bool: "Storage: file exists", "AccessManager: caller is not the owner", "AccessManager: caller is not authorized", - # Add all other known error strings here... ] @dataclass class SDKConfig: address: str - max_concurrency: int - block_part_size: int - use_connection_pool: bool + max_concurrency: int = 10 + block_part_size: int = 1024 * 1024 + use_connection_pool: bool = True parity_blocks_count: int = 0 chunk_buffer: int = 10 encryption_key: Optional[bytes] = None @@ -111,3 +118,4 @@ class SDKConfig: max_retries: Optional[int] = 3 backoff_delay: Optional[int] = 1 ipc_address: Optional[str] = None + erasure_code: Optional[ErasureCode] = None diff --git a/sdk/sdk_ipc.py b/sdk/sdk_ipc.py index 12d1d8b..b0ea9ae 100644 --- a/sdk/sdk_ipc.py +++ b/sdk/sdk_ipc.py @@ -8,7 +8,6 @@ import threading import time from datetime import datetime -from hashlib import sha256 from typing import Any, Callable, Dict, List, Optional, Tuple, Union import grpc @@ -772,9 +771,7 @@ def _calculate_file_id(self, bucket_id: bytes, file_name: str) -> bytes: from Crypto.Hash import keccak combined = bucket_id + file_name.encode() - hash_obj = keccak.new(digest_bits=256) - hash_obj.update(combined) - return hash_obj.digest() + return keccak(combined) except ImportError: raise SDKError("Failed to import required modules for file ID calculation") except Exception as e: @@ -1361,7 +1358,11 @@ def download_chunk_blocks( except Exception as e: raise SDKError(f"failed to download block: {str(e)}") - data = b"".join([b for b in blocks if b is not None]) + if hasattr(self, "erasure_code") and self.erasure_code is not None: + + data = self.erasure_code.extract_data_blocks(blocks, chunk_download.size) + else: + data = b"".join([b for b in blocks if b is not None]) if file_encryption_key: from private.encryption import decrypt diff --git a/setup.py b/setup.py index 7d2e2cc..289e920 100644 --- a/setup.py +++ b/setup.py @@ -23,4 +23,4 @@ ], python_requires=">=3.9", install_requires=requirements, -) \ No newline at end of file +) diff --git a/test_ipc_upload_integration.py b/test_ipc_upload_integration.py new file mode 100755 index 0000000..c850d03 --- /dev/null +++ b/test_ipc_upload_integration.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +import os +import time +from pathlib import Path + +from akavesdk import SDK, SDKConfig + + +def main(): + NODE_ADDRESS = "connect.akave.ai:5500" + PRIVATE_KEY = "a5c223e956644f1ba11f0dcc6f3df4992184ff3c919223744d0cf1db33dab4d6" + BUCKET_NAME = "finalfr" + + script_dir = Path(__file__).parent + test_file_path = script_dir / "random_3mb_file.bin" + + if not test_file_path.exists(): + print(f"❌ Error: Test file not found: {test_file_path}") + print(f" Please ensure random_3mb_file.bin exists in {script_dir}") + return 1 + + file_size = os.path.getsize(test_file_path) + # Use unique filename with timestamp to avoid conflicts + timestamp = int(time.time()) + file_name = f"test_{timestamp}.bin" + + print(f"\n{'='*70}") + print(f"πŸš€ Akave IPC Upload Integration Test") + print(f"{'='*70}") + print(f"πŸ“‘ Node Address: {NODE_ADDRESS}") + print(f"πŸ“¦ Bucket Name: {BUCKET_NAME}") + print(f"πŸ“„ File Name: {file_name}") + print(f"πŸ“‚ Source File: {test_file_path}") + print(f"πŸ“Š File Size: {file_size:,} bytes ({file_size / (1024*1024):.2f} MB)") + print(f"{'='*70}\n") + + try: + print("πŸ”§ Step 1: Initializing SDK...") + config = SDKConfig( + address=NODE_ADDRESS, + private_key=PRIVATE_KEY, + max_concurrency=5, + block_part_size=128 * 1024, + use_connection_pool=True, + chunk_buffer=10, + ) + sdk = SDK(config) + print("βœ… SDK initialized successfully\n") + + print("πŸ”§ Step 2: Creating IPC instance...") + ipc = sdk.ipc() + print("βœ… IPC instance created\n") + + print(f"πŸ”§ Step 3: Checking/Creating bucket '{BUCKET_NAME}'...") + existing_bucket = ipc.view_bucket(None, BUCKET_NAME) + + if existing_bucket is None: + print(f" Bucket doesn't exist, creating...") + result = ipc.create_bucket(None, BUCKET_NAME) + print(f"βœ… Bucket created successfully") + print(f" Bucket ID: {result.id}") + print(f" Bucket Name: {result.name}") + print(f" Created At: {result.created_at}") + time.sleep(2) + else: + print(f"βœ… Bucket already exists") + print(f" Bucket ID: {existing_bucket.id}") + print(f" Bucket Name: {existing_bucket.name}\n") + + print(f"πŸ”§ Step 4: Uploading file...") + print(f" File: {file_name}") + print(f" This may take a while for a {file_size / (1024*1024):.2f} MB file...") + print(f" Note: upload() will create file upload and handle all transactions\n") + + start_time = time.time() + + with open(test_file_path, "rb") as f: + file_meta = ipc.upload(None, BUCKET_NAME, file_name, f) + + upload_duration = time.time() - start_time + upload_speed = (file_size / (1024 * 1024)) / upload_duration if upload_duration > 0 else 0 + + print(f"\nβœ… File uploaded successfully!") + print(f" Root CID: {file_meta.root_cid}") + print(f" File Name: {file_meta.name}") + print(f" File Size: {file_meta.size:,} bytes") + print(f" Encoded Size: {file_meta.encoded_size:,} bytes") + print(f" Upload Duration: {upload_duration:.2f} seconds") + print(f" Upload Speed: {upload_speed:.2f} MB/s\n") + + print(f"πŸ”§ Step 5: Verifying file metadata...") + try: + retrieved_meta = ipc.file_info(None, BUCKET_NAME, file_name) + + if retrieved_meta is None: + print(f"⚠️ Could not retrieve file metadata (but upload succeeded!)") + else: + print(f"βœ… File metadata verified") + print(f" Name: {retrieved_meta.name}") + print(f" Bucket: {retrieved_meta.bucket_name}") + print(f" Root CID: {retrieved_meta.root_cid}") + print(f" Size: {retrieved_meta.encoded_size:,} bytes\n") + except Exception as e: + print(f"⚠️ File verification skipped (upload succeeded!)") + print(f" Note: {str(e)}") + print(f" The file was successfully uploaded and committed\n") + retrieved_meta = None + + if retrieved_meta and retrieved_meta.root_cid != file_meta.root_cid: + print(f"⚠️ Warning: Root CID mismatch!") + print(f" Upload CID: {file_meta.root_cid}") + print(f" Retrieved CID: {retrieved_meta.root_cid}") + elif retrieved_meta: + print(f"βœ… Root CID matches upload!\n") + + print(f"πŸ”§ Step 6: Listing files in bucket...") + files = ipc.list_files(None, BUCKET_NAME) + print(f"βœ… Found {len(files)} file(s) in bucket '{BUCKET_NAME}'") + + uploaded_file = next((f for f in files if f.name == file_name), None) + if uploaded_file: + print(f"βœ… Uploaded file found in bucket listing") + else: + print(f"⚠️ Warning: Uploaded file not found in listing") + + print(f"\n{'='*70}") + print(f"βœ… Upload Test Completed Successfully!") + print(f"{'='*70}") + print(f"\nπŸ“‹ Summary:") + print(f" β€’ File: {file_name}") + print(f" β€’ Size: {file_size:,} bytes ({file_size / (1024*1024):.2f} MB)") + print(f" β€’ Root CID: {file_meta.root_cid}") + print(f" β€’ Bucket: {BUCKET_NAME}") + print(f" β€’ Upload Time: {upload_duration:.2f}s") + print(f" β€’ Upload Speed: {upload_speed:.2f} MB/s") + print(f"{'='*70}\n") + + sdk.close() + return 0 + + except Exception as e: + print(f"\n❌ Error during upload test:") + print(f" {type(e).__name__}: {str(e)}") + import traceback + + print(f"\nπŸ“‹ Full traceback:") + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + exit_code = main() + exit(exit_code) diff --git a/tests/unit/test_erasure_code.py b/tests/unit/test_erasure_code.py new file mode 100644 index 0000000..7d68e32 --- /dev/null +++ b/tests/unit/test_erasure_code.py @@ -0,0 +1,373 @@ +import math +from unittest.mock import Mock, patch + +import pytest + +from private.erasure_code.erasure_code import ErasureCode, missing_shards_idx, split_into_blocks + + +class TestMissingShardsIdx: + + def test_missing_shards_idx_basic(self): + result = missing_shards_idx(5, 2) + assert isinstance(result, list) + assert len(result) == 10 + assert [0, 1] in result + assert [4, 3] in result or [3, 4] in result + + def test_missing_shards_idx_single(self): + result = missing_shards_idx(3, 1) + assert len(result) == 3 + assert [0] in result + assert [1] in result + assert [2] in result + + def test_missing_shards_idx_none(self): + result = missing_shards_idx(5, 0) + assert len(result) == 1 + assert result[0] == [] + + +class TestSplitIntoBlocks: + + def test_split_into_blocks_exact(self): + data = b"123456789012" + blocks = split_into_blocks(data, 4) + assert len(blocks) == 3 + assert blocks[0] == b"1234" + assert blocks[1] == b"5678" + assert blocks[2] == b"9012" + + def test_split_into_blocks_with_padding(self): + data = b"12345" + blocks = split_into_blocks(data, 3) + assert len(blocks) == 2 + assert blocks[0] == b"123" + assert blocks[1] == b"45\x00" + + def test_split_into_blocks_empty(self): + data = b"" + blocks = split_into_blocks(data, 4) + assert len(blocks) == 0 + + def test_split_into_blocks_single(self): + data = b"ab" + blocks = split_into_blocks(data, 5) + assert len(blocks) == 1 + assert blocks[0] == b"ab\x00\x00\x00" + + +class TestErasureCodeInit: + + def test_init_valid(self): + ec = ErasureCode(4, 2) + assert ec.data_blocks == 4 + assert ec.parity_blocks == 2 + assert ec.total_shards == 6 + + def test_init_invalid_data_blocks_zero(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(0, 2) + + def test_init_invalid_data_blocks_negative(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(-1, 2) + + def test_init_invalid_parity_blocks_zero(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(4, 0) + + def test_init_invalid_parity_blocks_negative(self): + with pytest.raises(ValueError, match="Data and parity shards must be > 0"): + ErasureCode(4, -2) + + def test_new_classmethod(self): + ec = ErasureCode.new(3, 2) + assert isinstance(ec, ErasureCode) + assert ec.data_blocks == 3 + assert ec.parity_blocks == 2 + + +class TestErasureCodeEncode: + + def test_encode_basic(self): + ec = ErasureCode(4, 2) + data = b"Hello, World! This is test data." + + encoded = ec.encode(data) + + assert isinstance(encoded, bytes) + assert len(encoded) >= len(data) + expected_shard_size = math.ceil(len(data) / 4) + expected_total = 6 * expected_shard_size + assert len(encoded) == expected_total + + def test_encode_empty_data(self): + ec = ErasureCode(2, 1) + data = b"" + + encoded = ec.encode(data) + + assert isinstance(encoded, bytes) + + def test_encode_small_data(self): + ec = ErasureCode(2, 1) + data = b"Hi" + + encoded = ec.encode(data) + + assert len(encoded) >= len(data) + expected_shard_size = math.ceil(len(data) / 2) + assert len(encoded) == 3 * expected_shard_size + + def test_encode_large_data(self): + ec = ErasureCode(8, 4) + data = b"x" * 10000 + + encoded = ec.encode(data) + + expected_shard_size = math.ceil(len(data) / 8) + expected_total = 12 * expected_shard_size + assert len(encoded) == expected_total + + +class TestErasureCodeExtractData: + + def test_extract_data_no_errors(self): + ec = ErasureCode(4, 2) + data = b"Test data for erasure coding" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_extract_data_with_erase_pos(self): + ec = ErasureCode(4, 2) + data = b"Test data with erasure" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + + erase_pos = list(range(0, shard_size)) + + corrupted = bytearray(encoded) + for pos in erase_pos: + corrupted[pos] = 0 + + decoded = ec.extract_data(bytes(corrupted), len(data), erase_pos=erase_pos) + + assert decoded == data + + def test_extract_data_too_many_errors(self): + ec = ErasureCode(3, 1) + data = b"Test" + + encoded = ec.encode(data) + shard_size = len(encoded) // 4 + + corrupted = bytearray(encoded) + for i in range(0, shard_size * 2): + corrupted[i] = 0xFF + + with pytest.raises(ValueError, match="Decoding error"): + ec.extract_data(bytes(corrupted), len(data)) + + def test_extract_data_empty(self): + ec = ErasureCode(2, 1) + data = b"" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, 0) + + assert decoded == b"" + + +class TestErasureCodeExtractDataBlocks: + + def test_extract_data_blocks_all_present(self): + ec = ErasureCode(4, 2) + data = b"Hello, erasure coding blocks!" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + blocks = split_into_blocks(encoded, shard_size) + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + def test_extract_data_blocks_with_missing(self): + ec = ErasureCode(4, 2) + data = b"Test missing blocks" + + encoded = ec.encode(data) + shard_size = len(encoded) // 6 + blocks = split_into_blocks(encoded, shard_size) + + blocks[0] = None + blocks[2] = None + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + def test_extract_data_blocks_no_blocks(self): + ec = ErasureCode(3, 2) + + with pytest.raises(ValueError, match="No blocks provided"): + ec.extract_data_blocks([], 100) + + def test_extract_data_blocks_all_missing(self): + ec = ErasureCode(3, 2) + blocks = [None, None, None, None, None] + + with pytest.raises(ValueError, match="All blocks are missing"): + ec.extract_data_blocks(blocks, 100) + + def test_extract_data_blocks_wrong_count(self): + ec = ErasureCode(4, 2) + blocks = [b"block1", b"block2"] + + with pytest.raises(ValueError, match="Expected 6 blocks"): + ec.extract_data_blocks(blocks, 100) + + def test_extract_data_blocks_partial_missing(self): + ec = ErasureCode(3, 1) + data = b"Partial missing test" + + encoded = ec.encode(data) + shard_size = len(encoded) // 4 + blocks = split_into_blocks(encoded, shard_size) + + blocks[1] = None + + decoded = ec.extract_data_blocks(blocks, len(data)) + + assert decoded == data + + +class TestErasureCodeRoundtrip: + + def test_roundtrip_simple(self): + ec = ErasureCode(5, 3) + data = b"Simple roundtrip test" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_roundtrip_with_unicode(self): + ec = ErasureCode(4, 2) + data = "Hello δΈ–η•Œ! 🌍".encode("utf-8") + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + assert decoded.decode("utf-8") == "Hello δΈ–η•Œ! 🌍" + + def test_roundtrip_binary_data(self): + ec = ErasureCode(6, 2) + data = bytes(range(256)) + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_roundtrip_large_file(self): + ec = ErasureCode(10, 4) + data = b"x" * 50000 + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + assert len(decoded) == 50000 + + +class TestErasureCodeEdgeCases: + + def test_single_byte_data(self): + ec = ErasureCode(2, 1) + data = b"x" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_high_redundancy(self): + ec = ErasureCode(2, 10) + data = b"High redundancy test" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_many_data_blocks(self): + ec = ErasureCode(20, 5) + data = b"y" * 1000 + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + def test_equal_data_parity(self): + ec = ErasureCode(5, 5) + data = b"Equal data and parity" + + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + + assert decoded == data + + +@pytest.mark.integration +class TestErasureCodeIntegration: + + def test_full_workflow(self): + ec = ErasureCode(4, 2) + original_data = b"Integration test data for erasure coding" + + encoded_data = ec.encode(original_data) + + shard_size = len(encoded_data) // 6 + blocks = split_into_blocks(encoded_data, shard_size) + + assert len(blocks) == 6 + + blocks[1] = None + + recovered_data = ec.extract_data_blocks(blocks, len(original_data)) + + assert recovered_data == original_data + + def test_multiple_missing_blocks(self): + ec = ErasureCode(6, 3) + data = b"Test recovery with multiple missing blocks" + + encoded = ec.encode(data) + shard_size = len(encoded) // 9 + blocks = split_into_blocks(encoded, shard_size) + + blocks[0] = None + blocks[4] = None + blocks[7] = None + + recovered = ec.extract_data_blocks(blocks, len(data)) + + assert recovered == data + + def test_stress_test(self): + ec = ErasureCode(8, 4) + + for size in [10, 100, 1000, 5000]: + data = b"z" * size + encoded = ec.encode(data) + decoded = ec.extract_data(encoded, len(data)) + assert decoded == data diff --git a/tests/unit/test_httpext.py b/tests/unit/test_httpext.py index 408d129..6cb6c4a 100644 --- a/tests/unit/test_httpext.py +++ b/tests/unit/test_httpext.py @@ -1,6 +1,663 @@ +""" +Unit tests for the HTTP Extension module (private/httpext/httpext.py). +Tests cover: +- range_download() with various byte ranges +- HTTP headers handling +- Response handling (200, 206, 416) +- Network errors +- Edge cases +""" +import io import pytest +from unittest.mock import Mock, patch, MagicMock +import requests +from private.httpext import ( + HTTPExtClient, + RangeDownloadResult, + HTTPExtError, + RangeNotSatisfiableError, +) +from private.httpext.httpext import NetworkError, InvalidRangeError -def test_placeholder(): - """Placeholder test to keep file valid.""" - pass +class TestHTTPExtClientInit: + """Tests for HTTPExtClient initialization.""" + + def test_init_default_values(self): + """Test client initialization with default values.""" + client = HTTPExtClient() + assert client.timeout == HTTPExtClient.DEFAULT_TIMEOUT + assert client.session is not None + client.close() + + def test_init_custom_values(self): + """Test client initialization with custom values.""" + client = HTTPExtClient(timeout=60, retries=5, backoff_factor=0.5) + assert client.timeout == 60 + client.close() + + def test_context_manager(self): + """Test client as context manager.""" + with HTTPExtClient() as client: + assert client.session is not None + # Session should be closed after context exits + + def test_close(self): + """Test client close method.""" + client = HTTPExtClient() + client.close() + # Should not raise even if called multiple times + client.close() + + +class TestRangeValidation: + """Tests for range parameter validation.""" + + def test_validate_range_valid(self): + """Test validation passes for valid ranges.""" + client = HTTPExtClient() + # Should not raise + client._validate_range(0, 100) + client._validate_range(0, None) + client._validate_range(100, 200) + client._validate_range(0, 0) # Single byte + client.close() + + def test_validate_range_negative_start(self): + """Test validation fails for negative start.""" + client = HTTPExtClient() + with pytest.raises(InvalidRangeError, match="Start position cannot be negative"): + client._validate_range(-1, 100) + client.close() + + def test_validate_range_negative_end(self): + """Test validation fails for negative end.""" + client = HTTPExtClient() + with pytest.raises(InvalidRangeError, match="End position cannot be negative"): + client._validate_range(0, -1) + client.close() + + def test_validate_range_end_less_than_start(self): + """Test validation fails when end < start.""" + client = HTTPExtClient() + with pytest.raises(InvalidRangeError, match="End position cannot be less than start"): + client._validate_range(100, 50) + client.close() + + +class TestBuildRangeHeader: + """Tests for Range header construction.""" + + def test_build_range_header_with_end(self): + """Test Range header with both start and end.""" + client = HTTPExtClient() + header = client._build_range_header(0, 499) + assert header == "bytes=0-499" + client.close() + + def test_build_range_header_without_end(self): + """Test Range header with only start (suffix range).""" + client = HTTPExtClient() + header = client._build_range_header(500, None) + assert header == "bytes=500-" + client.close() + + def test_build_range_header_single_byte(self): + """Test Range header for single byte.""" + client = HTTPExtClient() + header = client._build_range_header(100, 100) + assert header == "bytes=100-100" + client.close() + + def test_build_range_header_large_range(self): + """Test Range header for large range.""" + client = HTTPExtClient() + header = client._build_range_header(0, 1073741823) # ~1GB + assert header == "bytes=0-1073741823" + client.close() + + +class TestParseContentRange: + """Tests for Content-Range header parsing.""" + + def test_parse_content_range_valid(self): + """Test parsing valid Content-Range header.""" + client = HTTPExtClient() + start, end, total = client._parse_content_range("bytes 0-499/1000") + assert start == 0 + assert end == 499 + assert total == 1000 + client.close() + + def test_parse_content_range_unknown_total(self): + """Test parsing Content-Range with unknown total.""" + client = HTTPExtClient() + start, end, total = client._parse_content_range("bytes 0-499/*") + assert start == 0 + assert end == 499 + assert total is None + client.close() + + def test_parse_content_range_none(self): + """Test parsing None Content-Range.""" + client = HTTPExtClient() + start, end, total = client._parse_content_range(None) + assert start is None + assert end is None + assert total is None + client.close() + + def test_parse_content_range_invalid_format(self): + """Test parsing invalid Content-Range format.""" + client = HTTPExtClient() + # Missing "bytes " prefix + start, end, total = client._parse_content_range("0-499/1000") + assert start is None + # Invalid range format + start, end, total = client._parse_content_range("bytes invalid") + assert start is None + client.close() + + +class TestRangeDownload: + """Tests for range_download() method.""" + + @patch.object(requests.Session, 'get') + def test_range_download_206_partial_content(self, mock_get): + """Test successful range download with 206 Partial Content.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"Hello" + mock_response.headers = {"Content-Range": "bytes 0-4/1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, 4) + assert result.data == b"Hello" + assert result.start == 0 + assert result.end == 4 + assert result.total_size == 1000 + assert result.content_length == 5 + assert result.is_partial is True + + # Verify Range header was sent + mock_get.assert_called_once() + call_headers = mock_get.call_args[1]["headers"] + assert call_headers["Range"] == "bytes=0-4" + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_200_full_content(self, mock_get): + """Test range download when server returns full content (200).""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = b"Full file content" + mock_response.headers = {} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, 4) + assert result.data == b"Full file content" + assert result.start == 0 + assert result.end == len(b"Full file content") - 1 + assert result.total_size == len(b"Full file content") + assert result.is_partial is False + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_416_range_not_satisfiable(self, mock_get): + """Test range download with 416 Range Not Satisfiable.""" + mock_response = Mock() + mock_response.status_code = 416 + mock_response.headers = {"Content-Range": "bytes */1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + with pytest.raises(RangeNotSatisfiableError) as exc_info: + client.range_download("http://example.com/file", 2000, 3000) + assert exc_info.value.content_length == 1000 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_416_without_content_length(self, mock_get): + """Test 416 response without content length info.""" + mock_response = Mock() + mock_response.status_code = 416 + mock_response.headers = {} + mock_get.return_value = mock_response + + client = HTTPExtClient() + with pytest.raises(RangeNotSatisfiableError) as exc_info: + client.range_download("http://example.com/file", 2000, 3000) + assert exc_info.value.content_length is None + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_unexpected_status(self, mock_get): + """Test range download with unexpected status code.""" + mock_response = Mock() + mock_response.status_code = 403 + mock_get.return_value = mock_response + + client = HTTPExtClient() + with pytest.raises(HTTPExtError, match="Unexpected status code: 403"): + client.range_download("http://example.com/file", 0, 100) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_with_custom_headers(self, mock_get): + """Test range download with additional custom headers.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"data" + mock_response.headers = {"Content-Range": "bytes 0-3/100"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download( + "http://example.com/file", + 0, 3, + headers={"Authorization": "Bearer token123"} + ) + call_headers = mock_get.call_args[1]["headers"] + assert call_headers["Authorization"] == "Bearer token123" + assert call_headers["Range"] == "bytes=0-3" + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_suffix_range(self, mock_get): + """Test range download with suffix range (no end specified).""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"end of file" + mock_response.headers = {"Content-Range": "bytes 900-999/1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 900, None) + call_headers = mock_get.call_args[1]["headers"] + assert call_headers["Range"] == "bytes=900-" + assert result.start == 900 + assert result.end == 999 + client.close() + + +class TestRangeDownloadNetworkErrors: + """Tests for network error handling in range_download().""" + + @patch.object(requests.Session, 'get') + def test_range_download_timeout(self, mock_get): + """Test range download timeout handling.""" + mock_get.side_effect = requests.exceptions.Timeout("Connection timed out") + client = HTTPExtClient() + with pytest.raises(NetworkError, match="Request timed out"): + client.range_download("http://example.com/file", 0, 100) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_connection_error(self, mock_get): + """Test range download connection error handling.""" + mock_get.side_effect = requests.exceptions.ConnectionError("Connection refused") + client = HTTPExtClient() + with pytest.raises(NetworkError, match="Connection error"): + client.range_download("http://example.com/file", 0, 100) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_request_exception(self, mock_get): + """Test range download generic request exception.""" + mock_get.side_effect = requests.exceptions.RequestException("Unknown error") + client = HTTPExtClient() + with pytest.raises(HTTPExtError, match="Request failed"): + client.range_download("http://example.com/file", 0, 100) + client.close() + + +class TestRangeDownloadEdgeCases: + """Tests for edge cases in range_download().""" + + @patch.object(requests.Session, 'get') + def test_range_download_single_byte(self, mock_get): + """Test downloading a single byte.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"X" + mock_response.headers = {"Content-Range": "bytes 50-50/1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 50, 50) + assert result.data == b"X" + assert result.start == 50 + assert result.end == 50 + assert result.content_length == 1 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_first_byte(self, mock_get): + """Test downloading the first byte only.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"F" + mock_response.headers = {"Content-Range": "bytes 0-0/1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, 0) + assert result.data == b"F" + assert result.start == 0 + assert result.end == 0 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_last_byte(self, mock_get): + """Test downloading the last byte only.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"L" + mock_response.headers = {"Content-Range": "bytes 999-999/1000"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 999, 999) + assert result.data == b"L" + assert result.end == 999 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_empty_response(self, mock_get): + """Test handling empty response content.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"" + mock_response.headers = {"Content-Range": "bytes 0-0/0"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, 0) + assert result.data == b"" + assert result.content_length == 0 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_large_range(self, mock_get): + """Test downloading a large range.""" + large_data = b"X" * 1024 * 1024 # 1MB + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = large_data + mock_response.headers = {"Content-Range": f"bytes 0-{len(large_data)-1}/{len(large_data)}"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, len(large_data) - 1) + assert len(result.data) == len(large_data) + assert result.content_length == len(large_data) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_missing_content_range_header(self, mock_get): + """Test 206 response without Content-Range header.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.content = b"data" + mock_response.headers = {} + mock_get.return_value = mock_response + + client = HTTPExtClient() + result = client.range_download("http://example.com/file", 0, 3) + # Should use fallback values + assert result.data == b"data" + assert result.start == 0 + assert result.end == 3 # start + len(content) - 1 + assert result.total_size is None + client.close() + + def test_range_download_invalid_start(self): + """Test range download with invalid start position.""" + client = HTTPExtClient() + with pytest.raises(InvalidRangeError): + client.range_download("http://example.com/file", -1, 100) + client.close() + + def test_range_download_invalid_end(self): + """Test range download with end < start.""" + client = HTTPExtClient() + with pytest.raises(InvalidRangeError): + client.range_download("http://example.com/file", 100, 50) + client.close() + + +class TestRangeDownloadToFile: + """Tests for range_download_to_file() method.""" + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_success(self, mock_get): + """Test successful streaming download to file.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.headers = {"Content-Range": "bytes 0-99/1000"} + mock_response.iter_content = Mock(return_value=[b"chunk1", b"chunk2", b"chunk3"]) + mock_get.return_value = mock_response + + client = HTTPExtClient() + buffer = io.BytesIO() + result = client.range_download_to_file( + "http://example.com/file", 0, 99, buffer + ) + assert buffer.getvalue() == b"chunk1chunk2chunk3" + assert result.start == 0 + assert result.end == 99 + assert result.total_size == 1000 + assert result.data == b"" # Data written to file, not returned + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_416(self, mock_get): + """Test streaming download with 416 response.""" + mock_response = Mock() + mock_response.status_code = 416 + mock_response.headers = {"Content-Range": "bytes */500"} + mock_get.return_value = mock_response + + client = HTTPExtClient() + buffer = io.BytesIO() + with pytest.raises(RangeNotSatisfiableError) as exc_info: + client.range_download_to_file( + "http://example.com/file", 1000, 2000, buffer + ) + assert exc_info.value.content_length == 500 + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_timeout(self, mock_get): + """Test streaming download timeout.""" + mock_get.side_effect = requests.exceptions.Timeout() + client = HTTPExtClient() + buffer = io.BytesIO() + with pytest.raises(NetworkError, match="Request timed out"): + client.range_download_to_file( + "http://example.com/file", 0, 100, buffer + ) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_unexpected_status(self, mock_get): + """Test streaming download with unexpected status code.""" + mock_response = Mock() + mock_response.status_code = 403 + mock_response.headers = {} + mock_get.return_value = mock_response + + client = HTTPExtClient() + buffer = io.BytesIO() + with pytest.raises(HTTPExtError, match="Unexpected status code: 403"): + client.range_download_to_file( + "http://example.com/file", 0, 100, buffer + ) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_connection_error(self, mock_get): + """Test streaming download connection error.""" + mock_get.side_effect = requests.exceptions.ConnectionError("Connection refused") + client = HTTPExtClient() + buffer = io.BytesIO() + with pytest.raises(NetworkError, match="Connection error"): + client.range_download_to_file( + "http://example.com/file", 0, 100, buffer + ) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_request_exception(self, mock_get): + """Test streaming download generic request exception.""" + mock_get.side_effect = requests.exceptions.RequestException("Unknown error") + client = HTTPExtClient() + buffer = io.BytesIO() + with pytest.raises(HTTPExtError, match="Request failed"): + client.range_download_to_file( + "http://example.com/file", 0, 100, buffer + ) + client.close() + + @patch.object(requests.Session, 'get') + def test_range_download_to_file_with_custom_headers(self, mock_get): + """Test streaming download with custom headers.""" + mock_response = Mock() + mock_response.status_code = 206 + mock_response.headers = {"Content-Range": "bytes 0-99/1000"} + mock_response.iter_content = Mock(return_value=[b"data"]) + mock_get.return_value = mock_response + + client = HTTPExtClient() + buffer = io.BytesIO() + client.range_download_to_file( + "http://example.com/file", 0, 99, buffer, + headers={"Authorization": "Bearer token"} + ) + call_headers = mock_get.call_args[1]["headers"] + assert call_headers["Authorization"] == "Bearer token" + assert call_headers["Range"] == "bytes=0-99" + client.close() + + +class TestGetContentLength: + """Tests for get_content_length() method.""" + + @patch.object(requests.Session, 'head') + def test_get_content_length_success(self, mock_head): + """Test successful content length retrieval.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Length": "12345"} + mock_response.raise_for_status = Mock() + mock_head.return_value = mock_response + + client = HTTPExtClient() + length = client.get_content_length("http://example.com/file") + assert length == 12345 + client.close() + + @patch.object(requests.Session, 'head') + def test_get_content_length_not_available(self, mock_head): + """Test when Content-Length header is not available.""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.raise_for_status = Mock() + mock_head.return_value = mock_response + + client = HTTPExtClient() + length = client.get_content_length("http://example.com/file") + assert length is None + client.close() + + @patch.object(requests.Session, 'head') + def test_get_content_length_network_error(self, mock_head): + """Test content length with network error.""" + mock_head.side_effect = requests.exceptions.ConnectionError() + client = HTTPExtClient() + with pytest.raises(NetworkError, match="Connection error"): + client.get_content_length("http://example.com/file") + client.close() + + @patch.object(requests.Session, 'head') + def test_get_content_length_timeout(self, mock_head): + """Test content length with timeout error.""" + mock_head.side_effect = requests.exceptions.Timeout() + client = HTTPExtClient() + with pytest.raises(NetworkError, match="Request timed out"): + client.get_content_length("http://example.com/file") + client.close() + + @patch.object(requests.Session, 'head') + def test_get_content_length_request_exception(self, mock_head): + """Test content length with generic request exception.""" + mock_head.side_effect = requests.exceptions.RequestException("Unknown") + client = HTTPExtClient() + with pytest.raises(HTTPExtError, match="Request failed"): + client.get_content_length("http://example.com/file") + client.close() + + +class TestRangeDownloadResult: + """Tests for RangeDownloadResult dataclass.""" + + def test_is_partial_true(self): + """Test is_partial returns True for partial content.""" + result = RangeDownloadResult( + data=b"data", + start=0, + end=99, + total_size=1000, + content_length=100, + ) + assert result.is_partial is True + + def test_is_partial_false_full_content(self): + """Test is_partial returns False for full content.""" + result = RangeDownloadResult( + data=b"full", + start=0, + end=3, + total_size=4, + content_length=4, + ) + assert result.is_partial is False + + def test_is_partial_false_unknown_total(self): + """Test is_partial returns False when total_size is unknown.""" + result = RangeDownloadResult( + data=b"data", + start=0, + end=99, + total_size=None, + content_length=100, + ) + assert result.is_partial is False + + +class TestHTTPExtErrorHierarchy: + """Tests for exception hierarchy.""" + + def test_range_not_satisfiable_is_httpext_error(self): + """Test RangeNotSatisfiableError inherits from HTTPExtError.""" + error = RangeNotSatisfiableError("test") + assert isinstance(error, HTTPExtError) + + def test_network_error_is_httpext_error(self): + """Test NetworkError inherits from HTTPExtError.""" + error = NetworkError("test") + assert isinstance(error, HTTPExtError) + + def test_invalid_range_error_is_httpext_error(self): + """Test InvalidRangeError inherits from HTTPExtError.""" + error = InvalidRangeError("test") + assert isinstance(error, HTTPExtError) + + def test_range_not_satisfiable_with_content_length(self): + """Test RangeNotSatisfiableError stores content_length.""" + error = RangeNotSatisfiableError("test", content_length=1000) + assert error.content_length == 1000 + assert str(error) == "test" diff --git a/tests/unit/test_sdk_ipc.py b/tests/unit/test_sdk_ipc.py index e46bd75..9ca1618 100644 --- a/tests/unit/test_sdk_ipc.py +++ b/tests/unit/test_sdk_ipc.py @@ -10,6 +10,148 @@ class TestCreateBucket: """Test create bucket functionality.""" + +import io +from datetime import datetime +from unittest.mock import MagicMock, Mock, call, patch + +import pytest + +from sdk.config import SDKConfig, SDKError +from sdk.model import ( + Chunk, + FileBlockUpload, + IPCBucket, + IPCBucketCreateResult, + IPCFileDownload, + IPCFileMeta, + IPCFileUpload, +) +from sdk.sdk_ipc import IPC, TxWaitSignal, encryption_key, maybe_encrypt_metadata, to_ipc_proto_chunk + + +class TestTxWaitSignal: + + def test_init(self): + chunk = Mock() + tx = "0x123456" + signal = TxWaitSignal(chunk, tx) + + assert signal.FileUploadChunk == chunk + assert signal.Transaction == tx + + +class TestEncryptionKey: + + def test_encryption_key_empty_parent(self): + result = encryption_key(b"", "bucket", "file") + assert result == b"" + + @patch("sdk.sdk_ipc.derive_key") + def test_encryption_key_with_data(self, mock_derive): + parent = b"parent_key_32bytes_test123456789" + mock_derive.return_value = b"derived" + + result = encryption_key(parent, "bucket", "file") + + assert result == b"derived" + mock_derive.assert_called_once_with(parent, b"bucket/file") + + @patch("sdk.sdk_ipc.derive_key") + def test_encryption_key_multiple_info(self, mock_derive): + parent = b"key" + mock_derive.return_value = b"result" + + result = encryption_key(parent, "a", "b", "c") + + mock_derive.assert_called_once_with(parent, b"a/b/c") + + +class TestMaybeEncryptMetadata: + + def test_maybe_encrypt_metadata_no_key(self): + result = maybe_encrypt_metadata("plain_value", "path", b"") + assert result == "plain_value" + + @patch("sdk.sdk_ipc.derive_key") + @patch("sdk.sdk_ipc.encrypt") + def test_maybe_encrypt_metadata_with_key(self, mock_encrypt, mock_derive): + key = b"encryption_key_32bytes_test12345" + mock_derive.return_value = b"file_key" + mock_encrypt.return_value = b"\x01\x02\x03" + + result = maybe_encrypt_metadata("value", "path/to/file", key) + + assert result == "010203" + mock_derive.assert_called_once_with(key, b"path/to/file") + mock_encrypt.assert_called_once_with(b"file_key", b"value", b"metadata") + + @patch("sdk.sdk_ipc.derive_key") + @patch("sdk.sdk_ipc.encrypt") + def test_maybe_encrypt_metadata_error(self, mock_encrypt, mock_derive): + key = b"encryption_key_32bytes_test12345" + mock_derive.side_effect = Exception("Derive failed") + + with pytest.raises(SDKError, match="failed to encrypt metadata"): + maybe_encrypt_metadata("value", "path", key) + + +class TestToIPCProtoChunk: + + @patch("sdk.sdk_ipc.CID") + @patch("sdk.sdk_ipc.ipcnodeapi_pb2") + def test_to_ipc_proto_chunk_basic(self, mock_pb2, mock_cid): + mock_cid.decode.return_value = b"bytes" + mock_block_class = Mock() + mock_pb2.IPCChunk.Block = mock_block_class + mock_pb2.IPCChunk = Mock() + + blocks = [ + FileBlockUpload(cid="cid1", data=b"data1"), + FileBlockUpload(cid="cid2", data=b"data2"), + ] + + cids, sizes, proto_chunk, err = to_ipc_proto_chunk("chunk_cid", 0, 100, blocks) + + assert err is None + assert isinstance(cids, list) + assert isinstance(sizes, list) + assert len(sizes) == 2 + + def test_to_ipc_proto_chunk_empty_blocks(self): + cids, sizes, proto_chunk, err = to_ipc_proto_chunk("cid", 0, 100, []) + + assert err is None + assert cids == [] + assert sizes == [] + + +class TestIPCInit: + + def test_ipc_init(self): + mock_client = Mock() + mock_conn = Mock() + mock_ipc_instance = Mock() + config = SDKConfig( + address="test:5500", + max_concurrency=5, + block_part_size=128 * 1024, + use_connection_pool=True, + streaming_max_blocks_in_chunk=10, + ) + + ipc = IPC(mock_client, mock_conn, mock_ipc_instance, config) + + assert ipc.client == mock_client + assert ipc.conn == mock_conn + assert ipc.ipc == mock_ipc_instance + assert ipc.max_concurrency == 5 + assert ipc.block_part_size == 128 * 1024 + assert ipc.max_blocks_in_chunk == 10 + + +class TestCreateBucket: + def setup_method(self): self.mock_client = Mock() self.mock_conn = Mock() @@ -26,8 +168,14 @@ def setup_method(self): ) self.ipc = IPC(self.mock_client, self.mock_conn, self.mock_ipc, self.config) + self.config = SDKConfig(address="test:5500") + self.ipc = IPC(self.mock_client, self.mock_conn, self.mock_ipc, self.config) + + def test_create_bucket_invalid_name(self): + with pytest.raises(SDKError, match="invalid bucket name"): + self.ipc.create_bucket(None, "ab") + def test_create_bucket_success(self): - """Test successful bucket creation.""" mock_receipt = Mock() mock_receipt.status = 1 mock_receipt.blockNumber = 100