Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .envrc

This file was deleted.

5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ __pycache__/
# Distribution / packaging
.Python
build/
!src/tetra_rp/build/
develop-eggs/
dist/
downloads/
Expand Down Expand Up @@ -175,3 +176,7 @@ cython_debug/
.pypirc
.tetra_resources.pkl
.DS_Store

# dev only
.envrc
test_app/
2 changes: 1 addition & 1 deletion .python-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.9
3.11
6 changes: 5 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,9 @@
"tests"
],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
"python.testing.pytestEnabled": true,
"python.envFile": "${workspaceFolder}/.env",
"python.analysis.extraPaths": [
"${workspaceFolder}/src",
]
}
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ help: # Show this help menu
@awk 'BEGIN {FS = ":.*# "; printf "%-20s %s\n", "Target", "Description"} /^[a-zA-Z_-]+:.*# / {printf "%-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@echo ""

dev: # Install development dependencies
dev: # Install development dependencies and package in editable mode
uv sync --all-groups
uv pip install -e .

update:
uv sync --upgrade --all-groups
Expand Down
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ flash = "tetra_rp.cli.main:app"
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"

[tool.setuptools.packages.find]
where = ["src"]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
Expand Down
15 changes: 15 additions & 0 deletions src/tetra_rp/build/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"""
Tetra Build System.
Flash CLI Build Components:
- TarballStrategy: Tarball-based deployment for GPU workers
- CodeExtractor: Extract and clean source code
- VolumeManager: S3 volume management
Usage:
from tetra_rp.build.gpu_workers.strategies import TarballStrategyConfig
from tetra_rp.build.gpu_workers.strategies.tarball_strategy import TarballStrategy
from tetra_rp.build.shared.code_extractor import CodeExtractor
"""

__all__ = []
132 changes: 132 additions & 0 deletions src/tetra_rp/build/code_extractor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
"""
Code extraction and processing for production builds.

Handles extracting source code, removing decorators, and generating code hashes.
"""

import hashlib
import inspect
import logging
from dataclasses import dataclass
from typing import Any

log = logging.getLogger(__name__)


@dataclass
class ExtractedCode:
"""Container for extracted code information."""

source_code: str
cleaned_code: str
callable_name: str
callable_type: str # "function" or "class"
code_hash: str


class CodeExtractor:
"""Extract and process source code from callable objects."""

def extract(self, func_or_class: Any) -> ExtractedCode:
"""
Extract source code from a function or class.

Args:
func_or_class: The function or class to extract code from

Returns:
ExtractedCode object containing source and metadata

Raises:
ValueError: If source code cannot be extracted
"""
callable_name = func_or_class.__name__

try:
source_code = inspect.getsource(func_or_class)
except (OSError, TypeError) as e:
raise ValueError(
f"Failed to extract source code for {callable_name}: {e}"
) from e

# Determine callable type
callable_type = "class" if inspect.isclass(func_or_class) else "function"

# Clean the code (remove decorators)
cleaned_code = self._remove_remote_decorator(source_code)

# Generate hash for versioning
code_hash = self._generate_code_hash(source_code)

log.debug(
f"Extracted {callable_type} '{callable_name}' "
f"({len(source_code)} bytes, hash: {code_hash})"
)

return ExtractedCode(
source_code=source_code,
cleaned_code=cleaned_code,
callable_name=callable_name,
callable_type=callable_type,
code_hash=code_hash,
)

def _remove_remote_decorator(self, source_code: str) -> str:
"""
Remove @remote decorator from source code.

Uses line-based approach to handle both single-line and multi-line decorators.

Args:
source_code: Original source code with decorators

Returns:
Cleaned source code without @remote decorator
"""
lines = source_code.splitlines()
cleaned_lines = []
in_decorator = False

for line in lines:
stripped = line.strip()

# Start of @remote decorator
if stripped.startswith("@remote"):
in_decorator = True
# Check if it's a multi-line decorator
if "(" in stripped and ")" not in stripped:
continue # Multi-line, skip this line
elif "(" in stripped and ")" in stripped:
# Single-line decorator, skip it
continue
else:
# Just @remote without parens
continue

# Continuation of multi-line decorator
if in_decorator:
if ")" in stripped:
in_decorator = False
continue

cleaned_lines.append(line)

cleaned = "\n".join(cleaned_lines)

# Add header if needed
if "import" not in cleaned[:100]:
cleaned = "# Auto-generated baked code\n" + cleaned

return cleaned

def _generate_code_hash(self, source_code: str) -> str:
"""
Generate a short hash from source code for versioning.

Args:
source_code: Source code to hash

Returns:
8-character hex hash
"""
return hashlib.sha256(source_code.encode()).hexdigest()[:8]
37 changes: 37 additions & 0 deletions src/tetra_rp/build/gpu_workers/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""
GPU Worker deployment system - Tarball Strategy.

Flash CLI uses tarball-based deployment for GPU workers.
"""

from .strategies import (
# Base types
DeploymentStrategy,
DeploymentArtifact,
DeploymentConfig,
StrategyType,
# Tarball Strategy
TarballStrategy,
TarballStrategyConfig,
)

# Shared components
from .code_packager import CodePackager, PackageInfo
from .volume_manager import UploadResult, VolumeConfig, VolumeManager

__all__ = [
# Base types
"DeploymentStrategy",
"DeploymentArtifact",
"DeploymentConfig",
"StrategyType",
# Tarball Strategy
"TarballStrategy",
"TarballStrategyConfig",
# Shared Components
"CodePackager",
"PackageInfo",
"VolumeManager",
"VolumeConfig",
"UploadResult",
]
Loading
Loading