Skip to content

perf: encode file contents once in tree-sitter scoring #984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aliangm wants to merge 3 commits into from
Closed

perf: encode file contents once in tree-sitter scoring #984

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb1e04f
perf: encode file contents once in tree-sitter scoring
aliangm May 4, 2026
3144392
Merge branch 'test' into fix-tree-sitter-double-encode
aliangm May 5, 2026
82216d6
style: apply ruff format
aliangm May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 52 additions & 43 deletions gittensor/validator/utils/tree_sitter_scoring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,13 @@ def get_parser(language: str) -> Optional[Parser]:
return None


def parse_code(content: str, language: str) -> Optional[Tree]:
def parse_code(content: Union[str, bytes], language: str) -> Optional[Tree]:
"""
Parse source code into a tree-sitter AST.

Args:
content: Source code as string
content: Source code as ``str`` or pre-encoded UTF-8 ``bytes``. Bytes are
preferred on hot paths to avoid re-encoding the same string twice.
language: Tree-sitter language name

Returns:
Expand All @@ -73,8 +74,11 @@ def parse_code(content: str, language: str) -> Optional[Tree]:
if not parser:
return None

if isinstance(content, str):
content = content.encode('utf-8')

try:
return parser.parse(content.encode('utf-8'))
return parser.parse(content)
except Exception as e:
bt.logging.debug(f'Failed to parse code: {e}')
return None
Expand Down Expand Up @@ -157,8 +161,8 @@ def has_inline_tests(content: str, extension: str) -> bool:


def score_tree_diff(
old_content: Optional[str],
new_content: Optional[str],
old_content: Union[str, bytes, None],
new_content: Union[str, bytes, None],
extension: str,
weights: TokenConfig,
) -> ScoreBreakdown:
Expand All @@ -169,8 +173,10 @@ def score_tree_diff(
- Both additions (in new but not old) and deletions (in old but not new) are scored

Args:
old_content: Content of the file before changes (None for new files)
new_content: Content of the file after changes (None for deleted files)
old_content: Content of the file before changes as ``str`` or pre-encoded
UTF-8 ``bytes`` (None for new files). Bytes preferred on hot paths.
new_content: Content of the file after changes as ``str`` or pre-encoded
UTF-8 ``bytes`` (None for deleted files).
extension: File extension for language detection
weights: TokenConfig instance with scoring configuration

Expand Down Expand Up @@ -309,16 +315,6 @@ def calculate_token_score_from_file_changes(
is_test_file=is_test_file,
scoring_method='skipped-binary',
)
elif len(content_pair.new_content.encode('utf-8')) > MAX_FILE_SIZE_BYTES:
bt.logging.debug(f' │ {file.short_name}: skipped (file too large, >{MAX_FILE_SIZE_BYTES} bytes)')
file_result = FileScoreResult(
filename=file.short_name,
score=0.0,
nodes_scored=0,
total_lines=file.changes,
is_test_file=is_test_file,
scoring_method='skipped-large',
)
elif not weights.supports_tree_sitter(ext):
bt.logging.debug(f' │ {file.short_name}: skipped (extension .{ext} not supported)')
file_result = FileScoreResult(
Expand All @@ -331,33 +327,46 @@ def calculate_token_score_from_file_changes(
)
else:
# Tree diff scoring - compare old and new ASTs
old_content = content_pair.old_content
new_content = content_pair.new_content
file_breakdown = score_tree_diff(old_content, new_content, ext, weights)

lang_config = programming_languages.get(ext)
lang_weight = lang_config.weight if lang_config else 1.0

# For non-test files in inline-test languages, check if the current
# file contains inline tests and downweight the entire file if so
if not is_test_file and ext in INLINE_TEST_EXTENSIONS:
if has_inline_tests(new_content, ext):
is_test_file = True
file_weight = TEST_FILE_CONTRIBUTION_WEIGHT

combined_weight = lang_weight * file_weight
file_breakdown = file_breakdown.with_weight(combined_weight)
nodes_scored = file_breakdown.added_count + file_breakdown.deleted_count

file_result = FileScoreResult(
filename=file.short_name,
score=file_breakdown.total_score,
nodes_scored=nodes_scored,
total_lines=file.changes,
is_test_file=is_test_file,
scoring_method='tree-diff',
breakdown=file_breakdown,
)
new_bytes = new_content.encode('utf-8')
old_bytes = content_pair.old_content.encode('utf-8') if content_pair.old_content else None

if len(new_bytes) > MAX_FILE_SIZE_BYTES:
bt.logging.debug(f' │ {file.short_name}: skipped (file too large, >{MAX_FILE_SIZE_BYTES} bytes)')
file_result = FileScoreResult(
filename=file.short_name,
score=0.0,
nodes_scored=0,
total_lines=file.changes,
is_test_file=is_test_file,
scoring_method='skipped-large',
)
else:
file_breakdown = score_tree_diff(old_bytes, new_bytes, ext, weights)

lang_config = programming_languages.get(ext)
lang_weight = lang_config.weight if lang_config else 1.0

# For non-test files in inline-test languages, check if the current
# file contains inline tests and downweight the entire file if so
if not is_test_file and ext in INLINE_TEST_EXTENSIONS:
if has_inline_tests(new_content, ext):
is_test_file = True
file_weight = TEST_FILE_CONTRIBUTION_WEIGHT

combined_weight = lang_weight * file_weight
file_breakdown = file_breakdown.with_weight(combined_weight)
nodes_scored = file_breakdown.added_count + file_breakdown.deleted_count

file_result = FileScoreResult(
filename=file.short_name,
score=file_breakdown.total_score,
nodes_scored=nodes_scored,
total_lines=file.changes,
is_test_file=is_test_file,
scoring_method='tree-diff',
breakdown=file_breakdown,
)

# Accumulate into results and per-category totals
file_results.append(file_result)
Expand Down
Loading