Skip to content

Add LLM prompt view to Validation & Error overlays #948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cmer wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add LLM prompt view to Validation & Error overlays #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2799af9
Add LLM prompt view to Validation & Error overlays
cmer Dec 4, 2025
ce3c6bc
Fix tests
cmer Dec 4, 2025
0ef2975
Tweak LLM prompt
cmer Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
303 changes: 288 additions & 15 deletions javascript/packages/dev-tools/src/error-overlay.ts
View file
Open in desktop

Large diffs are not rendered by default.

29 changes: 27 additions & 2 deletions lib/herb/engine.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def initialize(input, properties = {})
@content_for_head = properties[:content_for_head]
@validation_error_template = nil
@validation_mode = properties.fetch(:validation_mode, :raise)
@default_view = properties.fetch(:default_view, :human).to_sym
@visitors = properties.fetch(:visitors, default_visitors)

if @debug && @visitors.empty?
Expand All @@ -67,6 +68,11 @@ def initialize(input, properties = {})
"validation_mode must be one of :raise, :overlay, or :none, got #{@validation_mode.inspect}"
end

unless [:human, :llm].include?(@default_view)
raise ArgumentError,
"default_view must be one of :human or :llm, got #{@default_view.inspect}"
end

@freeze = properties[:freeze]
@freeze_template_literals = properties.fetch(:freeze_template_literals, true)
@text_end = @freeze_template_literals ? "'.freeze" : "'"
Expand Down Expand Up @@ -321,6 +327,9 @@ def handle_validation_errors(errors, input)
def add_validation_overlay(errors, input = nil)
return unless errors.any?

# Group errors by file for LLM prompt generation
errors_by_file = {} #: Hash[String, Array[Hash[Symbol, untyped]]]

templates = errors.map { |error|
location = error[:location]
line = location&.start&.line || 0
Expand All @@ -333,6 +342,10 @@ def add_validation_overlay(errors, input = nil)
escaped_message = escape_attr(error[:message])
escaped_suggestion = error[:suggestion] ? escape_attr(error[:suggestion]) : ""

# Track errors by file for LLM prompt
errors_by_file[@relative_file_path] ||= []
errors_by_file[@relative_file_path] << error.merge(filename: @relative_file_path)

<<~TEMPLATE
<template
data-herb-validation-error
Expand All @@ -349,7 +362,18 @@ def add_validation_overlay(errors, input = nil)
TEMPLATE
}.join

@validation_error_template = templates
# Generate combined LLM prompt
error_count = errors.count { |e| e[:severity].to_s == "error" }
warning_count = errors.count { |e| e[:severity].to_s == "warning" }
counts = { errors: error_count, warnings: warning_count, total: errors.length }

llm_prompt = ValidationErrorOverlay.generate_llm_prompt(errors.map { |e|
e.merge(filename: @relative_file_path)
}, errors_by_file, counts)
llm_template = "<template data-herb-validation-llm-prompt data-default-view=\"#{@default_view}\">" \
"#{escape_attr(llm_prompt)}</template>"

@validation_error_template = templates + llm_template
end

def escape_attr(text)
Expand All @@ -370,7 +394,8 @@ def add_parser_error_overlay(parser_errors, input)
overlay_generator = ParserErrorOverlay.new(
input,
parser_errors,
filename: @relative_file_path
filename: @relative_file_path,
default_view: @default_view
)

error_html = overlay_generator.generate_html
Expand Down
222 changes: 219 additions & 3 deletions lib/herb/engine/parser_error_overlay.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@ class ParserErrorOverlay
Herb::Errors::MissingOpeningTagError
].freeze

def initialize(source, errors, filename: nil)
def initialize(source, errors, filename: nil, default_view: :human)
@source = source
@errors = errors.sort_by { |error|
[ERROR_CLASS_PRIORITRY.index(error.class) || -1, error.location.start.line, error.location.start.column]
}
@filename = filename || "unknown"
@lines = source.lines
@default_view = default_view.to_sym
end

def generate_html
Expand Down Expand Up @@ -308,6 +309,95 @@ def generate_html
border-radius: 3px;
}

/* View toggle button */
.herb-parser-error-overlay .herb-view-toggle {
color: rgba(255, 255, 255, 0.9);
font-family: inherit;
font-size: 13px;
font-weight: 500;
padding: 6px 12px;
background: rgba(255, 255, 255, 0.1);
border: none;
border-radius: 6px;
transition: background-color 0.2s;
cursor: pointer;
white-space: nowrap;
flex-shrink: 0;
}

.herb-parser-error-overlay .herb-view-toggle:hover {
background: rgba(255, 255, 255, 0.2);
}

/* LLM view styles */
.herb-parser-error-overlay .herb-llm-view {
display: flex;
flex-direction: column;
gap: 16px;
height: 100%;
}

/* LLM textarea container */
.herb-parser-error-overlay .herb-llm-textarea-container {
position: relative;
flex: 1;
display: flex;
flex-direction: column;
}

.herb-parser-error-overlay .herb-copy-button {
position: absolute;
top: 8px;
right: 25px;
color: #9ca3af;
background: rgba(0, 0, 0, 0.5);
border: 1px solid #374151;
border-radius: 6px;
padding: 6px;
cursor: pointer;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
z-index: 1;
}

.herb-parser-error-overlay .herb-copy-button:hover {
color: #e5e5e5;
background: rgba(0, 0, 0, 0.7);
border-color: #6b7280;
}

.herb-parser-error-overlay .herb-copy-button.copied {
color: #10b981;
border-color: #10b981;
}

.herb-parser-error-overlay .herb-copy-button svg {
width: 18px;
height: 18px;
}

.herb-parser-error-overlay .herb-llm-textarea {
flex: 1;
width: 100%;
min-height: 300px;
background: #111111;
border: 1px solid #374151;
border-radius: 8px;
color: #e5e5e5;
font-family: inherit;
font-size: 13px;
line-height: 1.6;
padding: 16px;
resize: none;
}

.herb-parser-error-overlay .herb-llm-textarea:focus {
outline: none;
border-color: #6366f1;
}

@media (max-width: 768px) {
.herb-parser-error-overlay {
padding: 10px;
Expand Down Expand Up @@ -342,11 +432,23 @@ def generate_html
#{escape_html(error_message)}
</div>
</div>
<button type="button" class="herb-view-toggle" id="herb-view-toggle">#{@default_view == :llm ? "Human view" : "LLM prompt"}</button>
</div>

<div class="herb-error-content">
<div class="herb-error-content" id="herb-human-view"#{' style="display: none;"' if @default_view == :llm}>
#{generate_error_sections}
</div>

<div class="herb-error-content herb-llm-view" id="herb-llm-view"#{if @default_view == :human
' style="display: none;"'
end}>
<div class="herb-llm-textarea-container">
<button type="button" class="herb-copy-button" id="herb-copy-button" title="Copy to clipboard">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path></svg>
</button>
<textarea class="herb-llm-textarea" id="herb-llm-prompt" readonly>#{escape_html(generate_llm_prompt)}</textarea>
</div>
</div>
</div>

<script>
Expand Down Expand Up @@ -409,7 +511,6 @@ def generate_html
document.body.style.overflow = 'hidden';

// Restore scroll when closed (cleanup for navigation)
const overlay = document.querySelector('.herb-parser-error-overlay');
if (overlay) {
const observer = new MutationObserver(function(mutations) {
mutations.forEach(function(mutation) {
Expand All @@ -422,6 +523,52 @@ def generate_html
});
observer.observe(overlay, { attributes: true });
}

// View toggle functionality
const viewToggle = document.getElementById('herb-view-toggle');
const humanView = document.getElementById('herb-human-view');
const llmView = document.getElementById('herb-llm-view');

if (viewToggle && humanView && llmView) {
viewToggle.addEventListener('click', function() {
const isShowingHuman = humanView.style.display !== 'none';

if (isShowingHuman) {
humanView.style.display = 'none';
llmView.style.display = 'flex';
viewToggle.textContent = 'Human view';
} else {
humanView.style.display = 'flex';
llmView.style.display = 'none';
viewToggle.textContent = 'LLM prompt';
}
});
}

const copyButton = document.getElementById('herb-copy-button');
const llmPrompt = document.getElementById('herb-llm-prompt');

if (copyButton && llmPrompt) {
copyButton.addEventListener('click', function() {
navigator.clipboard.writeText(llmPrompt.value).then(function() {
copyButton.textContent = 'Copied!';
copyButton.classList.add('copied');
setTimeout(function() {
copyButton.textContent = 'Copy to clipboard';
copyButton.classList.remove('copied');
}, 2000);
}).catch(function(err) {
llmPrompt.select();
document.execCommand('copy');
copyButton.textContent = 'Copied!';
copyButton.classList.add('copied');
setTimeout(function() {
copyButton.textContent = 'Copy to clipboard';
copyButton.classList.remove('copied');
}, 2000);
});
});
}
})();
</script>
</div>
Expand All @@ -444,6 +591,75 @@ def generate_error_sections
sections.uniq.join("\n")
end

def generate_llm_prompt
error_count = @errors.length
error_word = error_count == 1 ? "error" : "errors"

prompt = <<~MARKDOWN
# ERB Template Errors

Herb, an HTML-aware ERB parsing tool, detected #{error_count} #{error_word} in an ERB template that need to be fixed.

## File
`#{@filename}`

## Errors

MARKDOWN

@errors.each_with_index do |error, index|
location = error.respond_to?(:location) && error.location ? error.location : nil
line_num = 1
col_num = 1

if location.respond_to?(:start) && location.is_a?(Herb::Location) && location.start
line_num = location.start.line
col_num = location.start.column
end

error_class = error.class.name.split("::").last.gsub(/Error$/, "")
error_message = error.respond_to?(:message) ? error.message : error.to_s
suggestion = get_error_suggestion(error)

prompt += "### Error #{index + 1}: #{error_class}\n\n"
prompt += "**Location:** Line #{line_num}, Column #{col_num}\n\n"
prompt += "**Message:** #{error_message}\n\n"
prompt += "**Suggestion:** #{suggestion}\n\n" if suggestion
prompt += "**Code context:**\n\n```erb\n"
prompt += generate_code_snippet_for_llm(line_num)
prompt += "```\n\n"
end

prompt += <<~MARKDOWN
## Instructions

Fix all the errors listed above.

When fixing these errors, ensure that:
1. All HTML tags are properly opened and closed
2. Tags that span ERB control flow blocks are handled correctly
3. The resulting HTML structure is valid
4. Code duplication resulting from the fixes is kept to a minimum. Making use of methods (either View Component methods, helper methods, inline methods, etc.), HTML builder helper methods (ie: content_tag, etc.), procs, etc, if it means keeping the resulting code clean, readable and maintainable.
MARKDOWN

prompt
end

def generate_code_snippet_for_llm(error_line_num)
start_line = [error_line_num - CONTEXT_LINES, 1].max
end_line = [error_line_num + CONTEXT_LINES, @lines.length].min

snippet = ""
(start_line..end_line).each do |i|
line = @lines[i - 1] || ""
line_str = line.chomp
marker = i == error_line_num ? " <-- ERROR" : ""
snippet += "#{i.to_s.rjust(4)}: #{line_str}#{marker}\n"
end

snippet
end

def generate_code_section(error, index)
location = error.respond_to?(:location) && error.location ? error.location : nil
line_num = 1
Expand Down
Loading