Skip to content

Commit 50d5bde

Browse files
authored
fix(ir): preserve comprehension filter bindings across generators
Later generator sources, map keys, binary sizes and templates can read filter-bound values. Short-circuit filters carry only definitely bound values, preserving outer iteration and shadowing. Scope-sensitive diagnostics recognize bindings from earlier filters.
1 parent fddb2dd commit 50d5bde

14 files changed

Lines changed: 328 additions & 23 deletions

‎CHANGELOG.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,20 @@ All notable milestones for the nera project are documented in this file.
4444
mixed maps, arithmetic, nested generators, compound guards, record defaults,
4545
evaluation order, captures and ordinary exceptions.
4646

47+
## Comprehension filters retain bindings across generators
48+
49+
- Names bound in a list or binary comprehension filter remain available to
50+
later generator sources, map keys, binary sizes, filters and templates.
51+
- Generated helpers pass the bound values forward without repeating filters;
52+
later generator shadowing still preserves the values needed to resume outer
53+
iterations.
54+
- Only definitely bound filter values cross helper boundaries; skipped
55+
short-circuit operands do not supply values. Diagnostics recognize earlier
56+
filter bindings in conditional rematches and shadowing.
57+
- `comprehension_filter_bindings` runs unchanged natively and on BEAM, including
58+
mixed generators, closures, deferred bitstring tails, short-circuit filters,
59+
conditional rematches and evaluation traces.
60+
4761
## Unsupported float segment widths compile as on OTP 29
4862

4963
- Literal float segment widths outside the supported encodings no longer reject

‎crates/nera-ir/src/lower/list_comp.rs‎

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -64,27 +64,25 @@ impl IrBuilder {
6464

6565
// ── 3. Analyse free variables ──
6666
let captured_vars = self.collect_lc_captures(&groups, body_expr);
67-
let num_captured = captured_vars.len() as u32;
6867

6968
// ── 4. Pre-generate helper function names for all generators ──
70-
let mut helpers: Vec<(AtomId, String, u32)> = Vec::new(); // (name_id, mangled, arity)
69+
let mut helpers: Vec<(AtomId, String)> = Vec::new(); // (name_id, mangled)
7170
let prefix = if finalize_as_binary { "bc" } else { "lc" };
72-
for k in 0..n {
71+
for _ in 0..n {
7372
let id = self.lc_counter;
7473
self.lc_counter += 1;
7574
let name_str = format!("__{prefix}_{id}");
7675
let mangled = format!("{}__{prefix}_{id}", self.module_name.0);
7776
let name_id = self.fresh_atom_id();
7877
self.atom_names.insert(name_id, name_str);
79-
// params: iter + acc + k tails + k heads + captured
80-
let arity = 2 + 2 * k as u32 + num_captured;
81-
self.module_functions
82-
.insert((name_id, arity), mangled.clone());
83-
helpers.push((name_id, mangled, arity));
78+
helpers.push((name_id, mangled));
8479
}
8580

8681
// ── 5. Build all helper functions ──
8782
let saved = self.save_state();
83+
// Each completed helper records the names its filters introduced.
84+
// Later helpers carry their values, not expressions to evaluate again.
85+
let mut filter_bindings = Vec::new();
8886

8987
for k in 0..n {
9088
if k > 0 {
@@ -97,7 +95,7 @@ impl IrBuilder {
9795
self.next_block = 0;
9896
self.scopes = vec![HashMap::new()];
9997
}
100-
self.build_lc_helper(k, n, &groups, &helpers, &captured_vars, body_expr, finalize_as_binary, &bin_flags)?;
98+
self.build_lc_helper(k, n, &groups, &helpers, &captured_vars, body_expr, finalize_as_binary, &bin_flags, &mut filter_bindings)?;
10199
}
102100

103101
self.restore_state(saved);
@@ -234,7 +232,8 @@ impl IrBuilder {
234232
}
235233

236234
/// Capture expression reads before the generator that shadows their name.
237-
/// Earlier generator values travel through head parameters, not captures.
235+
/// Earlier generator and filter values travel through helper parameters,
236+
/// not captures.
238237
fn collect_lc_captures(
239238
&mut self,
240239
groups: &[GeneratorGroup<'_>],
@@ -313,7 +312,7 @@ impl IrBuilder {
313312
/// Build one LC helper function (helper_k of n total).
314313
///
315314
/// Parameters for helper_k:
316-
/// iter_k, acc, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…
315+
/// iter_k, acc, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…, filter_values…
317316
///
318317
/// The accumulator collects elements in reverse order. When the outermost
319318
/// helper (k==0) reaches the nil case, it reverses the accumulator via
@@ -324,13 +323,14 @@ impl IrBuilder {
324323
k: usize,
325324
n: usize,
326325
groups: &[GeneratorGroup<'_>],
327-
helpers: &[(AtomId, String, u32)], // (name_id, mangled, arity)
326+
helpers: &[(AtomId, String)], // (name_id, mangled)
328327
captured_vars: &[(AtomId, VarId)],
329328
body_expr: &ast::Expr,
330329
finalize_as_binary: bool,
331330
bin_gen_flags: &[bool],
331+
filter_bindings: &mut Vec<Vec<AtomId>>,
332332
) -> Result<(), LowerError> {
333-
let (helper_name_id, ref helper_mangled, helper_arity) = helpers[k];
333+
let (helper_name_id, ref helper_mangled) = helpers[k];
334334
// Each helper is its own function. Deferred tails recorded while
335335
// building the previous helper name that helper's variables.
336336
self.match_contexts.clear();
@@ -355,6 +355,18 @@ impl IrBuilder {
355355
self.bind_var(*name, param);
356356
}
357357

358+
// Preserve one slot per binding layer, even if a later generator
359+
// shadows the same name. Resuming an outer helper needs its old value.
360+
let filter_params: Vec<Vec<(AtomId, VarId)>> = filter_bindings.iter()
361+
.map(|names| names.iter().map(|name| (*name, self.fresh_var())).collect())
362+
.collect();
363+
let carried_filters: Vec<VarId> = filter_params.iter().flatten()
364+
.map(|(_, var)| *var).collect();
365+
let helper_arity = 2 + 2 * k as u32 + env_params.len() as u32
366+
+ carried_filters.len() as u32;
367+
self.module_functions.insert((helper_name_id, helper_arity), helper_mangled.clone());
368+
let mut new_filter_names = Vec::new();
369+
358370
// ── Nil block ──
359371
let nil_block = self.fresh_block_with_id();
360372
self.start_block(nil_block);
@@ -390,6 +402,7 @@ impl IrBuilder {
390402
chain_args.push(*item);
391403
}
392404
chain_args.extend_from_slice(&env_params);
405+
chain_args.extend(filter_params[..k - 1].iter().flatten().map(|(_, var)| *var));
393406
self.set_terminator(Terminator::TailCall(
394407
CallTarget::Local(helpers[k - 1].0),
395408
chain_args,
@@ -414,6 +427,9 @@ impl IrBuilder {
414427
// segment size), so they are extracted before it runs.
415428
for j in 0..k {
416429
self.bind_generator_pattern(&groups[j].0, head_params[j], false);
430+
for (name, var) in &filter_params[j] {
431+
self.bind_var(*name, *var);
432+
}
417433
}
418434

419435
let generator_inputs = self.collect_all_bindings();
@@ -442,11 +458,12 @@ impl IrBuilder {
442458
(generator_scopes, generator_match_contexts)
443459
};
444460

461+
let before_filters = self.collect_all_bindings();
445462
let filters = &groups[k].2;
446463
let is_innermost = k == n - 1;
447464

448465
// Helper: build args for a self tail-call.
449-
// self(tail_var, acc_var, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…)
466+
// self(tail_var, acc_var, tail_0…tail_{k-1}, head_0…head_{k-1}, captured…, filter_values…)
450467
let build_self_args = |tail_params: &[VarId],
451468
head_params: &[VarId],
452469
env_params: &[VarId],
@@ -457,6 +474,7 @@ impl IrBuilder {
457474
a.extend_from_slice(tail_params);
458475
a.extend_from_slice(head_params);
459476
a.extend_from_slice(env_params);
477+
a.extend_from_slice(&carried_filters);
460478
a
461479
};
462480

@@ -487,6 +505,7 @@ impl IrBuilder {
487505
inner_args.extend_from_slice(&head_params);
488506
inner_args.push(head_var);
489507
inner_args.extend_from_slice(&env_params);
508+
inner_args.extend_from_slice(&carried_filters);
490509
self.set_terminator(Terminator::TailCall(
491510
CallTarget::Local(helpers[k + 1].0),
492511
inner_args,
@@ -531,6 +550,21 @@ impl IrBuilder {
531550
self_args,
532551
));
533552
} else {
553+
// Lowering may retain names from skipped short-circuit
554+
// operands. Only definite exports can cross helper boundaries.
555+
let mut definite = HashSet::new();
556+
for filter in filters {
557+
crate::warnings::collect_exported_bindings(filter, &mut definite);
558+
}
559+
new_filter_names = definite.into_iter()
560+
.filter(|name| !before_filters.contains_key(name)
561+
&& self.lookup_var(*name).is_some()).collect();
562+
new_filter_names.sort_by_key(|name| name.0);
563+
// Materialize deferred tails before passing them to a helper.
564+
let new_filter_values: Vec<VarId> = new_filter_names.iter().map(|name| {
565+
let var = self.lookup_var(*name).expect("filter binding exists");
566+
self.materialize_bound_variable(*name, var)
567+
}).collect();
534568
let next_iter = self.lower_expr(groups[k + 1].1, false)?;
535569

536570
let mut inner_args = vec![next_iter, acc_param];
@@ -539,6 +573,8 @@ impl IrBuilder {
539573
inner_args.extend_from_slice(&head_params);
540574
inner_args.push(head_var);
541575
inner_args.extend_from_slice(&env_params);
576+
inner_args.extend_from_slice(&carried_filters);
577+
inner_args.extend_from_slice(&new_filter_values);
542578
self.set_terminator(Terminator::TailCall(
543579
CallTarget::Local(helpers[k + 1].0),
544580
inner_args,
@@ -627,6 +663,7 @@ impl IrBuilder {
627663
all_params.extend_from_slice(&tail_params);
628664
all_params.extend_from_slice(&head_params);
629665
all_params.extend_from_slice(&env_params);
666+
all_params.extend_from_slice(&carried_filters);
630667
let mut helper_func = IrFunction {
631668
name: helper_name_id,
632669
arity: helper_arity,
@@ -643,6 +680,7 @@ impl IrBuilder {
643680
#[cfg(test)]
644681
crate::passes::assert_valid(&helper_func, "after helper yield insertion");
645682
self.functions.push(helper_func);
683+
filter_bindings.push(new_filter_names);
646684

647685
Ok(())
648686
}

‎crates/nera-ir/src/lower/tests.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1925,6 +1925,12 @@ fn test_binary_bound_segment_lowers() {
19251925
.expect("bound binary segment values must lower to valid IR");
19261926
}
19271927

1928+
#[test]
1929+
fn test_comprehension_filter_bindings_lowers() {
1930+
lower_source(include_str!("../../../../tests/comprehension_filter_bindings.erl"))
1931+
.expect("filter bindings must reach later helpers as initialized values");
1932+
}
1933+
19281934
#[test]
19291935
fn test_generator_size_rebind_lowers() {
19301936
// NERA-52: the final binary-consumption branch must preserve initialized

‎crates/nera-ir/src/warnings/exported_vars.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,7 @@ fn check_exported_in_subexprs(
305305
}
306306
ast::LcQualifier::Filter(filter_expr, _) => {
307307
check_exported_in_subexprs(filter_expr, &lc_scope, atom_names, diagnostics);
308+
collect_exported_bindings(filter_expr, &mut lc_scope);
308309
}
309310
}
310311
}

‎crates/nera-ir/src/warnings/match_underscore_var.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use nera_syntax::error::Diagnostic;
1111
use nera_syntax::span::Span;
1212

1313
use super::ast_walkers::{collect_pattern_bindings, collect_pattern_bindings_with_spans};
14+
use super::unbound_vars::collect_exported_bindings;
1415

1516
/// Check a function for `_`-prefixed variables reused in pattern position.
1617
pub(super) fn check_match_underscore_vars_fn(
@@ -231,6 +232,7 @@ fn check_expr(
231232
}
232233
ast::LcQualifier::Filter(filter_expr, _) => {
233234
check_expr(filter_expr, &lc_scope, atom_names, warnings);
235+
collect_exported_bindings(filter_expr, &mut lc_scope);
234236
}
235237
}
236238
}

‎crates/nera-ir/src/warnings/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ use nera_syntax::error::Diagnostic;
5656
pub(crate) use ast_walkers::{collect_pattern_bindings, collect_pattern_refs, collect_expr_refs, collect_guard_refs};
5757
pub(crate) use binary_segments::first_refused_segment_error;
5858
pub(crate) use illegal_guards::is_guard_expr;
59+
pub(crate) use unbound_vars::collect_exported_bindings;
5960

6061
// Re-export for cross-module unused record detection.
6162
pub use unused_records::collect_used_record_names;

‎crates/nera-ir/src/warnings/shadowed.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use nera_syntax::ast::{self, AtomId};
1212
use nera_syntax::error::Diagnostic;
1313

1414
use super::ast_walkers::{collect_pattern_bindings, collect_pattern_bindings_with_spans};
15+
use super::unbound_vars::collect_exported_bindings;
1516

1617
/// Check a function for shadowed variables in nested scopes.
1718
pub(super) fn check_shadowed_variables_fn(
@@ -107,11 +108,12 @@ fn check_shadowed_in_expr(
107108
| ast::LcQualifier::BinGenerator(pat, source_expr, _) => {
108109
// The source expression is evaluated in the current scope.
109110
check_shadowed_in_expr(source_expr, &lc_scope, atom_names, warnings);
110-
// Generator pattern vars can shadow outer scope.
111+
// Generator patterns can shadow outer bindings,
112+
// including earlier generators and filters.
111113
let mut bound = Vec::new();
112114
collect_pattern_bindings_with_spans(pat, &mut bound);
113115
for (atom_id, span) in &bound {
114-
if outer_scope.contains(atom_id) {
116+
if lc_scope.contains(atom_id) {
115117
let name = atom_names
116118
.get(atom_id)
117119
.map(|s| s.as_str())
@@ -126,11 +128,14 @@ fn check_shadowed_in_expr(
126128
).with_code("shadowed_variable"));
127129
}
128130
}
129-
lc_scope.insert(*atom_id);
130131
}
132+
// Repeated names within this pattern are comparisons,
133+
// not shadowing of a name this same pattern introduces.
134+
lc_scope.extend(bound.iter().map(|(name, _)| *name));
131135
}
132136
ast::LcQualifier::Filter(filter_expr, _) => {
133137
check_shadowed_in_expr(filter_expr, &lc_scope, atom_names, warnings);
138+
collect_exported_bindings(filter_expr, &mut lc_scope);
134139
}
135140
}
136141
}

‎crates/nera-ir/src/warnings/tests.rs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1456,6 +1456,69 @@ fn pattern_inputs_keep_exported_bindings() {
14561456
}
14571457
}
14581458

1459+
#[test]
1460+
fn comprehension_filter_bindings_are_available_to_later_qualifiers() {
1461+
let (module, names) = parse_module(include_str!(
1462+
"../../../../tests/comprehension_filter_bindings.erl"
1463+
));
1464+
let diagnostics = analyze_module(&module, &names, None);
1465+
assert!(
1466+
!diagnostics.iter().any(|d| d.severity == nera_syntax::error::Severity::Error),
1467+
"filter bindings must be valid later inputs: {diagnostics:?}"
1468+
);
1469+
}
1470+
1471+
#[test]
1472+
fn comprehension_filter_bindings_reach_scope_sensitive_diagnostics() {
1473+
for (open, close) in [("[", "]"), ("<<", ">>")] {
1474+
for (template, qualifiers, expected) in [
1475+
("case V of N -> <<1>>; _ -> <<0>> end",
1476+
"X <- [1], begin N = X, true end", None),
1477+
("begin case V of true -> N = 1; false -> ok end, <<N>> end",
1478+
"X <- [1], begin N = X, true end", None),
1479+
("begin case V of true -> N = 1; false -> N = 1 end, <<N>> end",
1480+
"X <- [1], begin N = X, true end", None),
1481+
("begin F = fun(N) -> <<N>> end, F(V) end",
1482+
"X <- [1], begin N = X, true end", Some("shadowed_variable")),
1483+
("<<N>>", "X <- [1], begin N = X, true end, N <- [V]",
1484+
Some("shadowed_variable")),
1485+
("<<_N>>", "X <- [V], begin _N = X, true end, begin _N = 1, true end",
1486+
Some("match_underscore_var")),
1487+
("<<_N>>", "X <- [true], X orelse begin _N = 1, true end, _N <- [V]", None),
1488+
("<<N>>", "_ <- [V], {N, N} <- [{1, 1}]", None),
1489+
] {
1490+
let source = format!("-module(test). -export([f/1]). f(V) ->
1491+
{open} ({template}) || {qualifiers} {close}.");
1492+
let (module, names) = parse_module(&source);
1493+
let diagnostics = analyze_module(&module, &names, None);
1494+
assert!(
1495+
!diagnostics.iter().any(|d| d.severity == nera_syntax::error::Severity::Error),
1496+
"{source}\n{diagnostics:?}"
1497+
);
1498+
let relevant: Vec<_> = diagnostics.iter().filter_map(|d| d.code.as_deref())
1499+
.filter(|code| ["unsafe_variable", "exported_var", "unreachable_clause",
1500+
"unused_variable", "shadowed_variable", "match_underscore_var"].contains(code))
1501+
.collect();
1502+
assert_eq!(relevant, expected.into_iter().collect::<Vec<_>>(),
1503+
"{source}\n{diagnostics:?}");
1504+
}
1505+
}
1506+
}
1507+
1508+
#[test]
1509+
fn comprehension_filters_do_not_export_local_or_partial_bindings() {
1510+
for expr in [
1511+
"[N || X <- [1], begin N = X, true end, _ <- [ok]], N",
1512+
"[N || X <- [1], case X of 1 -> N = 1, true; _ -> true end, _ <- [ok]]",
1513+
"[N || X <- [1], X > 0 andalso begin N = 1, true end, _ <- [ok]]",
1514+
"[N || _ <- begin N = 1, [ok] end]",
1515+
] {
1516+
assert_unbound_pattern_input(
1517+
&format!("-module(test). -export([f/0]). f() -> {expr}."), "N",
1518+
);
1519+
}
1520+
}
1521+
14591522
#[test]
14601523
fn unbound_in_guard() {
14611524
let src = r#"

0 commit comments

Comments
 (0)