diff --git a/ExaModelsC/src/ExaModelsC.jl b/ExaModelsC/src/ExaModelsC.jl index d5dd7f7e..321939eb 100644 --- a/ExaModelsC/src/ExaModelsC.jl +++ b/ExaModelsC/src/ExaModelsC.jl @@ -175,22 +175,31 @@ is a complete model, so `compile_library(out, core)` compiles it as-is: no instantiation data is required. A core that *declared* placeholders (`nargs = Val(N)`, `N > 0`) is refused without examples. -The library exports, for `prefix` `P`: `P_nargs() -> 0 or 1` (how many -instantiation arguments `P_new` consumes), `P_new(n) -> id` (a positive -instance id; any number of instances may coexist), then id-first `P_nvar`, -`P_ncon`, `P_nnzj`, `P_nnzh`, `P_meta`, `P_obj`, `P_grad`, `P_cons`, -`P_jac_structure`, `P_jac`, `P_hess_structure`, `P_hess`. Indices are 1-based; -the Hessian is the lower triangle of `obj_weight * ∇²f + Σᵢ yᵢ ∇²cᵢ`; every -function returns a `Cint` status, `0` on success, and none of them throws -across the boundary. - -`P_new` takes a single integer, so the recipe form applies when the example -`arg` is an `Integer` — the "scalable model" case (`rosenbrock` at size `N`). -Structured instantiation (the schema + builder ABI, for data-defined models -such as OPF) is not built yet; `compile_library` says so rather than emitting -a library that would fail at load, and prints the schema the model would -need. That is the only surface, so several example values, a float, an array, -a table, or a `NamedTuple` are all refused for now. +The library exports, for `prefix` `P`, one of two **instantiation surfaces**, +then the shared evaluators — id-first `P_nvar`, `P_ncon`, `P_nnzj`, `P_nnzh`, +`P_meta`, `P_obj`, `P_grad`, `P_cons`, `P_jac_structure`, `P_jac`, +`P_hess_structure`, `P_hess`. Indices are 1-based; the Hessian is the lower +triangle of `obj_weight * ∇²f + Σᵢ yᵢ ∇²cᵢ`; every function returns a `Cint` +status, `0` on success, and none of them throws across the boundary. + +**One integer** (`rosenbrock` at size `N` — a bare `Integer` example, or a +`NamedTuple` holding exactly one): `P_new(n) -> id` (a positive instance id; +any number of instances may coexist), and `P_nargs() -> 0 or 1` saying whether +`n` is consumed or ignored (0 is the fixed-model case). + +**Anything else** — several example values, floats, arrays, tables (vectors +of NamedTuples), or NamedTuples of these, exactly as `ExaModel` takes them — +gets the schema + builder ABI instead of `P_new`: `P_schema` publishes a JSON +description of the fields, and `P_data_begin` / `P_set_scalar_{i64,f64}` / +`P_set_array_{i64,f64}` / `P_set_col_{i64,f64}` / `P_data_ready` / +`P_new_from_data -> id` take the values by field name and reassemble the +`ExaModel` arguments. A `NamedTuple` example flattens into one schema field +per entry, named by its key; bare values are named `arg1`, `arg2`, ... by +position. Both consumers already speak this surface, binding one value per +field positionally — `CNLPModel(lib, 3, lo, v)` — so a compiled model is +consumed the way it was written. Builder examples must be `Int64`/`Float64` +exactly (as scalars, `Vector`s, or table entries): the example's type IS the +compiled storage's type. ## Several models in one library @@ -244,9 +253,10 @@ one model, so `prefix =` has no meaning here and is not accepted — the model names supply the prefixes. Every other keyword behaves as in the single-model form. -Each model is subject to the same restriction as the single-model form: one -integer example value, or none for a fixed model. The schema + builder surface -(several arguments, arrays, tables) is not emitted yet, for any model. +Each model gets whichever instantiation surface its examples call for, exactly +as in the single-model form: `P_new(n)` for one integer, the schema + builder +ABI for anything else, per prefix — the surfaces coexist freely in one +library. """ function compile_library( out::AbstractString, @@ -330,15 +340,18 @@ function _model_spec(prefix::AbstractString, core::ExaModels.ExaCore, args::Tupl ) return ModelSpec(String(prefix), core, nothing, FixedModel()) end - fields = _schema(args) - _is_scalar_new(fields) || throw( - ArgumentError( - "`$prefix`: this recipe needs the schema + builder interface, which is " * - "not emitted yet — only a single integer placeholder " * - "(`$(prefix)_new(n)`) is. Its schema would be: " * _schema_json(fields), - ), - ) - return ModelSpec(String(prefix), core, only(args), nothing) + bm = _builder_model(args) + # One integer placeholder — bare, or a one-key NamedTuple — keeps the + # `P_new(n)` fast path: one C call, no builder. The two surfaces are + # disjoint on purpose: a library exports `P_new` exactly when its schema + # is a single integer scalar, which is what lets a consumer route a lone + # integer without guessing. + if _is_scalar_new(bm.fields) + spec = only(bm.argspec) + field = spec isa String ? nothing : first(only(spec)) + return ModelSpec(String(prefix), core, only(args), field) + end + return ModelSpec(String(prefix), core, args, bm) end # The shared back half: probe every model, generate one app carrying all of @@ -354,7 +367,24 @@ function _compile(specs::Vector{ModelSpec}, out, libname, trim, bundle, verbose) # the failure is far cheaper to read now than after a juliac run — the more # so with several models, where one bad core would waste the whole compile. for s in specs - probe = ExaModels.ExaModel(s.core, s.arg) + # A builder spec carries its examples as a tuple, one per placeholder; + # the other forms carry a single value (or `nothing` for a fixed core). + probe = try + s.arg isa Tuple ? ExaModels.ExaModel(s.core, s.arg...) : + ExaModels.ExaModel(s.core, s.arg) + catch err + # A shape mismatch between the examples and how the core reads its + # placeholders (a NamedTuple where a bare size is expected, a + # missing key, ...) surfaces here — as the caller's error, before + # minutes are spent compiling. + throw( + ArgumentError( + "`$(s.prefix)`: the example values do not instantiate this " * + "core — `ExaModel(core, example...)` failed with: " * + sprint(showerror, err), + ), + ) + end verbose && @info "compile_library: core instantiates" prefix = s.prefix nvar = probe.meta.nvar ncon = probe.meta.ncon end @@ -418,11 +448,13 @@ _default_out_prefix(out::AbstractString) = # ── Reading the example arguments ───────────────────────────────────────────── # -# The schema is derived from the example values' TYPES, one field per -# placeholder. Placeholders are positional, so the fields are named `arg1`, -# `arg2`, ... — and a consumer binds its own arguments positionally against -# that field order, `CNLPModel(lib, arg1, arg2, ...)`, the same spelling the -# example values are given in here. +# The schema is derived from the example values' TYPES. A bare number or +# vector is one field, named `arg1`, `arg2`, ... by position; a NamedTuple +# example — the shape `ExaModel` takes for a source carrying several values — +# flattens into one field per entry, named by its key. A consumer binds its +# own values positionally against the flat field order, +# `CNLPModel(lib, v1, v2, ...)`, and the library reassembles the NamedTuples +# before instantiating. struct Field name::String @@ -484,6 +516,108 @@ end _is_scalar_new(fields) = length(fields) == 1 && fields[1].kind == "scalar" && fields[1].type == "i64" +# Everything else gets the schema + builder surface (ABI v2): the consumer +# opens a builder, sets each field by name, and `P_new_from_data` reassembles +# the `ExaModel` arguments exactly as the example values were given here. +# `argspec` records that mapping — one entry per `ExaModel` positional +# argument: +# +# a `String` — a bare value, stored under that field +# a `Vector{Pair{Symbol,String}}` — a NamedTuple, one (key => field) per entry +struct BuilderModel + fields::Vector{Field} + argspec::Vector{Union{String, Vector{Pair{Symbol,String}}}} +end + +# Builder storage is GENERATED from the example types, and the model vector's +# element type is fixed by instantiating the example at precompile time — so +# the example must BE the type the builder will reconstruct: Int64/Float64 +# exactly, as scalars, `Vector`s, or vectors of NamedTuples of them. A looser +# example (`Int32`, a range, `Real[]`) would compile a MODELS vector the +# reconstruction cannot feed, and the mismatch would surface only inside the +# compiled library; refused here instead. +_check_exact(name, v::Union{Int64, Float64, Vector{Int64}, Vector{Float64}}) = v +function _check_exact(name, v::Vector{T}) where {T <: NamedTuple} + (isconcretetype(T) && all(t -> t <: Union{Int64, Float64}, fieldtypes(T))) || + _exact_err(name, v) + return v +end +_check_exact(name, v) = _exact_err(name, v) +_exact_err(name, v) = throw( + ArgumentError( + "`$name`: builder examples must be Int64/Float64 values — as scalars, " * + "as Vector{Int64}/Vector{Float64}, or as a Vector of NamedTuples of " * + "them (a table); got $(typeof(v)). The storage the library compiles " * + "is exactly the example's type.", + ), +) + +function _builder_model(args) + fields = Field[] + argspec = Union{String, Vector{Pair{Symbol,String}}}[] + seen = Set{String}() + claim = function (name, where_) + name in seen && throw( + ArgumentError( + "two schema fields would both be named `$name` (the second from " * + "$where_) — field names come from NamedTuple keys and `argN` " * + "positions, and must be distinct across all placeholders; " * + "rename one.", + ), + ) + push!(seen, name) + return name + end + for (i, a) in enumerate(args) + if a isa NamedTuple + entries = Pair{Symbol,String}[] + for k in keys(a) + name = claim(String(k), "argument $i") + push!(fields, _field(name, _check_exact(name, getfield(a, k)))) + push!(entries, k => name) + end + push!(argspec, entries) + else + name = claim("arg$i", "argument $i") + push!(fields, _field(name, _check_exact(name, a))) + push!(argspec, name) + end + end + # Tables flatten further, into one storage slot per column — those names + # must be distinct too, and a table needs at least one column to have an + # element type at all. + slots = String[] + for f in fields + f.kind == "table" && isempty(f.columns) && throw( + ArgumentError( + "`$(f.name)`: the example table has no columns — give " * + "NamedTuples with at least one entry.", + ), + ) + append!(slots, (s for (s, _, _) in _slots(f))) + end + allunique(slots) || throw( + ArgumentError( + "field and table-column names collide once flattened to storage " * + "slots ($(join(slots, ", "))) — rename one of the duplicates.", + ), + ) + return BuilderModel(fields, argspec) +end + +# The flat storage behind a builder: (slot identifier, storage type, zero +# value), one slot per scalar/array field and per table column. Each slot +# also carries a `_set::Bool` beside it in the generated struct. +function _slots(f::Field) + jt(t) = t == "i64" ? "Int64" : "Float64" + jz(t) = t == "i64" ? "0" : "0.0" + jvt(t) = t == "i64" ? "Vector{Int64}" : "Vector{Float64}" + jvz(t) = t == "i64" ? "Int64[]" : "Float64[]" + f.kind == "scalar" && return [(f.name, jt(f.type), jz(f.type))] + f.kind == "array" && return [(f.name, jvt(f.type), jvz(f.type))] + return [("$(f.name)_$(c.first)", jvt(c.second), jvz(c.second)) for c in f.columns] +end + # The prefix becomes a C symbol and a Julia identifier in generated source, so # it has to be one. Checked here rather than discovered as a syntax error in a # generated file nobody is looking at. @@ -672,6 +806,249 @@ _arg_expr(::FixedModel) = "nothing" _nargs_value(::FixedModel) = 0 _nargs_value(::Union{Nothing, Symbol}) = 1 +# ── Generating one model's instantiation surface ───────────────────────────── +# +# A fixed or one-integer model gets `P_nargs` + `P_new(n)`. Everything else +# gets the schema + builder surface (ABI v2) and NO `P_new` — the consumers +# rely on that disjointness to route a lone integer. All storage is +# concretely typed from the example values, so `--trim=safe` sees no dynamic +# containers. + +function _instantiation_source(p::AbstractString, field::Union{Nothing, Symbol, FixedModel}) + argexpr = _arg_expr(field) + return """ + # How many instantiation arguments `$(p)_new` consumes (0 = fixed model, + # `n` is ignored). Lets a consumer decide whether `args` are required + # before instantiating anything. + Base.@ccallable function $(p)_nargs()::Cint + return Cint($(_nargs_value(field))) + end + + Base.@ccallable function $(p)_new(n::Cint)::Cint + try + push!(MODELS_$p, ExaModels.ExaModel(CORE_$p, $argexpr; check = Val(false))) + return Cint(length(MODELS_$p)) + catch + return Cint(0) # 0 is the failure value for _new + end + end +""" +end + +# How a slot is read back out of a builder when the model is instantiated: a +# table reassembles into the example's row type, column by column. +function _slot_expr(f::Field) + f.kind == "table" || return "B.$(f.name)" + row = join(("$(c.first) = B.$(f.name)_$(c.first)[_k]" for c in f.columns), ", ") + return "[(; $row) for _k in eachindex(B.$(f.name)_$(first(f.columns).first))]" +end + +# One `if` arm per field a setter can legitimately name; a name that matches +# no arm is status 1, the caller's error, not ours. +function _setter_arms(fields, render) + isempty(fields) && return "" + return join((render(f) for f in fields), "") * "\n" +end + +function _instantiation_source(p::AbstractString, bm::BuilderModel) + fields = bm.fields + byname = Dict(f.name => f for f in fields) + slots = [sl for f in fields for sl in _slots(f)] + json = _schema_json(fields) + + decls = join((" $s::$t\n $(s)_set::Bool\n" for (s, t, _) in slots)) + zeros = join(("$z, false" for (_, _, z) in slots), ", ") + flags = join(("B.$(s)_set" for (s, _, _) in slots), " && ") + + scalar(f) = """ + if f == $(repr(f.name)) + B.$(f.name) = v + B.$(f.name)_set = true + return Cint(0) + end + """ + array(f) = """ + if f == $(repr(f.name)) + B.$(f.name) = copy(unsafe_wrap(Array, v, Int(len))) + B.$(f.name)_set = true + return Cint(0) + end + """ + col(f, c) = """ + if t == $(repr(f.name)) && c == $(repr(c.first)) + B.$(f.name)_$(c.first) = copy(unsafe_wrap(Array, v, Int(len))) + B.$(f.name)_$(c.first)_set = true + return Cint(0) + end + """ + pick(kind, type) = [f for f in fields if f.kind == kind && f.type == type] + cols(type) = [ + (f, c) for f in fields if f.kind == "table" for c in f.columns if c.second == type + ] + + # Table columns must agree in length before rows can be reassembled. + samelen = join( + ( + " (" * + join( + ("length(B.$(f.name)_$(c.first)) == length(B.$(f.name)_$(first(f.columns).first))" + for c in f.columns[2:end]), + " && ", + ) * + ") || return Cint(0)\n" + for f in fields if f.kind == "table" && length(f.columns) > 1 + ), + ) + + asm = join( + ( + spec isa String ? _slot_expr(byname[spec]) : + "(; " * join(("$(k) = $(_slot_expr(byname[n]))" for (k, n) in spec), ", ") * ")" + for spec in bm.argspec + ), + ",\n ", + ) + + return """ + # ── builder for `$p` (schema + typed setters, ABI v2) ──────────────────── + + const SCHEMA_$p = Vector{UInt8}($(repr(json))) + + # Returns the schema's byte length; copies what fits in `cap`. + Base.@ccallable function $(p)_schema(buf::Ptr{UInt8}, cap::Cint)::Cint + n = length(SCHEMA_$p) + k = min(Int(cap), n) + if k > 0 && buf != Ptr{UInt8}(0) + GC.@preserve SCHEMA_$p unsafe_copyto!(buf, pointer(SCHEMA_$p), k) + end + return Cint(n) + end + + # One concretely-typed slot per scalar/array field and per table column; + # the `_set` flags are what make completeness checkable without sentinels. + mutable struct Builder_$p +$decls end + Builder_$p() = Builder_$p($zeros) + const BUILDERS_$p = Builder_$p[] + + @inline _bvalid_$p(b::Cint) = 1 <= b <= length(BUILDERS_$p) + + Base.@ccallable function $(p)_data_begin()::Cint + try + push!(BUILDERS_$p, Builder_$p()) + return Cint(length(BUILDERS_$p)) + catch + return Cint(0) + end + end + + Base.@ccallable function $(p)_set_scalar_i64(b::Cint, field::Ptr{UInt8}, v::Clonglong)::Cint + _bvalid_$p(b) || return Cint(1) + try + f = unsafe_string(field) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(pick("scalar", "i64"), scalar)) return Cint(1) + catch + return Cint(2) + end + end + + Base.@ccallable function $(p)_set_scalar_f64(b::Cint, field::Ptr{UInt8}, v::Cdouble)::Cint + _bvalid_$p(b) || return Cint(1) + try + f = unsafe_string(field) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(pick("scalar", "f64"), scalar)) return Cint(1) + catch + return Cint(2) + end + end + + Base.@ccallable function $(p)_set_array_i64( + b::Cint, field::Ptr{UInt8}, v::Ptr{Clonglong}, len::Cint, + )::Cint + _bvalid_$p(b) || return Cint(1) + try + f = unsafe_string(field) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(pick("array", "i64"), array)) return Cint(1) + catch + return Cint(2) + end + end + + Base.@ccallable function $(p)_set_array_f64( + b::Cint, field::Ptr{UInt8}, v::Ptr{Cdouble}, len::Cint, + )::Cint + _bvalid_$p(b) || return Cint(1) + try + f = unsafe_string(field) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(pick("array", "f64"), array)) return Cint(1) + catch + return Cint(2) + end + end + + Base.@ccallable function $(p)_set_col_i64( + b::Cint, table::Ptr{UInt8}, column::Ptr{UInt8}, v::Ptr{Clonglong}, len::Cint, + )::Cint + _bvalid_$p(b) || return Cint(1) + try + t = unsafe_string(table) + c = unsafe_string(column) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(cols("i64"), fc -> col(fc...))) return Cint(1) + catch + return Cint(2) + end + end + + Base.@ccallable function $(p)_set_col_f64( + b::Cint, table::Ptr{UInt8}, column::Ptr{UInt8}, v::Ptr{Cdouble}, len::Cint, + )::Cint + _bvalid_$p(b) || return Cint(1) + try + t = unsafe_string(table) + c = unsafe_string(column) + B = BUILDERS_$p[Int(b)] +$(_setter_arms(cols("f64"), fc -> col(fc...))) return Cint(1) + catch + return Cint(2) + end + end + + # 1 iff every field is set and every table's columns agree in length. + Base.@ccallable function $(p)_data_ready(b::Cint)::Cint + _bvalid_$p(b) || return Cint(0) + B = BUILDERS_$p[Int(b)] + ($flags) || return Cint(0) +$samelen return Cint(1) + end + + Base.@ccallable function $(p)_new_from_data(b::Cint)::Cint + $(p)_data_ready(b) == Cint(1) || return Cint(0) + try + B = BUILDERS_$p[Int(b)] + push!(MODELS_$p, ExaModels.ExaModel( + CORE_$p, + $asm; + check = Val(false), + )) + return Cint(length(MODELS_$p)) + catch + return Cint(0) + end + end + + # Informative — there is no `$(p)_new` here; the consumers bind one value + # per schema field, positionally, and instantiate through the builder. + Base.@ccallable function $(p)_nargs()::Cint + return Cint($(length(fields))) + end +""" +end + function _module_source(modname::AbstractString, specs::Vector{ModelSpec}, pkgs = _Pkg[]) # Imported for their side effect on `Serialization`: a module has to be # loaded before a type it owns can be resolved by `PkgId`, and the @@ -695,7 +1072,9 @@ end # `add_ccallables` picks up all of them regardless of how many there are. function _model_source(s::ModelSpec) p = s.prefix - argexpr = _arg_expr(s.field) + # A builder spec's example is the whole tuple of values, splatted back the + # way `ExaModel` takes them; the other forms carry a single value. + example = s.field isa BuilderModel ? "ARG0_$p..." : "ARG0_$p" return """ # ── model `$p` ─────────────────────────────────────────────────────────── @@ -710,26 +1089,11 @@ function _model_source(s::ModelSpec) # placeholder-leak guard, which walks types reflectively and is not # trimmable — the check already ran, on this exact core, in the process # that called `compile_library`. - const MODELS_$p = typeof(ExaModels.ExaModel(CORE_$p, ARG0_$p; check = Val(false)))[] + const MODELS_$p = typeof(ExaModels.ExaModel(CORE_$p, $example; check = Val(false)))[] @inline _valid_$p(id::Cint) = 1 <= id <= length(MODELS_$p) - # How many instantiation arguments `$(p)_new` consumes (0 = fixed model, - # `n` is ignored). Lets a consumer decide whether `args` are required - # before instantiating anything. - Base.@ccallable function $(p)_nargs()::Cint - return Cint($(_nargs_value(s.field))) - end - - Base.@ccallable function $(p)_new(n::Cint)::Cint - try - push!(MODELS_$p, ExaModels.ExaModel(CORE_$p, $argexpr; check = Val(false))) - return Cint(length(MODELS_$p)) - catch - return Cint(0) # 0 is the failure value for _new - end - end - +$(_instantiation_source(p, s.field)) Base.@ccallable function $(p)_nvar(id::Cint)::Cint _valid_$p(id) || return Cint(-1) return Cint(MODELS_$p[Int(id)].meta.nvar) diff --git a/ExaModelsC/test/builder_check.py b/ExaModelsC/test/builder_check.py new file mode 100644 index 00000000..4f19f931 --- /dev/null +++ b/ExaModelsC/test/builder_check.py @@ -0,0 +1,31 @@ +# Drives the schema + builder ABI of a compiled structured model from Python: +# positional values against the schema's field order, the table as a dict of +# columns. Inputs mirror the S_* constants in runtests.jl — keep them in sync. +# +# usage: python builder_check.py +import sys + +import numpy as np + +import cnlpmodels + +libpath, prefix, n, outfile = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] + +lib = cnlpmodels.load(libpath) +v0 = np.linspace(0.1, 0.6, n) +lo = np.full(n, -5.0) +tab = { + "i": np.array([2, 5, 6]), + "w": np.array([1.5, 3.0, 0.5]), + "s": np.array([2.0, -1.0, 0.0]), +} + +m = cnlpmodels.CModel(lib, n, v0, lo, tab, prefix=prefix) +x = np.linspace(0.5, 3.0, n) + +with open(outfile, "w") as f: + f.write("nvar %d\n" % m.nvar) + f.write("ncon %d\n" % m.ncon) + f.write("obj %.17g\n" % m.obj(x)) + f.write("grad " + " ".join("%.17g" % v for v in m.grad(x)) + "\n") + f.write("cons " + " ".join("%.17g" % v for v in m.cons(x)) + "\n") diff --git a/ExaModelsC/test/runtests.jl b/ExaModelsC/test/runtests.jl index c86823a0..2cb2c7ab 100644 --- a/ExaModelsC/test/runtests.jl +++ b/ExaModelsC/test/runtests.jl @@ -28,13 +28,33 @@ end const OUT = get(ENV, "EXAMODELSC_TEST_OUT", joinpath(tempdir(), "examodelsc_test")) +# A three-placeholder recipe — a bare size, a NamedTuple carrying a start and +# a bound, and a table — the data-defined shape the builder ABI exists for. +function sbuild() + c, sz, dat, tab = ExaCore(nargs = Val(3)) + @add_var(c, x, sz; start = dat.v0, lvar = dat.lo) + @add_obj(c, t.w * (x[t.i] - t.s)^2 for t in tab) + @add_con(c, x[i] + x[i+1] for i in 1:(sz - 1); lcon = -100.0, ucon = 100.0) + return c +end + +# The example values `sbuild` is compiled against, and the instantiation data +# the compile never sees (different size, different rows). +const S_EXTAB = [(i = 1, w = 2.0, s = 1.0), (i = 3, w = 1.0, s = 0.5)] +const S_EXARGS = (4, (v0 = fill(0.5, 4), lo = fill(-10.0, 4)), S_EXTAB) +const S_N = 6 +const S_V0 = collect(range(0.1, 0.6; length = S_N)) +const S_LO = fill(-5.0, S_N) +const S_TAB = [(i = 2, w = 1.5, s = 2.0), (i = 5, w = 3.0, s = -1.0), (i = 6, w = 0.5, s = 0.0)] + function runtests() @testset "ExaModelsC" begin @testset "the example arg is read, not guessed" begin c = build() - # A shape `P_new(n)` cannot carry must be refused up front, with a - # reason — not discovered as a missing symbol at load time. + # `build`'s core reads its one placeholder as a bare size, so a + # NamedTuple example cannot instantiate it — refused up front by + # the probe, with a reason, not discovered inside juliac. @test_throws ArgumentError compile_library(OUT, c, (N = 4, v = [1.0])) @test_throws ArgumentError compile_library(OUT, c, (x = 1.5,)) # And a prefix that is not a C identifier is caught before juliac. @@ -42,6 +62,53 @@ function runtests() @test_throws ArgumentError compile_library(OUT, c, 4; prefix = "2fast") end + @testset "builder examples are read, not guessed" begin + c = build() + # Types that cannot cross the boundary stay refused with the + # builder there. + @test_throws ArgumentError compile_library(OUT, c, "c") + @test_throws ArgumentError compile_library(OUT, c, 4, "c", [1.0, 2.0]) + # Builder storage is the example's type EXACTLY — looser numeric + # types are named here, not discovered inside the compiled library. + @test_throws "Int64/Float64" compile_library(OUT, c, Int32(4), [1.0]) + @test_throws "Int64/Float64" compile_library(OUT, c, 4, Float32[1.0]) + @test_throws "Int64/Float64" compile_library(OUT, c, 4, 1:3) + # Flattened field names must be distinct across all placeholders. + @test_throws "both be named" compile_library(OUT, c, (n = 1,), (n = 2.0,)) + # An empty example, and rows with no columns, carry no types. + @test_throws ArgumentError compile_library(OUT, c, [1.0], Float64[]) + @test_throws "no columns" compile_library(OUT, c, [1.0], [(;), (;)]) + + # A one-key integer NamedTuple keeps the `P_new(n)` fast path — + # the generated constructor rebuilds the key. + cnt, _ = ExaCore(nargs = Val(1)) + s = ExaModelsC._model_spec("nt", cnt, ((N = 4,),)) + @test s.field === :N + ntsrc = ExaModelsC._module_source("M", [s]) + @test occursin("(; N = Int(n))", ntsrc) + @test occursin("nt_new(", ntsrc) + @test !occursin("nt_data_begin", ntsrc) + + # Anything else flattens to the builder: bare values by position, + # NamedTuple entries by key, and NO `P_new` — the surfaces are + # disjoint, which is how a consumer routes a lone integer. + b = ExaModelsC._model_spec( + "bs", cnt, (4, (v0 = [1.0], lo = [2.0]), [(i = 1, w = 0.5)]), + ) + @test b.field isa ExaModelsC.BuilderModel + @test [f.name for f in b.field.fields] == ["arg1", "v0", "lo", "arg3"] + bsrc = ExaModelsC._module_source("M", [b]) + @test !occursin("bs_new(", bsrc) + for sym in ( + "bs_schema", "bs_data_begin", "bs_set_scalar_i64", + "bs_set_scalar_f64", "bs_set_array_i64", "bs_set_array_f64", + "bs_set_col_i64", "bs_set_col_f64", "bs_data_ready", + "bs_new_from_data", "bs_nargs", + ) + @test occursin(sym, bsrc) + end + end + @testset "a recipe's own package travels into the generated app" begin # A modelling library's own function inside the core is the ordinary # case, not an exotic one: a per-index start and a size-dependent @@ -380,9 +447,11 @@ function runtests() @test_throws "both named `a`" compile_library(OUT, :a => (c, 4), :a => (c, 5)) @test_throws "must be a C identifier" compile_library( OUT, Symbol("2bad") => (c, 4)) - # The per-model restrictions are the single-model ones, and the - # message names which model is at fault. - @test_throws "`b`: this recipe needs the schema" compile_library( + # The per-model checks are the single-model ones, and the message + # names which model is at fault: two bare integers flatten to a + # perfectly valid two-field schema, but this core declared ONE + # placeholder, so the probe refuses before juliac spends minutes. + @test_throws "`b`: the example values do not instantiate" compile_library( OUT, :a => (c, 4), :b => (c, 4, 5)) @test_throws "`b`: this core declared 1 placeholder" compile_library( OUT, :a => (c, 4), :b => c) @@ -390,6 +459,78 @@ function runtests() @test_throws MethodError compile_library(OUT, :a => (c, 4); prefix = "z") end + @testset "a structured model instantiates through the builder" begin + # One compile carries BOTH surfaces — the builder model and a + # one-knob model in one library: the surface is per prefix, not + # per file. + sb = compile_library( + joinpath(OUT, "structs"), + :structm => (sbuild(), S_EXARGS...), + :knob => (build(), 4), + ) + @test sb.prefixes == ["structm", "knob"] + slib = CNLPModels.load(sb.libpath) + + # The builder model exports no one-integer constructor; the knob + # model exports no builder. Disjoint, as the consumers assume. + dl(s) = CNLPModels.Libdl.dlsym(slib.handle, s; throw_error = false) + @test dl(:structm_new) === nothing + @test dl(:structm_data_begin) !== nothing + @test dl(:knob_new) !== nothing + @test dl(:knob_data_begin) === nothing + @test ccall(dl(:structm_nargs), Cint, ()) == 4 + + # The published schema is the flattened example: bare values by + # position, NamedTuple entries by key, the table with its columns. + sj = CNLPModels.schema_json(slib; prefix = "structm") + for needle in ( + "\"arg1\"", "\"v0\"", "\"lo\"", + """{"name":"arg3","kind":"table","columns":[{"name":"i","type":"i64"},{"name":"w","type":"f64"},{"name":"s","type":"f64"}]}""", + ) + @test occursin(needle, sj) + end + + # Instantiated at a size and data the compile never saw, through + # the consumer's positional spelling — one value per schema field. + m = CNLPModels.CNLPModel(slib, S_N, S_V0, S_LO, S_TAB; prefix = "structm") + ref = ExaModel(sbuild(), S_N, (v0 = S_V0, lo = S_LO), S_TAB) + + @test m.meta.nvar == ref.meta.nvar == S_N + @test m.meta.ncon == ref.meta.ncon == S_N - 1 + @test m.meta.x0 ≈ ref.meta.x0 + @test m.meta.lvar ≈ ref.meta.lvar + + x = collect(range(0.5, 3.0; length = S_N)) + y = collect(range(-1.0, 1.0; length = S_N - 1)) + @test NLPModels.obj(m, x) ≈ NLPModels.obj(ref, x) + @test NLPModels.grad(m, x) ≈ NLPModels.grad(ref, x) + @test NLPModels.cons(m, x) ≈ NLPModels.cons(ref, x) + @test NLPModels.jac_coord(m, x) ≈ NLPModels.jac_coord(ref, x) + @test NLPModels.hess_coord(m, x, y; obj_weight = 0.5) ≈ + NLPModels.hess_coord(ref, x, y; obj_weight = 0.5) + + # A second builder instance and the sibling knob model, with the + # first instance undisturbed. + m2 = CNLPModels.CNLPModel( + slib, 4, fill(0.5, 4), fill(-10.0, 4), S_EXTAB; prefix = "structm", + ) + @test m2.meta.nvar == 4 + @test NLPModels.obj(m, x) ≈ NLPModels.obj(ref, x) + k = CNLPModels.CNLPModel(slib, 7; prefix = "knob") + @test k.meta.nvar == 7 + + # Wrong-arity and incomplete data are the consumer's errors, not + # aborts: the library reports, the consumer explains. + @test_throws ErrorException CNLPModels.CNLPModel( + slib, S_N, S_V0; prefix = "structm") + + # And a solve through the builder-instantiated model. + res = NLPModelsIpopt.ipopt(m; print_level = 0) + refres = NLPModelsIpopt.ipopt(ref; print_level = 0) + @test res.status == refres.status + @test res.objective ≈ refres.objective atol = 1e-6 + end + @testset "the Python consumer reads the same model" begin # cnlpmodels is an unrelated package (https://github.com/MadNLP/cnlpmodels-py), # so this leg is skipped rather than failed when it is not present. @@ -467,6 +608,54 @@ function runtests() end end + @testset "the Python consumer drives the builder" begin + # Same probing as the leg above; additionally needs the structured + # library the builder testset compiled. + pysrc = get( + ENV, "CNLPMODELS_PY", + joinpath(homedir(), "git", "pkg", "cnlpmodels-py", "src"), + ) + py = nothing + for cand in ("python3", "python") + ok = try + success(pipeline(`$cand -c "import numpy"`; stdout = devnull, stderr = devnull)) + catch + false + end + ok && (py = cand; break) + end + slibpath = joinpath( + OUT, "structs", "libstructs." * Base.BinaryPlatforms.platform_dlext(), + ) + if py === nothing || !isdir(pysrc) || !isfile(slibpath) + @info "skipping the Python builder leg" python = py lib = isfile(slibpath) + @test_skip false + else + script = joinpath(@__DIR__, "builder_check.py") + outfile = joinpath(mktempdir(), "py.txt") + env = copy(ENV) + sep = Sys.iswindows() ? ";" : ":" + env["PYTHONPATH"] = + pysrc * (haskey(env, "PYTHONPATH") ? sep * env["PYTHONPATH"] : "") + run(setenv(`$py $script $slibpath structm $S_N $outfile`, env)) + + vals = Dict{String,Vector{Float64}}() + for line in eachline(outfile) + parts = split(line) + vals[parts[1]] = parse.(Float64, parts[2:end]) + end + # The script's inputs mirror S_V0/S_LO/S_TAB — the reference + # is the same in-Julia model the in-process leg checked. + ref = ExaModel(sbuild(), S_N, (v0 = S_V0, lo = S_LO), S_TAB) + x = collect(range(0.5, 3.0; length = S_N)) + @test only(vals["nvar"]) == ref.meta.nvar + @test only(vals["ncon"]) == ref.meta.ncon + @test only(vals["obj"]) ≈ NLPModels.obj(ref, x) + @test vals["grad"] ≈ NLPModels.grad(ref, x) + @test vals["cons"] ≈ NLPModels.cons(ref, x) + end + end + @testset "solving through the library agrees with solving in Julia" begin N = 20 m = CNLPModels.CNLPModel(lib, N; prefix = r.prefix) diff --git a/src/argument.jl b/src/argument.jl index 69d5a9ba..645c1f2c 100644 --- a/src/argument.jl +++ b/src/argument.jl @@ -181,17 +181,17 @@ julia> ExaModels.instantiate(x, (nh = 2,)) === x # identity, no arg dependen true ``` """ -@inline instantiate(x, a...) = x -@inline instantiate(::ArgSource{K}, a...) where {K} = a[K] -@inline instantiate(n::ArgIndexed{I, J}, a...) where {I, J} = +@inline instantiate(x, a::Vararg{Any,N}) where {N} = x +@inline instantiate(::ArgSource{K}, a::Vararg{Any,N}) where {K, N} = a[K] +@inline instantiate(n::ArgIndexed{I, J}, a::Vararg{Any,N}) where {I, J, N} = _arg_access(instantiate(getfield(n, :inner), a...), J) -@inline instantiate(n::ArgNode1, a...) = +@inline instantiate(n::ArgNode1, a::Vararg{Any,N}) where {N} = getfield(n, :f)(instantiate(getfield(n, :inner), a...)) -@inline instantiate(n::ArgNode2, a...) = getfield(n, :f)( +@inline instantiate(n::ArgNode2, a::Vararg{Any,N}) where {N} = getfield(n, :f)( instantiate(getfield(n, :inner1), a...), instantiate(getfield(n, :inner2), a...), ) -@inline instantiate(n::ArgCall, a...) = +@inline instantiate(n::ArgCall, a::Vararg{Any,N}) where {N} = getfield(n, :f)(map(x -> instantiate(x, a...), getfield(n, :args))...) @inline _arg_access(x, j::Symbol) = getproperty(x, j) @@ -204,8 +204,15 @@ true # that looks fully instantiated and is not. Mapping costs nothing observable: # an immutable tuple rebuilds `===` to itself, and each element is passed # through by identity when it has no dependency of its own. -@inline instantiate(t::Tuple, a...) = map(x -> instantiate(x, a...), t) -@inline instantiate(t::NamedTuple, a...) = map(x -> instantiate(x, a...), t) +# `Vararg{Any,N}` forces specialization on the arguments' concrete types: +# a vararg that is only splatted through is otherwise left unspecialized +# (Julia's passthrough heuristic), and the `map` below then carries a dynamic +# call that `juliac --trim=safe` cannot resolve. Harmless under the JIT, +# load-bearing for AOT — same reason on the `ExaCore` method in nlp.jl. +@inline instantiate(t::Tuple, a::Vararg{Any,N}) where {N} = + map(x -> instantiate(x, a...), t) +@inline instantiate(t::NamedTuple, a::Vararg{Any,N}) where {N} = + map(x -> instantiate(x, a...), t) """ _anyarg(xs...) diff --git a/src/graph.jl b/src/graph.jl index a41ec40b..fc9316c1 100644 --- a/src/graph.jl +++ b/src/graph.jl @@ -542,30 +542,30 @@ end # (`Constant`, `Null`, `VarSource`, `DataSource`, …) fall through to the generic # identity in argument.jl. -@inline function instantiate(n::Var{I}, a...) where {I} +@inline function instantiate(n::Var{I}, a::Vararg{Any,N}) where {I, N} i = instantiate(n.i, a...) return Var{typeof(i)}(i) end -@inline function instantiate(n::ParameterNode{I}, a...) where {I} +@inline function instantiate(n::ParameterNode{I}, a::Vararg{Any,N}) where {I, N} i = instantiate(n.i, a...) return ParameterNode{typeof(i)}(i) end -@inline function instantiate(n::Node1{F,I}, a...) where {F,I} +@inline function instantiate(n::Node1{F,I}, a::Vararg{Any,N}) where {F,I, N} i = instantiate(n.inner, a...) return Node1{F,typeof(i)}(i) end -@inline function instantiate(n::Node2{F,I1,I2}, a...) where {F,I1,I2} +@inline function instantiate(n::Node2{F,I1,I2}, a::Vararg{Any,N}) where {F,I1,I2, N} i1 = instantiate(n.inner1, a...) i2 = instantiate(n.inner2, a...) return Node2{F,typeof(i1),typeof(i2)}(i1, i2) end -@inline instantiate(n::SumNode, a...) = SumNode(instantiate(n.inners, a...)) -@inline instantiate(n::ProdNode, a...) = ProdNode(instantiate(n.inners, a...)) +@inline instantiate(n::SumNode, a::Vararg{Any,N}) where {N} = SumNode(instantiate(n.inners, a...)) +@inline instantiate(n::ProdNode, a::Vararg{Any,N}) where {N} = ProdNode(instantiate(n.inners, a...)) # `DataIndexed` overrides `getproperty` to keep building access paths, so its # own field has to be read with `getfield`. -@inline instantiate(n::DataIndexed{I,J}, a...) where {I,J} = +@inline instantiate(n::DataIndexed{I,J}, a::Vararg{Any,N}) where {I,J, N} = DataIndexed(instantiate(getfield(n, :inner), a...), J) -@inline instantiate(p::Pair, a...) = instantiate(p.first, a...) => instantiate(p.second, a...) +@inline instantiate(p::Pair, a::Vararg{Any,N}) where {N} = instantiate(p.first, a...) => instantiate(p.second, a...) # An `ArgLeaf` does not survive instantiation: it *becomes* the scalar, leaving # a graph indistinguishable from one built with concrete sizes. -@inline instantiate(n::ArgLeaf, a...) = instantiate(getfield(n, :a), a...) +@inline instantiate(n::ArgLeaf, a::Vararg{Any,N}) where {N} = instantiate(getfield(n, :a), a...) diff --git a/src/nlp.jl b/src/nlp.jl index abf6b760..22ab9d9b 100644 --- a/src/nlp.jl +++ b/src/nlp.jl @@ -579,7 +579,9 @@ is known statically and destructuring stays inferable. # rather than re-derived: it is the float type the core was created with, and # instantiating changes sizes, never the element type. -function instantiate(c::ExaCore{T}, a...) where {T} +# `Vararg{Any,N}`: see the note on the `Tuple` method in argument.jl — forces +# specialization so the field-by-field mapping stays static under `--trim`. +function instantiate(c::ExaCore{T}, a::Vararg{Any,N}) where {T, N} return ExaCore{T}( c.name, c.backend, @@ -613,24 +615,24 @@ function instantiate(c::ExaCore{T}, a...) where {T} ) end -instantiate(v::Variable, a...) = +instantiate(v::Variable, a::Vararg{Any,N}) where {N} = Variable(instantiate(v.size, a...), instantiate(v.length, a...), instantiate(v.offset, a...), v.name, instantiate(v.tag, a...)) -instantiate(p::Parameter, a...) = +instantiate(p::Parameter, a::Vararg{Any,N}) where {N} = Parameter(instantiate(p.size, a...), instantiate(p.length, a...), instantiate(p.offset, a...), instantiate(p.tag, a...)) -instantiate(e::Expression, a...) = +instantiate(e::Expression, a::Vararg{Any,N}) where {N} = Expression(instantiate(e.size, a...), instantiate(e.length, a...), instantiate(e.f, a...), instantiate(e.iter, a...), instantiate(e.tag, a...)) -instantiate(o::Objective, a...) = Objective(instantiate(o.f, a...), instantiate(o.itr, a...)) -instantiate(c::Constraint, a...) = +instantiate(o::Objective, a::Vararg{Any,N}) where {N} = Objective(instantiate(o.f, a...), instantiate(o.itr, a...)) +instantiate(c::Constraint, a::Vararg{Any,N}) where {N} = Constraint(instantiate(c.f, a...), instantiate(c.itr, a...), instantiate(c.offset, a...), instantiate(c.size, a...), instantiate(c.tag, a...)) -instantiate(c::ConstraintAugmentation, a...) = +instantiate(c::ConstraintAugmentation, a::Vararg{Any,N}) where {N} = ConstraintAugmentation(instantiate(c.f, a...), instantiate(c.itr, a...), instantiate(c.oa, a...), instantiate(c.dims, a...), instantiate(c.tag, a...)) -instantiate(f::SIMDFunction, a...) = +instantiate(f::SIMDFunction, a::Vararg{Any,N}) where {N} = SIMDFunction(instantiate(f.f, a...), f.comp1, f.comp2, instantiate(f.o0, a...), instantiate(f.o1, a...), instantiate(f.o2, a...), f.o1step, f.o2step) @@ -1048,11 +1050,11 @@ end # `start = (f(i) for i in 1:arg.N)` — the body is untouched (it runs per element # once the iterator is concrete); only what it iterates is deferred. -@inline instantiate(g::Base.Generator, a...) = Base.Generator(g.f, instantiate(g.iter, a...)) +@inline instantiate(g::Base.Generator, a::Vararg{Any,N}) where {N} = Base.Generator(g.f, instantiate(g.iter, a...)) # A product iterator is arg-dependent when any of the ranges it crosses is. @inline _anyarg(p::Base.Iterators.ProductIterator, xs...) = _anyarg(p.iterators..., xs...) -@inline instantiate(p::Base.Iterators.ProductIterator, a...) = +@inline instantiate(p::Base.Iterators.ProductIterator, a::Vararg{Any,N}) where {N} = Base.Iterators.product(instantiate(p.iterators, a...)...) # `append!` mutates its accumulator and returns it. That is exactly right while