diff --git a/CHANGELOG.md b/CHANGELOG.md index 493bf8036f8..849f5e00dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Refactored rendering in GLMakie to go through a series of steps abstracted by a render pipeline. This allows rendering to be adjusted from outside and should simplify introducing more post-processing options in the future. [#4689](https://github.com/MakieOrg/Makie.jl/pull/4689) - Fixed an issue with transformations not propagating to child plots when their spaces only match indirectly. [#4723](https://github.com/MakieOrg/Makie.jl/pull/4723) - Added a tutorial on creating an inset plot [#4697](https://github.com/MakieOrg/Makie.jl/pull/4697) - Enhanced Pattern support: Added general CairoMakie implementation, improved quality, added anchoring, added support in band, density, added tests & fixed various bugs and inconsistencies. [#4715](https://github.com/MakieOrg/Makie.jl/pull/4715) diff --git a/GLMakie/assets/shader/fragment_output.frag b/GLMakie/assets/shader/fragment_output.frag index 837d5ccc3ec..7287d48781c 100644 --- a/GLMakie/assets/shader/fragment_output.frag +++ b/GLMakie/assets/shader/fragment_output.frag @@ -1,16 +1,20 @@ {{GLSL_VERSION}} +{{TARGET_STAGE}} + layout(location=0) out vec4 fragment_color; layout(location=1) out uvec2 fragment_groupid; -{{buffers}} -// resolves to: -// // if transparency == true -// layout(location=2) out float coverage; -// // if transparency == false && ssao = true -// layout(location=2) out vec3 fragment_position; -// layout(location=3) out vec3 fragment_normal_occlusion; +#ifdef SSAO_TARGET +layout(location=2) out vec3 fragment_position; +layout(location=3) out vec3 fragment_normal; +#endif + +#ifdef OIT_TARGET +layout(location=2) out float coverage; +uniform float oit_scale; +#endif in vec3 o_view_pos; in vec3 o_view_normal; @@ -22,20 +26,20 @@ void write2framebuffer(vec4 color, uvec2 id){ // For plot/sprite picking fragment_groupid = id; - {{buffer_writes}} - // resolves to: - - // if transparency == true - // float weight = color.a * max(0.01, $scale * pow((1 - gl_FragCoord.z), 3)); - // coverage = 1.0 - clamp(color.a, 0.0, 1.0); - // fragment_color.rgb = weight * color.rgb; - // fragment_color.a = weight; - - // // if transparency == false && ssao = true - // fragment_color = color; - // fragment_position = o_view_pos; - // fragment_normal_occlusion.xyz = o_view_normal; - - // // else - // fragment_color = color; +#ifdef DEFAULT_TARGET + fragment_color = color; +#endif + +#ifdef SSAO_TARGET + fragment_color = color; + fragment_position.xyz = o_view_pos; + fragment_normal.xyz = o_view_normal; +#endif + +#ifdef OIT_TARGET + float weight = color.a * max(0.01, oit_scale * pow((1 - gl_FragCoord.z), 3)); + coverage = 1.0 - clamp(color.a, 0.0, 1.0); + fragment_color.rgb = weight * color.rgb; + fragment_color.a = weight; +#endif } diff --git a/GLMakie/assets/shader/postprocessing/OIT_blend.frag b/GLMakie/assets/shader/postprocessing/OIT_blend.frag index 63d4343fb0c..a68693e301e 100644 --- a/GLMakie/assets/shader/postprocessing/OIT_blend.frag +++ b/GLMakie/assets/shader/postprocessing/OIT_blend.frag @@ -6,16 +6,16 @@ in vec2 frag_uv; // contains sum_i C_i * weight(depth_i, alpha_i) -uniform sampler2D sum_color; +uniform sampler2D color_sum_buffer; // contains pod_i (1 - alpha_i) -uniform sampler2D prod_alpha; +uniform sampler2D transmittance_buffer; out vec4 fragment_color; void main(void) { - vec4 summed_color_weight = texture(sum_color, frag_uv); - float transmittance = texture(prod_alpha, frag_uv).r; + vec4 summed_color_weight = texture(color_sum_buffer, frag_uv); + float transmittance = texture(transmittance_buffer, frag_uv).r; vec3 weighted_transparent = summed_color_weight.rgb / max(summed_color_weight.a, 0.00001); vec3 full_weighted_transparent = weighted_transparent * (1 - transmittance); diff --git a/GLMakie/assets/shader/postprocessing/SSAO.frag b/GLMakie/assets/shader/postprocessing/SSAO.frag index d107a4913f2..9b3064c38ab 100644 --- a/GLMakie/assets/shader/postprocessing/SSAO.frag +++ b/GLMakie/assets/shader/postprocessing/SSAO.frag @@ -2,7 +2,7 @@ // SSAO uniform sampler2D position_buffer; -uniform sampler2D normal_occlusion_buffer; +uniform sampler2D normal_buffer; uniform sampler2D noise; uniform vec3 kernel[{{N_samples}}]; uniform vec2 noise_scale; @@ -16,18 +16,18 @@ uniform float radius; in vec2 frag_uv; // occlusion.xyz is a normal vector, occlusion.w the occlusion value -out vec4 o_normal_occlusion; +out float o_occlusion; void main(void) { vec3 view_pos = texture(position_buffer, frag_uv).xyz; - vec3 normal = texture(normal_occlusion_buffer, frag_uv).xyz; + vec3 normal = texture(normal_buffer, frag_uv).xyz; // The normal buffer gets cleared every frame. (also position, color etc) // If normal == vec3(1) then there is no geometry at this fragment. // Therefore skip SSAO calculation - if (normal != vec3(1)) { + if (normal != vec3(0)) { vec3 rand_vec = vec3(texture(noise, frag_uv * noise_scale).xy, 0.0); vec3 tangent = normalize(rand_vec - normal * dot(rand_vec, normal)); vec3 bitangent = cross(normal, tangent); @@ -78,13 +78,14 @@ void main(void) sample_frag_pos.xyz = sample_frag_pos.xyz * 0.5 + 0.5; sample_frag_pos.xy += (frag_uv - 0.5) * clip_pos_w / sample_clip_pos_w; + if (texture(normal_buffer, sample_frag_pos.xy).xyz == vec3(0)) continue; float sample_depth = texture(position_buffer, sample_frag_pos.xy).z; float range_check = smoothstep(0.0, 1.0, radius / abs(view_pos.z - sample_depth)); occlusion += (sample_depth >= sample_view_offset.z + view_pos.z + bias ? 1.0 : 0.0) * range_check; } - o_normal_occlusion.w = occlusion / {{N_samples}}; + o_occlusion = occlusion / {{N_samples}}; } else { - o_normal_occlusion.w = 0.0; + o_occlusion = 0.0; } } diff --git a/GLMakie/assets/shader/postprocessing/SSAO_blur.frag b/GLMakie/assets/shader/postprocessing/SSAO_blur.frag index 6021a7379e9..660270fc4c1 100644 --- a/GLMakie/assets/shader/postprocessing/SSAO_blur.frag +++ b/GLMakie/assets/shader/postprocessing/SSAO_blur.frag @@ -1,9 +1,9 @@ {{GLSL_VERSION}} // occlusion.w is the occlusion value -uniform sampler2D normal_occlusion; -uniform sampler2D color_texture; -uniform usampler2D ids; +uniform sampler2D occlusion_buffer; +uniform sampler2D color_buffer; +uniform usampler2D objectid_buffer; uniform vec2 inv_texel_size; // Settings/Attributes uniform int blur_range; @@ -14,9 +14,9 @@ out vec4 fragment_color; void main(void) { // occlusion blur - uvec2 id0 = texture(ids, frag_uv).xy; + uvec2 id0 = texture(objectid_buffer, frag_uv).xy; if (id0.x == uint(0)){ - fragment_color = texture(color_texture, frag_uv); + fragment_color = texture(color_buffer, frag_uv); // fragment_color = vec4(1,0,1,1); // show discarded return; } @@ -24,6 +24,7 @@ void main(void) float blurred_occlusion = 0.0; float weight = 0; + // TODO: Could use :linear filtering and/ro mipmap to simplify this maybe? for (int x = -blur_range; x <= blur_range; ++x){ for (int y = -blur_range; y <= blur_range; ++y){ vec2 offset = vec2(float(x), float(y)) * inv_texel_size; @@ -31,14 +32,14 @@ void main(void) // Without this, a high (low) occlusion from one object can bleed // into the low (high) occlusion of another, giving an unwanted // shine effect. - uvec2 id = texture(ids, frag_uv + offset).xy; + uvec2 id = texture(objectid_buffer, frag_uv + offset).xy; float valid = float(id == id0); - blurred_occlusion += valid * texture(normal_occlusion, frag_uv + offset).w; + blurred_occlusion += valid * texture(occlusion_buffer, frag_uv + offset).x; weight += valid; } } blurred_occlusion = 1.0 - blurred_occlusion / weight; - fragment_color = texture(color_texture, frag_uv) * blurred_occlusion; + fragment_color = texture(color_buffer, frag_uv) * blurred_occlusion; // Display occlusion instead: // fragment_color = vec4(vec3(blurred_occlusion), 1.0); } diff --git a/GLMakie/assets/shader/postprocessing/copy.frag b/GLMakie/assets/shader/postprocessing/copy.frag index 9ec63ec90af..e3af207bbe2 100644 --- a/GLMakie/assets/shader/postprocessing/copy.frag +++ b/GLMakie/assets/shader/postprocessing/copy.frag @@ -1,12 +1,12 @@ {{GLSL_VERSION}} -uniform sampler2D color_texture; +uniform sampler2D color_buffer; in vec2 frag_uv; out vec4 fragment_color; void main(void) { - vec4 color = texture(color_texture, frag_uv); + vec4 color = texture(color_buffer, frag_uv); fragment_color.rgb = color.rgb; fragment_color.a = 1.0; } diff --git a/GLMakie/assets/shader/postprocessing/fxaa.frag b/GLMakie/assets/shader/postprocessing/fxaa.frag index 56d4eb678b9..87e9e2b555c 100644 --- a/GLMakie/assets/shader/postprocessing/fxaa.frag +++ b/GLMakie/assets/shader/postprocessing/fxaa.frag @@ -1019,7 +1019,7 @@ FxaaFloat4 FxaaPixelShader( precision highp float; -uniform sampler2D color_texture; +uniform sampler2D color_luma_buffer; uniform vec2 RCPFrame; in vec2 frag_uv; @@ -1028,23 +1028,24 @@ out vec4 fragment_color; void main(void) { - // fragment_color = texture(color_texture, frag_uv); + // fragment_color = texture(color_luma_buffer, frag_uv); + // fragment_color.rgb = vec3(texture(color_luma_buffer, frag_uv).a); fragment_color.rgb = FxaaPixelShader( frag_uv, - FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsolePosPos, - color_texture, // FxaaTex tex, - color_texture, // FxaaTex fxaaConsole360TexExpBiasNegOne, - color_texture, // FxaaTex fxaaConsole360TexExpBiasNegTwo, - RCPFrame, // FxaaFloat2 fxaaQualityRcpFrame, - FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsoleRcpFrameOpt, - FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsoleRcpFrameOpt2, - FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsole360RcpFrameOpt2, + FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsolePosPos, + color_luma_buffer, // FxaaTex tex, + color_luma_buffer, // FxaaTex fxaaConsole360TexExpBiasNegOne, + color_luma_buffer, // FxaaTex fxaaConsole360TexExpBiasNegTwo, + RCPFrame, // FxaaFloat2 fxaaQualityRcpFrame, + FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsoleRcpFrameOpt, + FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsoleRcpFrameOpt2, + FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f), // FxaaFloat4 fxaaConsole360RcpFrameOpt2, 0.75f, // FxaaFloat fxaaQualitySubpix, - 0.166f, // FxaaFloat fxaaQualityEdgeThreshold, - 0.0833f, // FxaaFloat fxaaQualityEdgeThresholdMin, - 0.0f, // FxaaFloat fxaaConsoleEdgeSharpness, - 0.0f, // FxaaFloat fxaaConsoleEdgeThreshold, - 0.0f, // FxaaFloat fxaaConsoleEdgeThresholdMin, + 0.166f, // FxaaFloat fxaaQualityEdgeThreshold, + 0.0833f, // FxaaFloat fxaaQualityEdgeThresholdMin, + 0.0f, // FxaaFloat fxaaConsoleEdgeSharpness, + 0.0f, // FxaaFloat fxaaConsoleEdgeThreshold, + 0.0f, // FxaaFloat fxaaConsoleEdgeThresholdMin, FxaaFloat4(0.0f, 0.0f, 0.0f, 0.0f) // FxaaFloat fxaaConsole360ConstDir, ).rgb; } diff --git a/GLMakie/assets/shader/postprocessing/postprocess.frag b/GLMakie/assets/shader/postprocessing/postprocess.frag index ff5e2bde304..3dc9b0c9ca0 100644 --- a/GLMakie/assets/shader/postprocessing/postprocess.frag +++ b/GLMakie/assets/shader/postprocessing/postprocess.frag @@ -1,9 +1,11 @@ {{GLSL_VERSION}} +{{FILTER_IN_SHADER}} + in vec2 frag_uv; -uniform sampler2D color_texture; -uniform usampler2D object_ids; +uniform sampler2D color_buffer; +uniform usampler2D objectid_buffer; layout(location=0) out vec4 fragment_color; @@ -21,15 +23,17 @@ bool unpack_bool(uint id) { void main(void) { - vec4 color = texture(color_texture, frag_uv).rgba; + vec4 color = texture(color_buffer, frag_uv).rgba; if(color.a <= 0){ discard; } - uint id = texture(object_ids, frag_uv).x; + uint id = texture(objectid_buffer, frag_uv).x; // do tonemappings //opaque = linear_tone_mapping(color.rgb, 1.8); // linear color output fragment_color.rgb = color.rgb; + +#ifdef FILTER_IN_SHADER // we store fxaa = true/false in highbit of the object id if (unpack_bool(id)) { fragment_color.a = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // compute luma @@ -37,4 +41,7 @@ void main(void) // we disable fxaa by setting luma to 1 fragment_color.a = 1.0; } +#else + fragment_color.a = dot(color.rgb, vec3(0.299, 0.587, 0.114)); // compute luma +#endif } diff --git a/GLMakie/src/GLAbstraction/GLAbstraction.jl b/GLMakie/src/GLAbstraction/GLAbstraction.jl index c5aaaef4004..3664e80bd38 100644 --- a/GLMakie/src/GLAbstraction/GLAbstraction.jl +++ b/GLMakie/src/GLAbstraction/GLAbstraction.jl @@ -126,4 +126,12 @@ export getUniformsInfo export getProgramInfo export getAttributesInfo +include("GLFramebuffer.jl") +export GLRenderbuffer +export GLFramebuffer +export attach_colorbuffer, attach_depthbuffer, attach_stencilbuffer, attach_depthstencilbuffer +export get_attachment, get_buffer +export set_draw_buffers +export check_framebuffer + end # module diff --git a/GLMakie/src/GLAbstraction/GLFramebuffer.jl b/GLMakie/src/GLAbstraction/GLFramebuffer.jl new file mode 100644 index 00000000000..57e04bd25a1 --- /dev/null +++ b/GLMakie/src/GLAbstraction/GLFramebuffer.jl @@ -0,0 +1,241 @@ +# TODO: Add RenderBuffer, resize!() with renderbuffer (recreate?) +mutable struct GLFramebuffer + id::GLuint + size::NTuple{2, Int} + + context::GLContext + + name2idx::Dict{Symbol, Int} + attachments::Vector{GLenum} + buffers::Vector{Texture} + counter::UInt32 # for color attachments + + function GLFramebuffer(context, size::NTuple{2, Int}) + require_context(context) + + # Create framebuffer + id = glGenFramebuffers() + glBindFramebuffer(GL_FRAMEBUFFER, id) + + obj = new( + id, size, context, + Dict{Symbol, Int}(), GLenum[], Texture[], UInt32(0) + ) + finalizer(verify_free, obj) + + return obj + end +end + +function bind(fb::GLFramebuffer, target = fb.id) + if fb.id == 0 + error("Binding freed GLFramebuffer") + end + glBindFramebuffer(GL_FRAMEBUFFER, target) + return +end + +# This allows you to just call `set_draw_buffers(framebuffer)` with a framebuffer +# dedicated to a specific (type of) draw call, i.e. one that only contains +# attachments matching the outputs of the draw call. But it restricts how you +# can modify the framebuffer a bit (1) +""" + draw_buffers(fb::GLFrameBuffer[, N::Int]) + +Activates the first N color buffers attached to the given GLFramebuffer. If N +is not given all color attachments are activated. +""" +function set_draw_buffers(fb::GLFramebuffer, N::Integer = fb.counter) + require_context(fb.context) + bind(fb) + glDrawBuffers(N, fb.attachments) + return +end +function set_draw_buffers(fb::GLFramebuffer, key::Symbol) + require_context(fb.context) + bind(fb) + glDrawBuffer(get_attachment(fb, key)) + return +end +function set_draw_buffers(fb::GLFramebuffer, keys::Symbol...) + require_context(fb.context) + bind(fb) + glDrawBuffer(get_attachment.(Ref(fb), keys)) + return +end + +function unsafe_free(x::GLFramebuffer) + id = Ref(x.id) + glDeleteFramebuffers(1, id) + x.id = 0 + return +end + +Base.size(fb::GLFramebuffer) = fb.size +Base.haskey(fb::GLFramebuffer, key::Symbol) = haskey(fb.name2idx, key) + +function Base.resize!(fb::GLFramebuffer, w::Int, h::Int) + (w > 0 && h > 0 && (w, h) != size(fb)) || return + for buffer in fb.buffers + resize_nocopy!(buffer, (w, h)) + end + fb.size = (w, h) + return +end + +function get_next_colorbuffer_attachment(fb::GLFramebuffer) + if fb.counter >= 15 + error("The framebuffer has exhausted its maximum number of color attachments.") + end + # (1) Disallows random deletion as that can change the order of buffers. As + # a result we don't have to worry about counter pointing to an existing item here. + attachment = GL_COLOR_ATTACHMENT0 + fb.counter + fb.counter += 1 + return (fb.counter, attachment) +end + +function attach_colorbuffer(fb::GLFramebuffer, key::Symbol, buffer) + return attach(fb, key, buffer, get_next_colorbuffer_attachment(fb)...) +end +function attach_depthbuffer(fb::GLFramebuffer, key::Symbol, buffer) + return attach(fb, key, buffer, length(fb.attachments) + 1, GL_DEPTH_ATTACHMENT) +end +function attach_stencilbuffer(fb::GLFramebuffer, key::Symbol, buffer) + return attach(fb, key, buffer, length(fb.attachments) + 1, GL_STENCIL_ATTACHMENT) +end +function attach_depthstencilbuffer(fb::GLFramebuffer, key::Symbol, buffer) + return attach(fb, key, buffer, length(fb.attachments) + 1, GL_DEPTH_STENCIL_ATTACHMENT) +end + +function attach(fb::GLFramebuffer, key::Symbol, buffer, idx::Integer, attachment::GLenum) + require_context(fb.context) + haskey(fb, key) && error("Cannot attach " * string(key) * " to Framebuffer because it is already set.") + if attachment in fb.attachments + if attachment == GL_DEPTH_ATTACHMENT + type = "depth" + elseif attachment == GL_STENCIL_ATTACHMENT + type = "stencil" + elseif attachment == GL_DEPTH_STENCIL_ATTACHMENT + type = "depth-stencil" + else + type = "color" + end + error("Cannot attach " * string(key) * " as a " * type * " attachment as it is already attached.") + end + + try + bind(fb) + gl_attach(buffer, attachment) + check_framebuffer() + catch e + if GL_COLOR_ATTACHMENT0 <= attachment <= GL_COLOR_ATTACHMENT15 + # If we failed to attach correctly we should probably overwrite + # the attachment next time we try? + fb.counter -= 1 + end + @info "$key -> $(GLENUM(attachment).name) failed with framebuffer id = $(fb.id)" + rethrow(e) + end + # (1) requires us to keep depth/stenctil/depth_stencil at end so that the first + # fb.counter buffers are usable draw buffers. + for (k, v) in fb.name2idx + fb.name2idx[k] = ifelse(v < idx, v, v+1) + end + fb.name2idx[key] = idx + insert!(fb.attachments, idx, attachment) + insert!(fb.buffers, idx, buffer) + return attachment +end + +function gl_attach(t::Texture{T, 2}, attachment::GLenum) where {T} + glFramebufferTexture2D(GL_FRAMEBUFFER, attachment, GL_TEXTURE_2D, t.id, 0) +end +function gl_attach(buffer::RenderBuffer, attachment::GLenum) + glFramebufferRenderbuffer(GL_FRAMEBUFFER, attachment, GL_RENDERBUFFER, buffer) +end + +# (1) disallows random deletion because that could disrupt the order of draw buffers -> no delete!() +function pop_colorbuffer!(fb::GLFramebuffer) + # (1) depth, stencil are attached after the last colorbuffer, after fb.counter. + # Need to fix their indices after deletion + attachment = fb.attachments[fb.counter] + glFramebufferTexture2D(GL_FRAMEBUFFER, attachment, GL_TEXTURE_2D, 0, 0) + deleteat!(fb.attachments, fb.counter) + deleteat!(fb.buffers, fb.counter) + key = :unknown + for (k, v) in fb.name2idx + (v == fb.counter) && (key = k) + (v > fb.counter) && (fb.name2idx[k] = v-1) + end + delete!(fb.name2idx, key) + fb.counter -= 1 + return fb +end + +get_attachment(fb::GLFramebuffer, key::Symbol) = fb.attachments[fb.name2idx[key]] +get_buffer(fb::GLFramebuffer, key::Symbol) = fb.buffers[fb.name2idx[key]] + +function enum_to_error(s) + s == GL_FRAMEBUFFER_COMPLETE && return + s == GL_FRAMEBUFFER_UNDEFINED && + error("GL_FRAMEBUFFER_UNDEFINED: The specified framebuffer is the default read or draw framebuffer, but the default framebuffer does not exist.") + s == GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT && + error("GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT: At least one of the framebuffer attachment points is incomplete.") + s == GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT && + error("GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: The framebuffer does not have at least one image attached to it.") + s == GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER && + error("GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER: The value of GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE is GL_NONE for any color attachment point(s) specified by GL_DRAW_BUFFERi.") + s == GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER && + error("GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER: GL_READ_BUFFER is not GL_NONE and the value of GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE is GL_NONE for the color attachment point specified by GL_READ_BUFFER.") + s == GL_FRAMEBUFFER_UNSUPPORTED && + error("GL_FRAMEBUFFER_UNSUPPORTED: The combination of internal formats of the attached images violates a driver implementation-dependent set of restrictions. Check your OpenGL driver!") + s == GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE && + error("GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE: The value of GL_RENDERBUFFER_SAMPLES is not the same for all attached renderbuffers; +if the value of GL_TEXTURE_SAMPLES is not the same for all attached textures; or, if the attached images consist of a mix of renderbuffers and textures, + the value of GL_RENDERBUFFER_SAMPLES does not match the value of GL_TEXTURE_SAMPLES. + GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE is also returned if the value of GL_TEXTURE_FIXED_SAMPLE_LOCATIONS is not consistent across all attached textures; + or, if the attached images include a mix of renderbuffers and textures, the value of GL_TEXTURE_FIXED_SAMPLE_LOCATIONS is not set to GL_TRUE for all attached textures.") + s == GL_FRAMEBUFFER_INCOMPLETE_LAYER_TARGETS && + error("GL_FRAMEBUFFER_INCOMPLETE_LAYER_TARGETS: Any framebuffer attachment is layered, and any populated attachment is not layered, or if all populated color attachments are not from textures of the same target.") + return error("Unknown framebuffer completion error code: $s") +end + +function check_framebuffer() + status = glCheckFramebufferStatus(GL_FRAMEBUFFER) + return enum_to_error(status) +end + +function Base.show(io::IO, fb::GLFramebuffer) + X, Y = fb.size + print(io, "$X×$Y GLFrameBuffer(:") + join(io, string.(keys(fb.name2idx)), ", :") + print(io, ") with id ", fb.id) +end + +function attachment_enum_to_string(x::GLenum) + x == GL_DEPTH_ATTACHMENT && return "GL_DEPTH_ATTACHMENT" + x == GL_STENCIL_ATTACHMENT && return "GL_STENCIL_ATTACHMENT" + x == GL_DEPTH_STENCIL_ATTACHMENT && return "GL_DEPTH_STENCIL_ATTACHMENT" + x == GL_STENCIL_ATTACHMENT && return "GL_STENCIL_ATTACHMENT" + i = Int(x - GL_COLOR_ATTACHMENT0) + return "GL_COLOR_ATTACHMENT$i" +end + +function Base.show(io::IO, ::MIME"text/plain", fb::GLFramebuffer) + X, Y = fb.size + print(io, "$X×$Y GLFrameBuffer() with id ", fb.id) + + ks = collect(keys(fb.name2idx)) + sort!(ks, by = k -> fb.name2idx[k]) + key_strings = [":$k" for k in ks] + key_pad = mapreduce(length, max, key_strings) + key_strings = rpad.(key_strings, key_pad) + + attachments = attachment_enum_to_string.(fb.attachments) + attachment_pad = mapreduce(length, max, attachments) + attachments = rpad.(attachments, attachment_pad) + + for (key, attachment, buffer) in zip(key_strings, attachments, fb.buffers) + print(io, "\n ", key, " => ", attachment, " ::", typeof(buffer)) + end +end \ No newline at end of file diff --git a/GLMakie/src/GLAbstraction/GLRender.jl b/GLMakie/src/GLAbstraction/GLRender.jl index 9a60734a454..4ae81d1645c 100644 --- a/GLMakie/src/GLAbstraction/GLRender.jl +++ b/GLMakie/src/GLAbstraction/GLRender.jl @@ -176,8 +176,8 @@ end # Generic render functions ##### function enabletransparency() + glDisable(GL_BLEND) glEnablei(GL_BLEND, 0) - glDisablei(GL_BLEND, 1) # This does: # target.rgb = source.a * source.rgb + (1 - source.a) * target.rgb # target.a = 0 * source.a + 1 * target.a diff --git a/GLMakie/src/GLAbstraction/GLRenderObject.jl b/GLMakie/src/GLAbstraction/GLRenderObject.jl index e7f333f3d87..01245a8838c 100644 --- a/GLMakie/src/GLAbstraction/GLRenderObject.jl +++ b/GLMakie/src/GLAbstraction/GLRenderObject.jl @@ -1,5 +1,5 @@ function Base.show(io::IO, obj::RenderObject) - println(io, "RenderObject with ID: ", obj.id) + print(io, "RenderObject with ID: ", obj.id) end Base.getindex(obj::RenderObject, symbol::Symbol) = obj.uniforms[symbol] @@ -33,30 +33,6 @@ function (sp::StandardPrerender)() # Disable cullface for now, until all rendering code is corrected! glDisable(GL_CULL_FACE) # glCullFace(GL_BACK) - - if sp.transparency[] - # disable depth buffer writing - glDepthMask(GL_FALSE) - - # Blending - glEnable(GL_BLEND) - glBlendEquation(GL_FUNC_ADD) - - # buffer 0 contains weight * color.rgba, should do sum - # destination <- 1 * source + 1 * destination - glBlendFunci(0, GL_ONE, GL_ONE) - - # buffer 1 is objectid, do nothing - glDisablei(GL_BLEND, 1) - - # buffer 2 is color.a, should do product - # destination <- 0 * source + (source) * destination - glBlendFunci(2, GL_ZERO, GL_SRC_COLOR) - - else - glDepthMask(GL_TRUE) - enabletransparency() - end end struct StandardPostrender diff --git a/GLMakie/src/GLAbstraction/GLTexture.jl b/GLMakie/src/GLAbstraction/GLTexture.jl index 67415d89c97..d30500030d0 100644 --- a/GLMakie/src/GLAbstraction/GLTexture.jl +++ b/GLMakie/src/GLAbstraction/GLTexture.jl @@ -4,6 +4,7 @@ struct TextureParameters{NDim} repeat ::NTuple{NDim, Symbol} anisotropic::Float32 swizzle_mask::Vector{GLenum} + mipmap::Bool end abstract type OpenglTexture{T, NDIM} <: GPUArray{T, NDIM} end @@ -92,7 +93,7 @@ Makie.@noconstprop function Texture( mipmap = false, parameters... # rest should be texture parameters ) where {T, NDim} - texparams = TextureParameters(T, NDim; parameters...) + texparams = TextureParameters(T, NDim; mipmap, parameters...) id = glGenTextures() glBindTexture(texturetype, id) set_packing_alignment(data) @@ -231,7 +232,7 @@ Creates a texture from an Image # AbstractArrays default show assumes `getindex`. Try to catch all calls # https://discourse.julialang.org/t/overload-show-for-array-of-custom-types/9589 -Base.show(io::IO, t::Texture) = show(IOContext(io), MIME"text/plain"(), t) +Base.show(io::IO, t::Texture{T, D}) where {T, D} = print(io, "Texture{$T, $D}(ID: $(t.id), Size: $(size(t)))") function Base.show(io::IOContext, mime::MIME"text/plain", t::Texture{T,D}) where {T,D} if get(io, :compact, false) @@ -523,7 +524,8 @@ function TextureParameters(T, NDim; x_repeat = :clamp_to_edge, #wrap_s y_repeat = x_repeat, #wrap_t z_repeat = x_repeat, #wrap_r - anisotropic = 1f0 + anisotropic = 1f0, + mipmap = false ) T <: Integer && (minfilter === :linear || magfilter === :linear) && error("Wrong Texture Parameter: Integer texture can't interpolate. Try :nearest") repeat = (x_repeat, y_repeat, z_repeat) @@ -536,7 +538,7 @@ function TextureParameters(T, NDim; end TextureParameters( minfilter, magfilter, ntuple(i->repeat[i], NDim), - anisotropic, swizzle_mask + anisotropic, swizzle_mask, mipmap ) end function TextureParameters(t::Texture{T, NDim}; kw_args...) where {T, NDim} diff --git a/GLMakie/src/GLAbstraction/GLTypes.jl b/GLMakie/src/GLAbstraction/GLTypes.jl index f50b16f29db..3296444c8bb 100644 --- a/GLMakie/src/GLAbstraction/GLTypes.jl +++ b/GLMakie/src/GLAbstraction/GLTypes.jl @@ -108,26 +108,6 @@ function resize!(rb::RenderBuffer, newsize::AbstractArray) glRenderbufferStorage(GL_RENDERBUFFER, rb.format, newsize...) end -struct FrameBuffer{T} - id::GLuint - attachments::Vector{Any} - context::GLContext - function FrameBuffer{T}(dimensions::Observable) where T - fb = glGenFramebuffers() - glBindFramebuffer(GL_FRAMEBUFFER, fb) - new(id, attachments, current_context()) - end -end - -function resize!(fbo::FrameBuffer, newsize::AbstractArray) - if length(newsize) != 2 - error("FrameBuffer needs to be 2 dimensional. Dimension found: ", newsize) - end - for elem in fbo.attachments - resize!(elem) - end -end - ######################################################################################## # OpenGL Arrays diff --git a/GLMakie/src/GLMakie.jl b/GLMakie/src/GLMakie.jl index 40642e1c077..286619b5ea8 100644 --- a/GLMakie/src/GLMakie.jl +++ b/GLMakie/src/GLMakie.jl @@ -48,13 +48,38 @@ struct ShaderSource name::String end -function ShaderSource(path) +""" + ShaderSource(filepath::String) + +Loads the source code from the given filepath. The shader type is derived from +the file extension (.vert, .frag, .geom or .comp). +""" +function ShaderSource(path::String) typ = GLAbstraction.shadertype(splitext(path)[2]) source = read(path, String) name = String(path) return ShaderSource(typ, source, name) end +const shader_counter = Ref(0) +function next_shader_num() + shader_counter[] = shader_counter[] + 1 + return shader_counter[] +end + +""" + ShaderSource(code::String, type::Symbol[, name]) + +Bundles the given source code with a shader type for later shader compilation. +`type` can be :vert, :frag, :geom or :comp. + +Use `loadshader(filename)` to load a cached shader from GLMakie's assets folder. +""" +function ShaderSource(source::String, type::Symbol, name = "inline_shader$(next_shader_num())") + type2gltype = (comp = GL_COMPUTE_SHADER, vert = GL_VERTEX_SHADER, frag = GL_FRAGMENT_SHADER, geom = GL_GEOMETRY_SHADER) + return ShaderSource(type2gltype[type], source, name) +end + const SHADER_DIR = normpath(joinpath(@__DIR__, "..", "assets", "shader")) const LOADED_SHADERS = Dict{String, ShaderSource}() const WARN_ON_LOAD = Ref(false) diff --git a/GLMakie/src/gl_backend.jl b/GLMakie/src/gl_backend.jl index cfb260552d9..a4081847d06 100644 --- a/GLMakie/src/gl_backend.jl +++ b/GLMakie/src/gl_backend.jl @@ -73,6 +73,7 @@ end include("glwindow.jl") include("postprocessing.jl") include("screen.jl") +include("render_pipeline.jl") include("glshaders/visualize_interface.jl") include("glshaders/lines.jl") include("glshaders/image_like.jl") diff --git a/GLMakie/src/glshaders/image_like.jl b/GLMakie/src/glshaders/image_like.jl index df862391097..5302280d074 100644 --- a/GLMakie/src/glshaders/image_like.jl +++ b/GLMakie/src/glshaders/image_like.jl @@ -43,10 +43,7 @@ function draw_heatmap(screen, data::Dict) shader = GLVisualizeShader( screen, "fragment_output.frag", "heatmap.vert", "heatmap.frag", - view = Dict( - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) - ) + view = Dict("TARGET_STAGE" => target_stage(screen, data)) ) fxaa = false end @@ -85,8 +82,7 @@ function draw_volume(screen, main::VolumeTypes, data::Dict) "depth_default" => vol_depth_default(to_value(enable_depth)), "depth_main" => vol_depth_main(to_value(enable_depth)), "depth_write" => vol_depth_write(to_value(enable_depth)), - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) prerender = VolumePrerender(data[:transparency], data[:overdraw]) diff --git a/GLMakie/src/glshaders/lines.jl b/GLMakie/src/glshaders/lines.jl index 81bc6ec8141..76eabe0b98d 100644 --- a/GLMakie/src/glshaders/lines.jl +++ b/GLMakie/src/glshaders/lines.jl @@ -140,8 +140,7 @@ function draw_lines(screen, position::Union{VectorTypes{T}, MatTypes{T}}, data:: screen, "fragment_output.frag", "lines.vert", "lines.geom", "lines.frag", view = Dict( - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)), + "TARGET_STAGE" => target_stage(screen, data), "define_fast_path" => to_value(fast) ? "#define FAST_PATH" : "", "stripped_color_type" => color_type ) @@ -187,8 +186,7 @@ function draw_linesegments(screen, positions::VectorTypes{T}, data::Dict) where "fragment_output.frag", "line_segment.vert", "line_segment.geom", "lines.frag", view = Dict( - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)), + "TARGET_STAGE" => target_stage(screen, data), "stripped_color_type" => color_type ) ) diff --git a/GLMakie/src/glshaders/mesh.jl b/GLMakie/src/glshaders/mesh.jl index 3cd14d5f7a3..ac4bf28cc76 100644 --- a/GLMakie/src/glshaders/mesh.jl +++ b/GLMakie/src/glshaders/mesh.jl @@ -62,8 +62,7 @@ function draw_mesh(screen, data::Dict) "picking_mode" => to_value(get(data, :picking_mode, "")), "MAX_LIGHTS" => "#define MAX_LIGHTS $(screen.config.max_lights)", "MAX_LIGHT_PARAMETERS" => "#define MAX_LIGHT_PARAMETERS $(screen.config.max_light_parameters)", - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) end diff --git a/GLMakie/src/glshaders/particles.jl b/GLMakie/src/glshaders/particles.jl index 36fd98e5131..c314832f9ea 100644 --- a/GLMakie/src/glshaders/particles.jl +++ b/GLMakie/src/glshaders/particles.jl @@ -120,8 +120,7 @@ function draw_mesh_particle(screen, p, data) "shading" => light_calc(shading), "MAX_LIGHTS" => "#define MAX_LIGHTS $(screen.config.max_lights)", "MAX_LIGHT_PARAMETERS" => "#define MAX_LIGHT_PARAMETERS $(screen.config.max_light_parameters)", - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) end @@ -149,10 +148,7 @@ function draw_pixel_scatter(screen, position::VectorTypes, data::Dict) shader = GLVisualizeShader( screen, "fragment_output.frag", "dots.vert", "dots.frag", - view = Dict( - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) - ) + view = Dict("TARGET_STAGE" => target_stage(screen, data)) ) gl_primitive = GL_POINTS end @@ -271,8 +267,7 @@ function draw_scatter(screen, (marker, position), data) "sprites.vert", "distance_shape.frag", view = Dict( "position_calc" => position_calc(position, nothing, nothing, nothing, GLBuffer), - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) scale_primitive = true diff --git a/GLMakie/src/glshaders/surface.jl b/GLMakie/src/glshaders/surface.jl index 384c3e246ae..8f8b621ac9a 100644 --- a/GLMakie/src/glshaders/surface.jl +++ b/GLMakie/src/glshaders/surface.jl @@ -156,8 +156,7 @@ function draw_surface(screen, main, data::Dict) "picking_mode" => "#define PICKING_INDEX_FROM_UV", "MAX_LIGHTS" => "#define MAX_LIGHTS $(screen.config.max_lights)", "MAX_LIGHT_PARAMETERS" => "#define MAX_LIGHT_PARAMETERS $(screen.config.max_light_parameters)", - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) end diff --git a/GLMakie/src/glshaders/visualize_interface.jl b/GLMakie/src/glshaders/visualize_interface.jl index 9cdc5ebbd29..b8ec05a33be 100644 --- a/GLMakie/src/glshaders/visualize_interface.jl +++ b/GLMakie/src/glshaders/visualize_interface.jl @@ -150,37 +150,20 @@ to_index_buffer(ctx, x) = error( Please choose from Int, Vector{UnitRange{Int}}, Vector{Int} or a signal of either of them" ) -function output_buffers(screen::Screen, transparency = false) - if transparency - """ - layout(location=2) out float coverage; - """ - elseif screen.config.ssao - """ - layout(location=2) out vec3 fragment_position; - layout(location=3) out vec3 fragment_normal_occlusion; - """ +function target_stage(screen, data) + idx = findfirst(step -> renders_in_stage(data, step), screen.render_pipeline.steps) + @assert !isnothing(idx) "Could not find a render stage compatible with the given settings." + + # Keep it simple for now + fb = screen.render_pipeline.steps[idx].framebuffer + if fb.counter == 2 + return "#define DEFAULT_TARGET" + elseif fb.counter == 3 + data[:oit_scale] = screen.config.transparency_weight_scale + return "#define OIT_TARGET" + elseif fb.counter == 4 + return "#define SSAO_TARGET" else - "" + error("Number of colorbuffers in render framebuffer does not match any known configurations ($(fb.counter))") end -end - -function output_buffer_writes(screen::Screen, transparency = false) - if transparency - scale = screen.config.transparency_weight_scale - """ - float weight = color.a * max(0.01, $scale * pow((1 - gl_FragCoord.z), 3)); - coverage = 1.0 - clamp(color.a, 0.0, 1.0); - fragment_color.rgb = weight * color.rgb; - fragment_color.a = weight; - """ - elseif screen.config.ssao - """ - fragment_color = color; - fragment_position = o_view_pos; - fragment_normal_occlusion.xyz = o_view_normal; - """ - else - "fragment_color = color;" - end -end +end \ No newline at end of file diff --git a/GLMakie/src/glshaders/voxel.jl b/GLMakie/src/glshaders/voxel.jl index 58f9b7d383b..9d9ae9338f9 100644 --- a/GLMakie/src/glshaders/voxel.jl +++ b/GLMakie/src/glshaders/voxel.jl @@ -24,8 +24,7 @@ function draw_voxels(screen, main::VolumeTypes, data::Dict) "shading" => light_calc(shading), "MAX_LIGHTS" => "#define MAX_LIGHTS $(screen.config.max_lights)", "MAX_LIGHT_PARAMETERS" => "#define MAX_LIGHT_PARAMETERS $(screen.config.max_light_parameters)", - "buffers" => output_buffers(screen, to_value(transparency)), - "buffer_writes" => output_buffer_writes(screen, to_value(transparency)) + "TARGET_STAGE" => target_stage(screen, data) ) ) end diff --git a/GLMakie/src/glwindow.jl b/GLMakie/src/glwindow.jl index 5c300670042..af8ec40dfaa 100644 --- a/GLMakie/src/glwindow.jl +++ b/GLMakie/src/glwindow.jl @@ -10,99 +10,24 @@ end Base.convert(::Type{SelectionID{T}}, s::SelectionID) where T = SelectionID{T}(T(s.id), T(s.index)) Base.zero(::Type{GLMakie.SelectionID{T}}) where T = SelectionID{T}(T(0), T(0)) -mutable struct GLFramebuffer - resolution::Observable{NTuple{2, Int}} - id::GLuint - buffer_ids::Dict{Symbol, GLuint} - buffers::Dict{Symbol, Texture} - render_buffer_ids::Vector{GLuint} -end - -# it's guaranteed, that they all have the same size -Base.size(fb::GLFramebuffer) = size(fb.buffers[:color]) -Base.haskey(fb::GLFramebuffer, key::Symbol) = haskey(fb.buffers, key) -Base.getindex(fb::GLFramebuffer, key::Symbol) = fb.buffer_ids[key] => fb.buffers[key] - -function getfallback(fb::GLFramebuffer, key::Symbol, fallback_key::Symbol) - haskey(fb, key) ? fb[key] : fb[fallback_key] -end - - -function attach_framebuffer(t::Texture{T, 2}, attachment) where T - glFramebufferTexture2D(GL_FRAMEBUFFER, attachment, GL_TEXTURE_2D, t.id, 0) -end - -# attach texture as color attachment with automatic id picking -function attach_colorbuffer!(fb::GLFramebuffer, key::Symbol, t::Texture{T, 2}) where T - if haskey(fb.buffer_ids, key) || haskey(fb.buffers, key) - error("Key $key already exists.") - end - - max_color_id = GL_COLOR_ATTACHMENT0 - for id in values(fb.buffer_ids) - if GL_COLOR_ATTACHMENT0 <= id <= GL_COLOR_ATTACHMENT15 && id > max_color_id - max_color_id = id - end - end - next_color_id = max_color_id + 0x1 - if next_color_id > GL_COLOR_ATTACHMENT15 - error("Ran out of color buffers.") - end - glFramebufferTexture2D(GL_FRAMEBUFFER, next_color_id, GL_TEXTURE_2D, t.id, 0) - push!(fb.buffer_ids, key => next_color_id) - push!(fb.buffers, key => t) - return next_color_id -end +mutable struct FramebufferFactory + fb::GLFramebuffer # core framebuffer (more or less for #4150) + # holding depth, stencil, objectid[, output_color] -function enum_to_error(s) - s == GL_FRAMEBUFFER_COMPLETE && return - s == GL_FRAMEBUFFER_UNDEFINED && - error("GL_FRAMEBUFFER_UNDEFINED: The specified framebuffer is the default read or draw framebuffer, but the default framebuffer does not exist.") - s == GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT && - error("GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT: At least one of the framebuffer attachment points is incomplete.") - s == GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT && - error("GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: The framebuffer does not have at least one image attached to it.") - s == GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER && - error("GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER: The value of GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE is GL_NONE for any color attachment point(s) specified by GL_DRAW_BUFFERi.") - s == GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER && - error("GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER: GL_READ_BUFFER is not GL_NONE and the value of GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE is GL_NONE for the color attachment point specified by GL_READ_BUFFER.") - s == GL_FRAMEBUFFER_UNSUPPORTED && - error("GL_FRAMEBUFFER_UNSUPPORTED: The combination of internal formats of the attached images violates a driver implementation-dependent set of restrictions. Check your OpenGL driver!") - s == GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE && - error("GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE: The value of GL_RENDERBUFFER_SAMPLES is not the same for all attached renderbuffers; -if the value of GL_TEXTURE_SAMPLES is not the same for all attached textures; or, if the attached images consist of a mix of renderbuffers and textures, - the value of GL_RENDERBUFFER_SAMPLES does not match the value of GL_TEXTURE_SAMPLES. - GL_FRAMEBUFFER_INCOMPLETE_MULTISAMPLE is also returned if the value of GL_TEXTURE_FIXED_SAMPLE_LOCATIONS is not consistent across all attached textures; - or, if the attached images include a mix of renderbuffers and textures, the value of GL_TEXTURE_FIXED_SAMPLE_LOCATIONS is not set to GL_TRUE for all attached textures.") - s == GL_FRAMEBUFFER_INCOMPLETE_LAYER_TARGETS && - error("GL_FRAMEBUFFER_INCOMPLETE_LAYER_TARGETS: Any framebuffer attachment is layered, and any populated attachment is not layered, or if all populated color attachments are not from textures of the same target.") - return error("Unknown framebuffer completion error code: $s") + buffers::Vector{Texture} + children::Vector{GLFramebuffer} # TODO: how else can we handle resizing? end -function check_framebuffer() - status = glCheckFramebufferStatus(GL_FRAMEBUFFER) - return enum_to_error(status) -end +Base.size(fb::FramebufferFactory) = size(fb.fb) +GLAbstraction.get_buffer(fb::FramebufferFactory, idx::Int) = fb.buffers[idx] +GLAbstraction.bind(fb::FramebufferFactory) = GLAbstraction.bind(fb.fb) -Makie.@noconstprop function GLFramebuffer(context, fb_size::NTuple{2, Int}) +Makie.@noconstprop function FramebufferFactory(context, fb_size::NTuple{2, Int}) ShaderAbstractions.switch_context!(context) require_context(context) - # Create framebuffer - frambuffer_id = glGenFramebuffers() - glBindFramebuffer(GL_FRAMEBUFFER, frambuffer_id) - - # Buffers we always need - # Holds the image that eventually gets displayed - color_buffer = Texture( - context, RGBA{N0f8}, fb_size, minfilter = :nearest, x_repeat = :clamp_to_edge - ) - # Holds a (plot id, element id) for point picking - objectid_buffer = Texture( - context, Vec{2, GLuint}, fb_size, minfilter = :nearest, x_repeat = :clamp_to_edge - ) # holds depth and stencil values depth_buffer = Texture( context, Ptr{GLAbstraction.DepthStencil_24_8}(C_NULL), fb_size, @@ -110,82 +35,79 @@ Makie.@noconstprop function GLFramebuffer(context, fb_size::NTuple{2, Int}) internalformat = GL_DEPTH24_STENCIL8, format = GL_DEPTH_STENCIL ) - # Order Independent Transparency - HDR_color_buffer = Texture( - context, RGBA{Float16}, fb_size, minfilter = :linear, x_repeat = :clamp_to_edge - ) - OIT_weight_buffer = Texture( - context, N0f8, fb_size, minfilter = :nearest, x_repeat = :clamp_to_edge - ) - attach_framebuffer(color_buffer, GL_COLOR_ATTACHMENT0) - attach_framebuffer(objectid_buffer, GL_COLOR_ATTACHMENT1) - attach_framebuffer(HDR_color_buffer, GL_COLOR_ATTACHMENT2) - attach_framebuffer(OIT_weight_buffer, GL_COLOR_ATTACHMENT3) - attach_framebuffer(depth_buffer, GL_DEPTH_ATTACHMENT) - attach_framebuffer(depth_buffer, GL_STENCIL_ATTACHMENT) - - check_framebuffer() - - fb_size_node = Observable(fb_size) - - # To allow adding postprocessors in various combinations we need to keep - # track of the buffer ids that are already in use. We may also want to reuse - # buffers so we give them names for easy fetching. - buffer_ids = Dict{Symbol,GLuint}( - :color => GL_COLOR_ATTACHMENT0, - :objectid => GL_COLOR_ATTACHMENT1, - :HDR_color => GL_COLOR_ATTACHMENT2, - :OIT_weight => GL_COLOR_ATTACHMENT3, - :depth => GL_DEPTH_ATTACHMENT, - :stencil => GL_STENCIL_ATTACHMENT, - ) - buffers = Dict{Symbol, Texture}( - :color => color_buffer, - :objectid => objectid_buffer, - :HDR_color => HDR_color_buffer, - :OIT_weight => OIT_weight_buffer, - :depth => depth_buffer, - :stencil => depth_buffer - ) + fb = GLFramebuffer(context, fb_size) + attach_depthstencilbuffer(fb, :depth_stencil, depth_buffer) - return GLFramebuffer( - fb_size_node, frambuffer_id, - buffer_ids, buffers, - [GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1] - )::GLFramebuffer + return FramebufferFactory(fb, Texture[], GLFramebuffer[]) end -function destroy!(fb::GLFramebuffer) - fb.id == 0 && return - @assert !isempty(fb.buffers) "GLFramebuffer was cleared incorrectly (i.e. not by destroy!())" - ctx = first(values(fb.buffers)).context - with_context(ctx) do - for buff in values(fb.buffers) - GLAbstraction.free(buff) - end - # Only print error if the context is not alive/active - id = fb.id - fb.id = 0 - if GLAbstraction.context_alive(ctx) && id > 0 - glDeleteFramebuffers(1, Ref(id)) - end - end +function Base.resize!(fb::FramebufferFactory, w::Int, h::Int) + ShaderAbstractions.switch_context!(first(values(fb.buffers)).context) + foreach(tex -> GLAbstraction.resize_nocopy!(tex, (w, h)), fb.buffers) + resize!(fb.fb, w, h) + filter!(fb -> fb.id != 0, fb.children) # TODO: is this ok for cleanup? + foreach(fb -> resize!(fb, w, h), fb.children) return end -function Base.resize!(fb::GLFramebuffer, w::Int, h::Int) - (w > 0 && h > 0 && (w, h) != size(fb)) || return - isempty(fb.buffers) && return # or error? - ShaderAbstractions.switch_context!(first(values(fb.buffers)).context) - for (name, buffer) in fb.buffers - resize_nocopy!(buffer, (w, h)) +function unsafe_empty!(factory::FramebufferFactory) + empty!(factory.buffers) + empty!(factory.children) + fb = GLFramebuffer(size(factory)) + attach_depthstencilbuffer(fb, :depth_stencil, get_buffer(factory.fb, :depth_stencil)) + factory.fb = fb + return factory +end + +function destroy!(factory::FramebufferFactory) + ctx = factory.fb.context + ShaderAbstractions.switch_context!(ctx) + # avoid try .. catch at call site, and allow cleanup to run + GLAbstraction.require_context_no_error(ctx) + + GLAbstraction.free.(factory.buffers) + GLAbstraction.free.(factory.children) + # make sure depth, stencil get cleared too (and maybe core color buffers in the future) + GLAbstraction.free.(factory.fb.buffers) + GLAbstraction.free(factory.fb) + empty!(factory.buffers) + empty!(factory.children) + return +end + +function Base.push!(factory::FramebufferFactory, tex::Texture) + push!(factory.buffers, tex) + return factory +end + +function generate_framebuffer(factory::FramebufferFactory, args...) + parse_arg(name::Symbol) = name => name + parse_arg(p::Pair{Symbol, Symbol}) = p + parse_arg(x::Any) = error("$x not accepted") + + return generate_framebuffer(factory, parse_arg.(args)...) +end + +Makie.@noconstprop function generate_framebuffer(factory::FramebufferFactory, idx2name::Pair{Int, Symbol}...) + filter!(fb -> fb.id != 0, factory.children) # cleanup? + + fb = GLFramebuffer(factory.fb.context, size(factory)) + + for (idx, name) in idx2name + haskey(fb, name) && error("Can't add duplicate buffer $lookup => $name") + attach_colorbuffer(fb, name, factory.buffers[idx]) end - fb.resolution[] = (w, h) - return nothing + + attach_depthstencilbuffer(fb, :depth_stencil, get_buffer(factory.fb, :depth_stencil)) + + push!(factory.children, fb) + + return fb end + struct MonitorProperties name::String isprimary::Bool diff --git a/GLMakie/src/picking.jl b/GLMakie/src/picking.jl index cde1cd4d59f..908e443fae7 100644 --- a/GLMakie/src/picking.jl +++ b/GLMakie/src/picking.jl @@ -6,10 +6,10 @@ function pick_native(screen::Screen, rect::Rect2i) isopen(screen) || return Matrix{SelectionID{Int}}(undef, 0, 0) ShaderAbstractions.switch_context!(screen.glscreen) - fb = screen.framebuffer - buff = fb.buffers[:objectid] - glBindFramebuffer(GL_FRAMEBUFFER, fb.id[1]) - glReadBuffer(GL_COLOR_ATTACHMENT1) + fb = screen.framebuffer_factory.fb + buff = get_buffer(fb, :objectid) + GLAbstraction.bind(fb) + glReadBuffer(get_attachment(fb, :objectid)) rx, ry = minimum(rect) rw, rh = widths(rect) w, h = size(screen.scene) @@ -27,10 +27,10 @@ end function pick_native(screen::Screen, xy::Vec{2, Float64}) isopen(screen) || return SelectionID{Int}(0, 0) ShaderAbstractions.switch_context!(screen.glscreen) - fb = screen.framebuffer - buff = fb.buffers[:objectid] - glBindFramebuffer(GL_FRAMEBUFFER, fb.id[1]) - glReadBuffer(GL_COLOR_ATTACHMENT1) + fb = screen.framebuffer_factory.fb + buff = get_buffer(fb, :objectid) + GLAbstraction.bind(fb) + glReadBuffer(get_attachment(fb, :objectid)) x, y = floor.(Int, xy) w, h = size(screen.scene) ppu = screen.px_per_unit[] @@ -70,7 +70,7 @@ function Makie.pick_closest(scene::Scene, screen::Screen, xy, range) w, h = size(screen.scene) # unitless dimensions ((1.0 <= xy[1] <= w) && (1.0 <= xy[2] <= h)) || return (nothing, 0) - fb = screen.framebuffer + fb = screen.framebuffer_factory.fb ppu = screen.px_per_unit[] w, h = size(fb) # pixel dimensions x0, y0 = max.(1, floor.(Int, ppu .* (xy .- range))) @@ -78,9 +78,9 @@ function Makie.pick_closest(scene::Scene, screen::Screen, xy, range) dx = x1 - x0; dy = y1 - y0 ShaderAbstractions.switch_context!(screen.glscreen) - glBindFramebuffer(GL_FRAMEBUFFER, fb.id[1]) - glReadBuffer(GL_COLOR_ATTACHMENT1) - buff = fb.buffers[:objectid] + GLAbstraction.bind(fb) + glReadBuffer(get_attachment(fb, :objectid)) + buff = get_buffer(fb, :objectid) sids = zeros(SelectionID{UInt32}, dx, dy) glReadPixels(x0, y0, dx, dy, buff.format, buff.pixeltype, sids) @@ -111,7 +111,7 @@ function Makie.pick_sorted(scene::Scene, screen::Screen, xy, range) return Tuple{AbstractPlot, Int}[] end - fb = screen.framebuffer + fb = screen.framebuffer_factory.fb ppu = screen.px_per_unit[] w, h = size(fb) # pixel dimensions x0, y0 = max.(1, floor.(Int, ppu .* (xy .- range))) @@ -119,9 +119,9 @@ function Makie.pick_sorted(scene::Scene, screen::Screen, xy, range) dx = x1 - x0; dy = y1 - y0 ShaderAbstractions.switch_context!(screen.glscreen) - glBindFramebuffer(GL_FRAMEBUFFER, fb.id[1]) - glReadBuffer(GL_COLOR_ATTACHMENT1) - buff = fb.buffers[:objectid] + GLAbstraction.bind(fb) + glReadBuffer(get_attachment(fb, :objectid)) + buff = get_buffer(fb, :objectid) picks = zeros(SelectionID{UInt32}, dx, dy) glReadPixels(x0, y0, dx, dy, buff.format, buff.pixeltype, picks) diff --git a/GLMakie/src/postprocessing.jl b/GLMakie/src/postprocessing.jl index b03f8a3ad31..acb62c3e8c9 100644 --- a/GLMakie/src/postprocessing.jl +++ b/GLMakie/src/postprocessing.jl @@ -18,32 +18,233 @@ end rcpframe(x) = 1f0 ./ Vec2f(x[1], x[2]) -struct PostProcessor{F} - robjs::Vector{RenderObject} - render::F - constructor::Any + +# or maybe Task? Stage? +""" + AbstractRenderStep + +Represents a task or step that needs to run when rendering a frame. These +tasks are collected in the RenderPipeline. + +Each task may implement: +- `prepare_step(screen, glscene, step)`: Initialize the task. +- `run_step(screen, glscene, step)`: Run the task. + +Initialization is grouped together and runs before all run steps. If you need +to initialize just before your run, bundle it with the run. +""" +abstract type AbstractRenderStep end +run_step(screen, glscene, ::AbstractRenderStep) = nothing + +function destroy!(step::T, keep_alive) where {T <: AbstractRenderStep} + @debug "Default destructor of $T" + hasfield(T, :robj) && destroy!(step.robj, keep_alive) + return +end + +# fore reference: +# construct(::Val{Name}, screen, framebuffer, inputs, parent::Makie.Stage) +function reconstruct(old::T, screen, framebuffer, inputs, parent::Makie.Stage, keep_alive) where {T <: AbstractRenderStep} + # @debug "reconstruct() not defined for $T, calling construct()" + destroy!(old, keep_alive) + return construct(Val(parent.name), screen, framebuffer, inputs, parent) +end + +# convenience +Broadcast.broadcastable(x::AbstractRenderStep) = Ref(x) + + +struct GLRenderPipeline + parent::Makie.RenderPipeline + steps::Vector{AbstractRenderStep} +end +GLRenderPipeline() = GLRenderPipeline(Makie.RenderPipeline(), AbstractRenderStep[]) + +function render_frame(screen, glscene, pipeline::GLRenderPipeline) + for step in pipeline.steps + require_context(screen.glscreen) + run_step(screen, glscene, step) + end + return +end + +function destroy!(pipeline::GLRenderPipeline) + destroy!.(pipeline.steps, Ref(UInt32[])) + empty!(pipeline.steps) + return +end + + +# TODO: temporary, we should get to the point where this is not needed +struct EmptyRenderStep <: AbstractRenderStep end + + + +struct SortPlots <: AbstractRenderStep end + +construct(::Val{:ZSort}, screen, framebuffer, inputs, parent) = SortPlots() + +function run_step(screen, glscene, ::SortPlots) + function sortby(x) + robj = x[3] + plot = screen.cache2plot[robj.id] + # TODO, use actual boundingbox + # ~7% faster than calling zvalue2d doing the same thing? + return Makie.transformationmatrix(plot)[][3, 4] + # return Makie.zvalue2d(plot) + end + + sort!(screen.renderlist; by=sortby) + return +end + + + +@enum FilterOptions begin + FilterFalse = 0 + FilterTrue = 1 + FilterAny = 2 +end +compare(val::Bool, filter::FilterOptions) = (filter == FilterAny) || (val == Int(filter)) +compare(val::Integer, filter::FilterOptions) = (filter == FilterAny) || (val == Int(filter)) + +struct RenderPlots <: AbstractRenderStep + framebuffer::GLFramebuffer + clear::Vector{Pair{Int, Vec4f}} # target index -> color + + ssao::FilterOptions + transparency::FilterOptions + fxaa::FilterOptions + + for_oit::Bool +end + +function construct(::Val{:Render}, screen, framebuffer, inputs, parent) + ssao = FilterOptions(get(parent.attributes, :ssao, 2)) # can't do FilterOptions(::FilterOptions) ??? + fxaa = FilterOptions(get(parent.attributes, :fxaa, 2)) + transparency = FilterOptions(get(parent.attributes, :transparency, 2)) + return RenderPlots(framebuffer, [3 => Vec4f(0), 4 => Vec4f(0)], ssao, transparency, fxaa, false) +end + +function construct(::Val{Symbol("OIT Render")}, screen, framebuffer, inputs, parent) + # HDR_color containing sums clears to 0 + # OIT_weight containing products clears to 1 + clear = [1 => Vec4f(0), 3 => Vec4f(1)] + return RenderPlots(framebuffer, clear, FilterAny, FilterTrue, FilterAny, true) +end + +function id2scene(screen, id1) + # TODO maybe we should use a different data structure + for (id2, scene) in screen.screens + id1 == id2 && return true, scene + end + return false, nothing +end + +renders_in_stage(robj, ::AbstractRenderStep) = false +renders_in_stage(robj::RenderObject, step::RenderPlots) = renders_in_stage(robj.uniforms, step) +function renders_in_stage(robj, step::RenderPlots) + return compare(to_value(get(robj, :ssao, false)), step.ssao) && + compare(to_value(get(robj, :transparency, false)), step.transparency) && + compare(to_value(get(robj, :fxaa, false)), step.fxaa) +end + +function run_step(screen, glscene, step::RenderPlots) + # Somehow errors in here get ignored silently!? + try + require_context(screen.glscreen) + GLAbstraction.bind(step.framebuffer) + + for (idx, color) in step.clear + idx <= step.framebuffer.counter || continue + glDrawBuffer(step.framebuffer.attachments[idx]) + glClearColor(color...) + glClear(GL_COLOR_BUFFER_BIT) + end + + set_draw_buffers(step.framebuffer) + + for (zindex, screenid, elem) in screen.renderlist + elem.visible && renders_in_stage(elem, step) || continue + + found, scene = id2scene(screen, screenid) + (found && scene.visible[]) || continue + + ppu = screen.px_per_unit[] + a = viewport(scene)[] + + require_context(screen.glscreen) + glViewport(round.(Int, ppu .* minimum(a))..., round.(Int, ppu .* widths(a))...) + + # TODO: Can we move this outside the loop? + if step.for_oit + # disable depth buffer writing + glDepthMask(GL_FALSE) + + # Blending + glEnable(GL_BLEND) + glBlendEquation(GL_FUNC_ADD) + + # buffer 0 contains weight * color.rgba, should do sum + # destination <- 1 * source + 1 * destination + glBlendFunci(0, GL_ONE, GL_ONE) + + # buffer 1 is objectid, do nothing + glDisablei(GL_BLEND, 1) + + # buffer 2 is color.a, should do product + # destination <- 0 * source + (source) * destination + glBlendFunci(2, GL_ZERO, GL_SRC_COLOR) + + else + glDepthMask(GL_TRUE) + GLAbstraction.enabletransparency() + end + + render(elem) + end + catch e + @error "Error while rendering!" exception = e + rethrow(e) + end + return +end + + + + +# TODO: maybe call this a PostProcessor? +# Vaguely leaning on Vulkan Terminology +struct RenderPass{Name} <: AbstractRenderStep + framebuffer::GLFramebuffer + robj::RenderObject end -function empty_postprocessor(args...; kwargs...) - PostProcessor(RenderObject[], screen -> nothing, empty_postprocessor) +function reconstruct(pass::RP, screen, framebuffer, inputs, ::Makie.Stage) where {RP <: RenderPass} + for (k, v) in inputs + if haskey(pass.robj.uniforms, k) + pass.robj.uniforms[k] = v + else + @error("Input $k does not exist in recreated RenderPass.") + end + end + return RP(framebuffer, pass.robj) end +function construct(::Val{:OIT}, screen, framebuffer, inputs, parent) + @debug "Creating OIT postprocessor" + require_context(screen.glscreen) -function OIT_postprocessor(framebuffer, shader_cache) # Based on https://jcgt.org/published/0002/02/09/, see #1390 # OIT setup shader = LazyShader( - shader_cache, + screen.shader_cache, loadshader("postprocessing/fullscreen.vert"), loadshader("postprocessing/OIT_blend.frag") ) - data = Dict{Symbol, Any}( - # :opaque_color => framebuffer[:color][2], - :sum_color => framebuffer[:HDR_color][2], - :prod_alpha => framebuffer[:OIT_weight][2], - ) - pass = RenderObject( - data, shader, + # TODO: rename in shader + robj = RenderObject( + inputs, shader, () -> begin glDepthMask(GL_TRUE) glDisable(GL_DEPTH_TEST) @@ -57,52 +258,24 @@ function OIT_postprocessor(framebuffer, shader_cache) # opaque.a = 0 * src.a + 1 * opaque.a glBlendFuncSeparate(GL_ONE, GL_SRC_ALPHA, GL_ZERO, GL_ONE) end, - nothing, shader_cache.context + nothing, screen.glscreen ) - pass.postrenderfunction = () -> draw_fullscreen(pass.vertexarray.id) - - color_id = framebuffer[:color][1] - full_render = screen -> begin - fb = screen.framebuffer - w, h = size(fb) + robj.postrenderfunction = () -> draw_fullscreen(robj.vertexarray.id) - # Blend transparent onto opaque - glDrawBuffer(color_id) - glViewport(0, 0, w, h) - GLAbstraction.render(pass) - end - - PostProcessor(RenderObject[pass], full_render, OIT_postprocessor) + return RenderPass{:OIT}(framebuffer, robj) end +function run_step(screen, glscene, step::RenderPass{:OIT}) + # Blend transparent onto opaque + wh = size(step.framebuffer) + set_draw_buffers(step.framebuffer) + glViewport(0, 0, wh[1], wh[2]) + GLAbstraction.render(step.robj) + return +end - - -function ssao_postprocessor(framebuffer, shader_cache) - ShaderAbstractions.switch_context!(shader_cache.context) - require_context(shader_cache.context) # for framebuffer, uniform textures - - # Add missing buffers - if !haskey(framebuffer, :position) - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer.id[1]) - position_buffer = Texture( - shader_cache.context, Vec3f, size(framebuffer), minfilter = :nearest, x_repeat = :clamp_to_edge - ) - pos_id = attach_colorbuffer!(framebuffer, :position, position_buffer) - push!(framebuffer.render_buffer_ids, pos_id) - end - if !haskey(framebuffer, :normal) - if !haskey(framebuffer, :HDR_color) - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer.id[1]) - normal_occlusion_buffer = Texture( - shader_cache.context, Vec4{Float16}, size(framebuffer), minfilter = :nearest, x_repeat = :clamp_to_edge - ) - normal_occ_id = attach_colorbuffer!(framebuffer, :normal_occlusion, normal_occlusion_buffer) - else - normal_occ_id = framebuffer[:HDR_color][1] - end - push!(framebuffer.render_buffer_ids, normal_occ_id) - end +function construct(::Val{:SSAO1}, screen, framebuffer, inputs, parent) + require_context(screen.glscreen) # SSAO setup N_samples = 64 @@ -111,204 +284,199 @@ function ssao_postprocessor(framebuffer, shader_cache) kernel = map(1:N_samples) do i n = normalize([2.0rand() .- 1.0, 2.0rand() .- 1.0, rand()]) scale = lerp_min + (lerp_max - lerp_min) * (i / N_samples)^2 - v = Vec3f(scale * rand() * n) + return Vec3f(scale * rand() * n) end + noise = [normalize(Vec2f(2.0rand(2) .- 1.0)) for _ in 1:4, __ in 1:4] # compute occlusion - shader1 = LazyShader( - shader_cache, + shader = LazyShader( + screen.shader_cache, loadshader("postprocessing/fullscreen.vert"), loadshader("postprocessing/SSAO.frag"), view = Dict( "N_samples" => "$N_samples" ) ) - data1 = Dict{Symbol, Any}( - :position_buffer => framebuffer[:position][2], - :normal_occlusion_buffer => getfallback(framebuffer, :normal_occlusion, :HDR_color)[2], - :kernel => kernel, - :noise => Texture( - shader_cache.context, [normalize(Vec2f(2.0rand(2) .- 1.0)) for _ in 1:4, __ in 1:4], - minfilter = :nearest, x_repeat = :repeat - ), - :noise_scale => map(s -> Vec2f(s ./ 4.0), framebuffer.resolution), - :projection => Observable(Mat4f(I)), - :bias => 0.025f0, - :radius => 0.5f0 - ) - pass1 = RenderObject(data1, shader1, PostprocessPrerender(), nothing, shader_cache.context) - pass1.postrenderfunction = () -> draw_fullscreen(pass1.vertexarray.id) + inputs[:kernel] = kernel + inputs[:noise] = Texture(screen.glscreen, noise, minfilter = :nearest, x_repeat = :repeat) + inputs[:noise_scale] = Vec2f(0.25f0 .* size(screen)) + inputs[:projection] = Mat4f(I) + inputs[:bias] = 0.025f0 + inputs[:radius] = 0.5f0 + robj = RenderObject(inputs, shader, PostprocessPrerender(), nothing, screen.glscreen) + robj.postrenderfunction = () -> draw_fullscreen(robj.vertexarray.id) + + return RenderPass{:SSAO1}(framebuffer, robj) +end +function construct(::Val{:SSAO2}, screen, framebuffer, inputs, parent) + require_context(screen.glscreen) # blur occlusion and combine with color - shader2 = LazyShader( - shader_cache, + shader = LazyShader( + screen.shader_cache, loadshader("postprocessing/fullscreen.vert"), loadshader("postprocessing/SSAO_blur.frag") ) - data2 = Dict{Symbol, Any}( - :normal_occlusion => getfallback(framebuffer, :normal_occlusion, :HDR_color)[2], - :color_texture => framebuffer[:color][2], - :ids => framebuffer[:objectid][2], - :inv_texel_size => lift(rcpframe, framebuffer.resolution), - :blur_range => Int32(2) - ) - pass2 = RenderObject(data2, shader2, PostprocessPrerender(), nothing, shader_cache.context) - pass2.postrenderfunction = () -> draw_fullscreen(pass2.vertexarray.id) - color_id = framebuffer[:color][1] - - full_render = screen -> begin - fb = screen.framebuffer - w, h = size(fb) - - # Setup rendering - # SSAO - calculate occlusion - glDrawBuffer(normal_occ_id) # occlusion buffer - glViewport(0, 0, w, h) - glEnable(GL_SCISSOR_TEST) - ppu = (x) -> round.(Int, screen.px_per_unit[] .* x) - - for (screenid, scene) in screen.screens - # Select the area of one leaf scene - # This should be per scene because projection may vary between - # scenes. It should be a leaf scene to avoid repeatedly shading - # the same region (though this is not guaranteed...) - isempty(scene.children) || continue - a = viewport(scene)[] - glScissor(ppu(minimum(a))..., ppu(widths(a))...) - # update uniforms - data1[:projection] = Mat4f(scene.camera.projection[]) - data1[:bias] = scene.ssao.bias[] - data1[:radius] = scene.ssao.radius[] - GLAbstraction.render(pass1) - end + inputs[:inv_texel_size] = rcpframe(size(screen)) + inputs[:blur_range] = Int32(2) + robj = RenderObject(inputs, shader, PostprocessPrerender(), nothing, screen.glscreen) + robj.postrenderfunction = () -> draw_fullscreen(robj.vertexarray.id) - # SSAO - blur occlusion and apply to color - glDrawBuffer(color_id) # color buffer - for (screenid, scene) in screen.screens - # Select the area of one leaf scene - isempty(scene.children) || continue - a = viewport(scene)[] - glScissor(ppu(minimum(a))..., ppu(widths(a))...) - # update uniforms - data2[:blur_range] = scene.ssao.blur - GLAbstraction.render(pass2) + return RenderPass{:SSAO2}(framebuffer, robj) +end + +function run_step(screen, glscene, step::RenderPass{:SSAO1}) + set_draw_buffers(step.framebuffer) # occlusion buffer + + wh = size(step.framebuffer) + glViewport(0, 0, wh[1], wh[2]) + glEnable(GL_SCISSOR_TEST) + ppu = (x) -> round.(Int, screen.px_per_unit[] .* x) + + data = step.robj.uniforms + for (screenid, scene) in screen.screens + # Select the area of one leaf scene + # This should be per scene because projection may vary between + # scenes. It should be a leaf scene to avoid repeatedly shading + # the same region (though this is not guaranteed...) + if !isempty(scene.children) || isempty(scene.plots) || + !any(p -> to_value(get(p.attributes, :ssao, false)), scene.plots) + continue end - glDisable(GL_SCISSOR_TEST) + a = viewport(scene)[] + glScissor(ppu(minimum(a))..., ppu(widths(a))...) + # update uniforms + data[:projection] = Mat4f(scene.camera.projection[]) + data[:bias] = scene.ssao.bias[] + data[:radius] = scene.ssao.radius[] + data[:noise_scale] = Vec2f(0.25f0 .* size(step.framebuffer)) + GLAbstraction.render(step.robj) end - require_context(shader_cache.context) + glDisable(GL_SCISSOR_TEST) - PostProcessor(RenderObject[pass1, pass2], full_render, ssao_postprocessor) + return end -""" - fxaa_postprocessor(framebuffer, shader_cache) - -Returns a PostProcessor that handles fxaa. -""" -function fxaa_postprocessor(framebuffer, shader_cache) - ShaderAbstractions.switch_context!(shader_cache.context) - require_context(shader_cache.context) # for framebuffer, uniform textures - - # Add missing buffers - if !haskey(framebuffer, :color_luma) - if !haskey(framebuffer, :HDR_color) - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer.id[1]) - color_luma_buffer = Texture( - shader_cache.context, RGBA{N0f8}, size(framebuffer), minfilter=:linear, x_repeat=:clamp_to_edge - ) - luma_id = attach_colorbuffer!(framebuffer, :color_luma, color_luma_buffer) - else - luma_id = framebuffer[:HDR_color][1] - end +function run_step(screen, glscene, step::RenderPass{:SSAO2}) + # TODO: SSAO doesn't copy the full color buffer and writes to a buffer + # previously used for normals. Figure out a better solution than this: + setup!(screen, step.framebuffer) + + # SSAO - blur occlusion and apply to color + set_draw_buffers(step.framebuffer) # color buffer + wh = size(step.framebuffer) + glViewport(0, 0, wh[1], wh[2]) + + glEnable(GL_SCISSOR_TEST) + ppu = (x) -> round.(Int, screen.px_per_unit[] .* x) + data = step.robj.uniforms + for (screenid, scene) in screen.screens + # Select the area of one leaf scene + isempty(scene.children) || continue + a = viewport(scene)[] + glScissor(ppu(minimum(a))..., ppu(widths(a))...) + # update uniforms + data[:blur_range] = scene.ssao.blur + data[:inv_texel_size] = rcpframe(size(step.framebuffer)) + GLAbstraction.render(step.robj) end + glDisable(GL_SCISSOR_TEST) + + return +end + +function construct(::Val{:FXAA1}, screen, framebuffer, inputs, parent) + + filter_fxaa_in_shader = get(parent.attributes, :filter_in_shader, true) + + require_context(screen.glscreen) # calculate luma for FXAA - shader1 = LazyShader( - shader_cache, + shader = LazyShader( + screen.shader_cache, loadshader("postprocessing/fullscreen.vert"), - loadshader("postprocessing/postprocess.frag") + loadshader("postprocessing/postprocess.frag"), + view = Dict("FILTER_IN_SHADER" => filter_fxaa_in_shader ? "#define FILTER_IN_SHADER" : "") ) - data1 = Dict{Symbol, Any}( - :color_texture => framebuffer[:color][2], - :object_ids => framebuffer[:objectid][2] - ) - pass1 = RenderObject(data1, shader1, PostprocessPrerender(), nothing, shader_cache.context) - pass1.postrenderfunction = () -> draw_fullscreen(pass1.vertexarray.id) + filter_fxaa_in_shader || pop!(inputs, :objectid_buffer) + robj = RenderObject(inputs, shader, PostprocessPrerender(), nothing, screen.glscreen) + robj.postrenderfunction = () -> draw_fullscreen(robj.vertexarray.id) + + return RenderPass{:FXAA1}(framebuffer, robj) +end + +function construct(::Val{:FXAA2}, screen, framebuffer, inputs, parent) + require_context(screen.glscreen) # perform FXAA - shader2 = LazyShader( - shader_cache, + shader = LazyShader( + screen.shader_cache, loadshader("postprocessing/fullscreen.vert"), loadshader("postprocessing/fxaa.frag") ) - data2 = Dict{Symbol, Any}( - :color_texture => getfallback(framebuffer, :color_luma, :HDR_color)[2], - :RCPFrame => lift(rcpframe, framebuffer.resolution), - ) - pass2 = RenderObject(data2, shader2, PostprocessPrerender(), nothing, shader_cache.context) - pass2.postrenderfunction = () -> draw_fullscreen(pass2.vertexarray.id) - - color_id = framebuffer[:color][1] - full_render = screen -> begin - fb = screen.framebuffer - w, h = size(fb) - - # FXAA - calculate LUMA - glDrawBuffer(luma_id) - glViewport(0, 0, w, h) - # necessary with negative SSAO bias... - glClearColor(1, 1, 1, 1) - glClear(GL_COLOR_BUFFER_BIT) - GLAbstraction.render(pass1) - - # FXAA - perform anti-aliasing - glDrawBuffer(color_id) # color buffer - GLAbstraction.render(pass2) - end + inputs[:RCPFrame] = rcpframe(size(framebuffer)) + robj = RenderObject(inputs, shader, PostprocessPrerender(), nothing, screen.glscreen) + robj.postrenderfunction = () -> draw_fullscreen(robj.vertexarray.id) + + return RenderPass{:FXAA2}(framebuffer, robj) +end - require_context(shader_cache.context) +function run_step(screen, glscene, step::RenderPass{:FXAA1}) + # FXAA - calculate LUMA + set_draw_buffers(step.framebuffer) + # TODO: make scissor explicit? + wh = size(step.framebuffer) + glViewport(0, 0, wh[1], wh[2]) + # TODO: is this still true? + # necessary with negative SSAO bias... + # glClearColor(1, 1, 1, 1) + # glClear(GL_COLOR_BUFFER_BIT) + GLAbstraction.render(step.robj) + return +end - PostProcessor(RenderObject[pass1, pass2], full_render, fxaa_postprocessor) +function run_step(screen, glscene, step::RenderPass{:FXAA2}) + # FXAA - perform anti-aliasing + set_draw_buffers(step.framebuffer) # color buffer + step.robj[:RCPFrame] = rcpframe(size(step.framebuffer)) + GLAbstraction.render(step.robj) + return end -""" - to_screen_postprocessor(framebuffer, shader_cache, default_id = nothing) -Sets up a Postprocessor which copies the color buffer to the screen. Used as a -final step for displaying the screen. The argument `screen_fb_id` can be used -to pass in a reference to the framebuffer ID of the screen. If `nothing` is -used (the default), 0 is used. -""" -function to_screen_postprocessor(framebuffer, shader_cache, screen_fb_id = nothing) - # draw color buffer - shader = LazyShader( - shader_cache, - loadshader("postprocessing/fullscreen.vert"), - loadshader("postprocessing/copy.frag") - ) - data = Dict{Symbol, Any}( - :color_texture => framebuffer[:color][2] - ) - pass = RenderObject(data, shader, PostprocessPrerender(), nothing, shader_cache.context) - pass.postrenderfunction = () -> draw_fullscreen(pass.vertexarray.id) - - full_render = (screen) -> begin - # transfer everything to the screen - default_id = isnothing(screen_fb_id) ? 0 : screen_fb_id[] - # GLFW uses 0, Gtk uses a value that we have to probe at the beginning of rendering - glBindFramebuffer(GL_FRAMEBUFFER, default_id) - glViewport(0, 0, makie_window_size(screen)...) - glClear(GL_COLOR_BUFFER_BIT) - GLAbstraction.render(pass) # copy postprocess - end +# TODO: Could also handle integration with Gtk, CImGui, etc with a dedicated struct +struct BlitToScreen <: AbstractRenderStep + framebuffer::GLFramebuffer + screen_framebuffer_id::Int +end - PostProcessor(RenderObject[pass], full_render, to_screen_postprocessor) +function construct(::Val{:Display}, screen, ::Nothing, inputs, parent::Makie.Stage) + require_context(screen.glscreen) + framebuffer = screen.framebuffer_factory.fb + id = get(parent.attributes, :screen_framebuffer_id, 0) + return BlitToScreen(framebuffer, id) end -function destroy!(pp::PostProcessor) - foreach(destroy!, pp.robjs) +function run_step(screen, ::Nothing, step::BlitToScreen) + # Set source + glBindFramebuffer(GL_READ_FRAMEBUFFER, step.framebuffer.id) + glReadBuffer(get_attachment(step.framebuffer, :color)) # for safety + + # TODO: Is this an observable? Can this be static? + # GLFW uses 0, Gtk uses a value that we have to probe at the beginning of rendering + glBindFramebuffer(GL_DRAW_FRAMEBUFFER, step.screen_framebuffer_id) + + src_w, src_h = framebuffer_size(screen) + trg_w, trg_h = makie_window_size(screen) + + glBlitFramebuffer( + 0, 0, src_w, src_h, + 0, 0, trg_w, trg_h, + GL_COLOR_BUFFER_BIT, GL_LINEAR + ) + return end diff --git a/GLMakie/src/precompiles.jl b/GLMakie/src/precompiles.jl index 56770ad1164..70ecc78510b 100644 --- a/GLMakie/src/precompiles.jl +++ b/GLMakie/src/precompiles.jl @@ -59,7 +59,7 @@ let end precompile(Screen, (Scene, ScreenConfig)) -precompile(GLFramebuffer, (NTuple{2,Int},)) +precompile(FramebufferFactory, (NTuple{2,Int},)) precompile(glTexImage, (GLenum, Int, GLenum, Int, Int, Int, GLenum, GLenum, Ptr{Float32})) precompile(glTexImage, (GLenum, Int, GLenum, Int, Int, Int, GLenum, GLenum, Ptr{RGBAf})) precompile(glTexImage, (GLenum, Int, GLenum, Int, Int, Int, GLenum, GLenum, Ptr{RGBf})) diff --git a/GLMakie/src/render_pipeline.jl b/GLMakie/src/render_pipeline.jl new file mode 100644 index 00000000000..b87580963b5 --- /dev/null +++ b/GLMakie/src/render_pipeline.jl @@ -0,0 +1,158 @@ +function Makie.reset!(factory::FramebufferFactory, formats::Vector{Makie.BufferFormat}) + @assert factory.fb.id != 0 "Cannot reset a destroyed FramebufferFactory" + GLAbstraction.free.(factory.children) + empty!(factory.children) + + # function barrier for types? + function get_buffer!(context, buffers, T, format) + # reuse + internalformat = GLAbstraction.default_internalcolorformat(T) + is_float_format = eltype(T) == N0f8 || eltype(T) <: AbstractFloat + default_filter = ifelse(is_float_format, :linear, :nearest) + minfilter = ifelse(format.minfilter === :any, ifelse(format.mipmap, :linear_mipmap_linear, default_filter), format.minfilter) + magfilter = ifelse(format.magfilter === :any, default_filter, format.magfilter) + + for (i, buffer) in enumerate(buffers) + if (buffer.id != 0) && (internalformat == buffer.internalformat) && + (minfilter == buffer.parameters.minfilter) && + (magfilter == buffer.parameters.magfilter) && + (format.repeat == buffer.parameters.repeat) && + (format.mipmap == buffer.parameters.mipmap) + + return popat!(buffers, i) + end + end + + # create new + if !is_float_format && (format.mipmap || minfilter != :nearest || magfilter != :nearest) + error("Cannot use :linear interpolation or mipmaps with non float buffers.") + end + format.mipmap && error("Mipmaps not yet implemented for Pipeline buffers") + # TODO: Figure out what we need to do for mipmaps. Do we need to call + # glGenerateMipmap before every stage using the buffer? Should + # Stages do that? Do we need anything extra for setup? + return Texture(context, T, size(factory), + minfilter = minfilter, magfilter = magfilter, + x_repeat = format.repeat[1], y_repeat = format.repeat[2]) + end + + # reuse buffers that match formats (and make sure that factory.buffers + # follows the order of formats) + buffers = copy(factory.buffers) + empty!(factory.buffers) + for format in formats + T = Makie.format_to_type(format) + tex = get_buffer!(factory.fb.context, buffers, T, format) + push!(factory.buffers, tex) + end + + # clean up leftovers + foreach(GLAbstraction.free, buffers) + + # Always rebuild this though, since we don't know which buffers are the + # final output buffers + fb = GLFramebuffer(factory.fb.context, size(factory)) + attach_depthstencilbuffer(fb, :depth_stencil, get_buffer(factory.fb, :depth_stencil)) + GLAbstraction.free(factory.fb) + factory.fb = fb + + return factory +end + + +function gl_render_pipeline!(screen::Screen, pipeline::Makie.RenderPipeline) + pipeline.stages[end].name === :Display || error("RenderPipeline must end with a Display stage") + previous_pipeline = screen.render_pipeline + + # Exit early if the pipeline is already up to date + previous_pipeline.parent == pipeline && return + + # TODO: check if pipeline is different from the last one before replacing it + factory = screen.framebuffer_factory + + # TODO: OpengGL ERRORS + # Maybe safer to wait on rendertask to finish and replace the GLRenderPipeline + # with an empty one while we mess with it? + # was_running = renderloop_running(screen) + # was_running && stop_renderloop!(screen, close_after_renderloop = false) + ShaderAbstractions.switch_context!(screen.glscreen) + + screen.render_pipeline = GLRenderPipeline() + + # Resolve pipeline + buffers, remap = Makie.generate_buffers(pipeline) + + # Add required buffers + reset!(factory, buffers) + + # Add back output color and objectid attachments + N = length(pipeline.stages) + buffer_idx = remap[pipeline.stageio2idx[(N, -1)]] + attach_colorbuffer(factory.fb, :color, get_buffer(factory, buffer_idx)) + buffer_idx = remap[pipeline.stageio2idx[(N, -2)]] + attach_colorbuffer(factory.fb, :objectid, get_buffer(factory, buffer_idx)) + + # Careful - framebuffer attachments are used as inputs so they need to be + # filtered when destroying the stage! (Which may include other + # other textures that do need cleanup) + keep_alive = [tex.id for tex in screen.framebuffer_factory.buffers] + push!(keep_alive, get_buffer(factory.fb, :depth_stencil).id) + + needs_cleanup = collect(eachindex(previous_pipeline.steps)) + render_pipeline = AbstractRenderStep[] + + for (stage_idx, stage) in enumerate(pipeline.stages) + inputs = Dict{Symbol, Any}() + for (key, input_idx) in stage.inputs + idx = remap[pipeline.stageio2idx[(stage_idx, -input_idx)]] + inputs[Symbol(key, :_buffer)] = get_buffer(factory, idx) + end + + connection_indices = map(eachindex(stage.output_formats)) do output_idx + return get(pipeline.stageio2idx, (stage_idx, output_idx), -1) + end + + N = length(connection_indices) + 1 + while (N > 1) && (connection_indices[N-1] == -1) + N = N-1 + end + + if isempty(connection_indices) + framebuffer = nothing + else + try + idx2name = Dict([idx => k for (k, idx) in stage.outputs]) + outputs = ntuple(N-1) do n + remap[connection_indices[n]] => idx2name[n] + end + framebuffer = generate_framebuffer(factory, outputs...) + catch e + rethrow(e) + end + end + + # can we reconstruct? (reconstruct should update framebuffer, and replace + # inputs if necessary, i.e. handle differences in connections and attributes) + idx = findfirst(==(stage), previous_pipeline.parent.stages) + + if idx === nothing + pass = construct(Val(stage.name), screen, framebuffer, inputs, stage) + else + pass = reconstruct(previous_pipeline.steps[idx], screen, framebuffer, inputs, stage, keep_alive) + filter!(!=(idx), needs_cleanup) + end + + # I guess stage should also have extra information for settings? Or should + # that be in scene.theme? + # Maybe just leave it there for now + push!(render_pipeline, pass) + end + + # Cleanup orphaned stages + foreach(i -> destroy!(previous_pipeline.steps[i], keep_alive), needs_cleanup) + + screen.render_pipeline = GLRenderPipeline(pipeline, render_pipeline) + # was_running && start_renderloop!(screen) + + return +end \ No newline at end of file diff --git a/GLMakie/src/rendering.jl b/GLMakie/src/rendering.jl index 16ac2935c7f..512302eed80 100644 --- a/GLMakie/src/rendering.jl +++ b/GLMakie/src/rendering.jl @@ -1,139 +1,76 @@ -function setup!(screen::Screen) +# TODO: needs to run before first draw to color buffers. +# With SSAO that ends up being at first render and in SSAO, as SSAO2 draws +# color to a buffer previously used for normals +function setup!(screen::Screen, fb) + GLAbstraction.bind(fb) + + # clear color buffer + glDrawBuffer(get_attachment(fb, :color)) + glClearColor(1, 1, 1, 1) + glClear(GL_COLOR_BUFFER_BIT) + + # draw scene backgrounds glEnable(GL_SCISSOR_TEST) if isopen(screen) && !isnothing(screen.scene) ppu = screen.px_per_unit[] - glScissor(0, 0, round.(Int, size(screen.scene) .* ppu)...) - glClearColor(1, 1, 1, 1) - glClear(GL_COLOR_BUFFER_BIT) for (id, scene) in screen.screens - if scene.visible[] + if scene.visible[] && scene.clear[] a = viewport(scene)[] rt = (round.(Int, ppu .* minimum(a))..., round.(Int, ppu .* widths(a))...) glViewport(rt...) - if scene.clear[] - c = scene.backgroundcolor[] - glScissor(rt...) - glClearColor(red(c), green(c), blue(c), alpha(c)) - glClear(GL_COLOR_BUFFER_BIT) - end + glScissor(rt...) + c = scene.backgroundcolor[] + glClearColor(red(c), green(c), blue(c), alpha(c)) + glClear(GL_COLOR_BUFFER_BIT) end end end glDisable(GL_SCISSOR_TEST) + return end """ -Renders a single frame of a `window` + render_frame(screen[; resize_buffer = true]) + +Renders a single frame of a `screen` """ function render_frame(screen::Screen; resize_buffers=true) - isnothing(screen.scene) && return + if isempty(screen.framebuffer_factory.children) || isnothing(screen.scene) + return + end + # Make sure this context is active (for multi-window rendering) nw = to_native(screen) ShaderAbstractions.switch_context!(nw) GLAbstraction.require_context(nw) - function sortby(x) - robj = x[3] - plot = screen.cache2plot[robj.id] - # TODO, use actual boundingbox - # ~7% faster than calling zvalue2d doing the same thing? - return Makie.transformationmatrix(plot)[][3, 4] - # return Makie.zvalue2d(plot) - end - - sort!(screen.renderlist; by=sortby) - - # NOTE - # The transparent color buffer is reused by SSAO and FXAA. Changing the - # render order here may introduce artifacts because of that. - - fb = screen.framebuffer + # Resize framebuffer to window size + # TODO: Hacky, assumes our first draw is a render (ZSort doesn't draw) and + # no earlier stage uses color or objectid + # Also assumes specific names + fb = screen.framebuffer_factory.children[1] if resize_buffers ppu = screen.px_per_unit[] - resize!(fb, round.(Int, ppu .* size(screen.scene::Scene))...) - end - - # prepare stencil (for sub-scenes) - glBindFramebuffer(GL_FRAMEBUFFER, fb.id) - glDrawBuffers(length(fb.render_buffer_ids), fb.render_buffer_ids) - glClearColor(0, 0, 0, 0) - glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT) - - glDrawBuffer(fb.render_buffer_ids[1]) - setup!(screen) - glDrawBuffers(length(fb.render_buffer_ids), fb.render_buffer_ids) - - # render with SSAO - glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE) - GLAbstraction.render(screen) do robj - return !Bool(robj[:transparency][]) && Bool(robj[:ssao][]) + resize!(screen.framebuffer_factory, round.(Int, ppu .* size(screen.scene))...) end - # SSAO - screen.postprocessors[1].render(screen) - # render no SSAO - glDrawBuffers(2, [GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1]) - # render all non ssao - GLAbstraction.render(screen) do robj - return !Bool(robj[:transparency][]) && !Bool(robj[:ssao][]) - end + GLAbstraction.bind(fb) + glViewport(0, 0, size(fb)...) - # TRANSPARENT RENDER - # clear sums to 0 - glDrawBuffer(GL_COLOR_ATTACHMENT2) + # clear objectid, depth and stencil + glDrawBuffer(get_attachment(fb, :objectid)) glClearColor(0, 0, 0, 0) - glClear(GL_COLOR_BUFFER_BIT) - # clear alpha product to 1 - glDrawBuffer(GL_COLOR_ATTACHMENT3) - glClearColor(1, 1, 1, 1) - glClear(GL_COLOR_BUFFER_BIT) - # draw - glDrawBuffers(3, [GL_COLOR_ATTACHMENT2, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT3]) - # Render only transparent objects - GLAbstraction.render(screen) do robj - return Bool(robj[:transparency][]) - end - - # TRANSPARENT BLEND - screen.postprocessors[2].render(screen) + glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT) - # FXAA - screen.postprocessors[3].render(screen) + # TODO: figure out something better for setup!() + setup!(screen, fb) - # transfer everything to the screen - screen.postprocessors[4].render(screen) + render_frame(screen, nothing, screen.render_pipeline) GLAbstraction.require_context(nw) - return -end - -function id2scene(screen, id1) - # TODO maybe we should use a different data structure - for (id2, scene) in screen.screens - id1 == id2 && return true, scene - end - return false, nothing -end - -function GLAbstraction.render(filter_elem_func, screen::Screen) - # Somehow errors in here get ignored silently!? - try - for (zindex, screenid, elem) in screen.renderlist - filter_elem_func(elem)::Bool || continue + GLAbstraction.require_context(nw) - found, scene = id2scene(screen, screenid) - found || continue - scene.visible[] || continue - ppu = screen.px_per_unit[] - a = viewport(scene)[] - glViewport(round.(Int, ppu .* minimum(a))..., round.(Int, ppu .* widths(a))...) - render(elem) - end - catch e - @error "Error while rendering!" exception = e - rethrow(e) - end return end diff --git a/GLMakie/src/screen.jl b/GLMakie/src/screen.jl index 92270e68938..c110cf444fd 100644 --- a/GLMakie/src/screen.jl +++ b/GLMakie/src/screen.jl @@ -60,9 +60,7 @@ mutable struct ScreenConfig scalefactor::Union{Nothing, Float32} # Render Constants & Postprocessor - oit::Bool - fxaa::Bool - ssao::Bool + render_pipeline::Makie.RenderPipeline transparency_weight_scale::Float32 max_lights::Int max_light_parameters::Int @@ -87,9 +85,7 @@ mutable struct ScreenConfig scalefactor::Union{Makie.Automatic, Number}, # Preprocessor - oit::Bool, - fxaa::Bool, - ssao::Bool, + render_pipeline::Makie.RenderPipeline, transparency_weight_scale::Number, max_lights::Int, max_light_parameters::Int) @@ -111,11 +107,7 @@ mutable struct ScreenConfig monitor, visible, scalefactor isa Makie.Automatic ? nothing : Float32(scalefactor), - # Preproccessor - # Preprocessor - oit, - fxaa, - ssao, + render_pipeline, transparency_weight_scale, max_lights, max_light_parameters) @@ -164,7 +156,7 @@ mutable struct Screen{GLWindow} <: MakieScreen owns_glscreen::Bool shader_cache::GLAbstraction.ShaderCache - framebuffer::GLFramebuffer + framebuffer_factory::FramebufferFactory config::Union{Nothing, ScreenConfig} stop_renderloop::Threads.Atomic{Bool} rendertask::Union{Task, Nothing} @@ -175,7 +167,7 @@ mutable struct Screen{GLWindow} <: MakieScreen screen2scene::Dict{WeakRef, ScreenID} screens::Vector{ScreenArea} renderlist::Vector{Tuple{ZIndex, ScreenID, RenderObject}} - postprocessors::Vector{PostProcessor} + render_pipeline::GLRenderPipeline cache::Dict{UInt64, RenderObject} cache2plot::Dict{UInt32, Plot} framecache::Matrix{RGB{N0f8}} @@ -193,7 +185,7 @@ mutable struct Screen{GLWindow} <: MakieScreen glscreen::GLWindow, owns_glscreen::Bool, shader_cache::GLAbstraction.ShaderCache, - framebuffer::GLFramebuffer, + framebuffer_factory::FramebufferFactory, config::Union{Nothing, ScreenConfig}, stop_renderloop::Bool, rendertask::Union{Nothing, Task}, @@ -201,18 +193,17 @@ mutable struct Screen{GLWindow} <: MakieScreen screen2scene::Dict{WeakRef, ScreenID}, screens::Vector{ScreenArea}, renderlist::Vector{Tuple{ZIndex, ScreenID, RenderObject}}, - postprocessors::Vector{PostProcessor}, cache::Dict{UInt64, RenderObject}, cache2plot::Dict{UInt32, AbstractPlot}, reuse::Bool ) where {GLWindow} - s = size(framebuffer) + s = size(framebuffer_factory) screen = new{GLWindow}( - glscreen, (10,10), owns_glscreen, shader_cache, framebuffer, + glscreen, (10,10), owns_glscreen, shader_cache, framebuffer_factory, config, Threads.Atomic{Bool}(stop_renderloop), rendertask, BudgetedTimer(1.0 / 30.0), Observable(0f0), screen2scene, - screens, renderlist, postprocessors, cache, cache2plot, + screens, renderlist, GLRenderPipeline(), cache, cache2plot, Matrix{RGB{N0f8}}(undef, s), Observable(Makie.UnknownTickState), Observable(true), Observable(0f0), nothing, reuse, true, false ) @@ -222,12 +213,12 @@ mutable struct Screen{GLWindow} <: MakieScreen end # The exact size in pixel of the render targert (the actual matrix of pixels) -framebuffer_size(screen::Screen) = screen.framebuffer.resolution[] +framebuffer_size(screen::Screen) = size(screen.framebuffer_factory) # The size of the window in Makie's own units makie_window_size(screen::Screen) = round.(Int, scene_size(screen) .* screen.scalefactor[]) -# The size of the window in Makie, device indepentent units +# The size of the window in Makie, device independent units scene_size(screen::Screen) = size(screen.scene) Makie.isvisible(screen::Screen) = screen.config.visible @@ -306,13 +297,7 @@ Makie.@noconstprop function empty_screen(debugging::Bool, reuse::Bool, window) # This is important for resource tracking, and only needed for the first context ShaderAbstractions.switch_context!(window) shader_cache = GLAbstraction.ShaderCache(window) - fb = GLFramebuffer(window, initial_resolution) - postprocessors = [ - empty_postprocessor(), - empty_postprocessor(), - empty_postprocessor(), - to_screen_postprocessor(fb, shader_cache) - ] + fb = FramebufferFactory(window, initial_resolution) screen = Screen( window, owns_glscreen, shader_cache, fb, @@ -321,7 +306,6 @@ Makie.@noconstprop function empty_screen(debugging::Bool, reuse::Bool, window) Dict{WeakRef, ScreenID}(), ScreenArea[], Tuple{ZIndex, ScreenID, RenderObject}[], - postprocessors, Dict{UInt64, RenderObject}(), Dict{UInt32, AbstractPlot}(), reuse, @@ -399,6 +383,7 @@ end function apply_config!(screen::Screen, config::ScreenConfig; start_renderloop::Bool=true) @debug("Applying screen config! to existing screen") + glw = screen.glscreen if screen.owns_glscreen @@ -416,20 +401,8 @@ function apply_config!(screen::Screen, config::ScreenConfig; start_renderloop::B screen.scalefactor[] = !isnothing(config.scalefactor) ? config.scalefactor : scale_factor(glw) screen.px_per_unit[] = !isnothing(config.px_per_unit) ? config.px_per_unit : screen.scalefactor[] - function replace_processor!(postprocessor, idx) - fb = screen.framebuffer - shader_cache = screen.shader_cache - post = screen.postprocessors[idx] - if post.constructor !== postprocessor - destroy!(screen.postprocessors[idx]) - screen.postprocessors[idx] = postprocessor(fb, shader_cache) - end - return - end - replace_processor!(config.ssao ? ssao_postprocessor : empty_postprocessor, 1) - replace_processor!(config.oit ? OIT_postprocessor : empty_postprocessor, 2) - replace_processor!(config.fxaa ? fxaa_postprocessor : empty_postprocessor, 3) + gl_render_pipeline!(screen, config.render_pipeline) # TODO: replace shader programs with lighting to update N_lights & N_light_parameters @@ -444,6 +417,9 @@ function apply_config!(screen::Screen, config::ScreenConfig; start_renderloop::B resize!(screen, size(screen.scene)...) end set_screen_visibility!(screen, config.visible) + + screen.requires_update = true + return screen end @@ -548,7 +524,7 @@ Base.wait(scene::Scene) = wait(Makie.getscreen(scene)) Base.show(io::IO, screen::Screen) = print(io, "GLMakie.Screen(...)") Base.isopen(x::Screen) = isopen(x.glscreen) -Base.size(x::Screen) = size(x.framebuffer) +Base.size(x::Screen) = size(x.framebuffer_factory) function add_scene!(screen::Screen, scene::Scene) get!(screen.screen2scene, WeakRef(scene)) do @@ -617,7 +593,7 @@ function Base.delete!(screen::Screen, scene::Scene) end # Note: can be called from scene finalizer, must not error or print unless to Core.stdout -function destroy!(rob::RenderObject) +function destroy!(rob::RenderObject, keep_alive = UInt32[]) # These need explicit clean up because (some of) the source observables # remain when the plot is deleted. with_context(rob.context) do @@ -626,7 +602,7 @@ function destroy!(rob::RenderObject) for (k, v) in rob.uniforms if v isa Observable Observables.clear(v) - elseif v isa GPUArray && v !== tex + elseif (v isa GPUArray) && (v !== tex) && (!(v isa Texture) || !in(v.id, keep_alive)) # We usually don't share gpu data and it should be hard for users to share buffers.. # but we do share the texture atlas, so we check v !== tex, since we can't just free shared resources @@ -644,7 +620,23 @@ function destroy!(rob::RenderObject) return end -# Note: called from scene finalizer, must not error +# Note: can be called from scene finalizer, must not error or print unless to Core.stdout +function with_context(f, context) + CTX = ShaderAbstractions.ACTIVE_OPENGL_CONTEXT + old_ctx = isassigned(CTX) ? CTX[] : nothing + GLAbstraction.switch_context!(context) + try + f() + finally + if isnothing(old_ctx) + GLAbstraction.switch_context!() + else + GLAbstraction.switch_context!(old_ctx) + end + end +end + +# Note: can be called from scene finalizer, must not error or print unless to Core.stdout function Base.delete!(screen::Screen, scene::Scene, plot::AbstractPlot) if !isempty(plot.plots) @@ -720,12 +712,12 @@ function destroy!(screen::Screen) empty!(SINGLETON_SCREEN) end - foreach(destroy!, screen.postprocessors) # before texture atlas, otherwise it regenerates - destroy!(screen.framebuffer) - with_context(window) do - cleanup_texture_atlas!(window) - GLAbstraction.free(screen.shader_cache) - end + # before texture atlas, otherwise it regenerates + destroy!(screen.render_pipeline) + destroy!(screen.framebuffer_factory) + cleanup_texture_atlas!(window) + GLAbstraction.free(screen.shader_cache) + destroy!(window) return end @@ -802,7 +794,7 @@ function Base.resize!(screen::Screen, w::Int, h::Int) # w/h are in device independent Makie units (scene size) ppu = screen.px_per_unit[] fbw, fbh = round.(Int, ppu .* (w, h)) - resize!(screen.framebuffer, fbw, fbh) + resize!(screen.framebuffer_factory, fbw, fbh) if screen.owns_glscreen # Resize the window which appears on the user desktop (if necessary). @@ -829,6 +821,17 @@ function fast_color_data!(dest::Array{RGB{N0f8}, 2}, source::Texture{T, 2}) wher return end +function Makie.colorbuffer(screen::Screen, source::Texture{T, 2}) where T + ShaderAbstractions.switch_context!(screen.glscreen) + # render_frame(screen, resize_buffers=false) # let it render + glFinish() # block until opengl is done rendering + img = Matrix{eltype(source)}(undef, size(source)) + GLAbstraction.bind(source) + GLAbstraction.glGetTexImage(source.texturetype, 0, source.format, source.pixeltype, img) + GLAbstraction.bind(source, 0) + return img +end + """ depthbuffer(screen::Screen) @@ -849,7 +852,7 @@ function depthbuffer(screen::Screen) ShaderAbstractions.switch_context!(screen.glscreen) render_frame(screen, resize_buffers=false) # let it render glFinish() # block until opengl is done rendering - source = screen.framebuffer.buffers[:depth] + source = get_buffer(screen.framebuffer_factory.fb, :depth_stencil) depth = Matrix{Float32}(undef, size(source)) GLAbstraction.bind(source) GLAbstraction.glGetTexImage(source.texturetype, 0, GL_DEPTH_COMPONENT, GL_FLOAT, depth) @@ -862,7 +865,7 @@ function Makie.colorbuffer(screen::Screen, format::Makie.ImageStorageFormat = Ma error("Screen not open!") end ShaderAbstractions.switch_context!(screen.glscreen) - ctex = screen.framebuffer.buffers[:color] + ctex = get_buffer(screen.framebuffer_factory.fb, :color) # polling may change window size, when its bigger than monitor! # we still need to poll though, to get all the newest events! pollevents(screen, Makie.BackendTick) diff --git a/GLMakie/test/glmakie_refimages.jl b/GLMakie/test/glmakie_refimages.jl index afc81dd8c17..665c183cae1 100644 --- a/GLMakie/test/glmakie_refimages.jl +++ b/GLMakie/test/glmakie_refimages.jl @@ -167,4 +167,124 @@ end mesh!(scene, Rect2f(0, 0, 300, 200), color = :red) scene +end + +@reference_test "render stage parameters with SSAO postprocessor" begin + GLMakie.closeall() # (mostly?) for recompilation of plot shaders + GLMakie.activate!(ssao = true) + f = Figure() + ps = [Point3f(x, y, sin(x * y + y-x)) for x in range(-2, 2, length=21) for y in range(-2, 2, length=21)] + for (i, ssao) in zip(1:2, (false, true)) + for (j, fxaa) in zip(1:2, (false, true)) + for (k, transparency) in zip(1:2, (false, true)) + Label(f[2i-1, k + 2(j-1)], "$ssao, $fxaa, $transparency", tellwidth = false) + a, p = meshscatter( + f[2i, k + 2(j-1)], ps, marker = Rect3f(Point3f(-0.5), Vec3f(1)), + markersize = 1, color = :white; ssao, fxaa, transparency) + end + end + end + f +end + +@reference_test "render stage parameters without SSAO postprocessor" begin + GLMakie.closeall() + GLMakie.activate!(ssao = false) + f = Figure() + ps = [Point3f(x, y, sin(x * y + y-x)) for x in range(-2, 2, length=21) for y in range(-2, 2, length=21)] + for (i, ssao) in zip(1:2, (false, true)) + for (j, fxaa) in zip(1:2, (false, true)) + for (k, transparency) in zip(1:2, (false, true)) + Label(f[2i-1, k + 2(j-1)], "$ssao, $fxaa, $transparency", tellwidth = false) + a, p = meshscatter( + f[2i, k + 2(j-1)], ps, marker = Rect3f(Point3f(-0.5), Vec3f(1)), + markersize = 1, color = :white; ssao, fxaa, transparency) + end + end + end + f +end + +@reference_test "Custom stage in render pipeline" begin + GLMakie.closeall() + + function GLMakie.construct(::Val{:Tint}, screen, framebuffer, inputs, parent) + frag_shader = """ + {{GLSL_VERSION}} + + in vec2 frag_uv; + out vec4 fragment_color; + + uniform sampler2D color_buffer; // \$(name of input)_buffer + uniform mat3 color_transform; // from Stage attributes + + void main(void) { + vec4 c = texture(color_buffer, frag_uv).rgba; + fragment_color = vec4(color_transform * c.rgb, c.a); + // fragment_color = vec4(1,0,0, c.a); + } + """ + + shader = GLMakie.LazyShader( + screen.shader_cache, + GLMakie.loadshader("postprocessing/fullscreen.vert"), + GLMakie.ShaderSource(frag_shader, :frag) + ) + + inputs[:color_transform] = parent.attributes[:color_transform] + + robj = GLMakie.RenderObject(inputs, shader, + GLMakie.PostprocessPrerender(), nothing, screen.glscreen + ) + robj.postrenderfunction = () -> GLMakie.draw_fullscreen(robj.vertexarray.id) + + return GLMakie.RenderPass{:Tint}(framebuffer, robj) + end + + function GLMakie.run_step(screen, glscene, step::GLMakie.RenderPass{:Tint}) + # Blend transparent onto opaque + wh = size(step.framebuffer) + GLMakie.set_draw_buffers(step.framebuffer) + GLMakie.glViewport(0, 0, wh[1], wh[2]) + GLMakie.GLAbstraction.render(step.robj) + return + end + + + begin + # Pipeline matches test_pipeline_2D up to color_tint + pipeline = Makie.RenderPipeline() + + render1 = push!(pipeline, Makie.RenderStage(transparency = false, fxaa = true)) + render2 = push!(pipeline, Makie.TransparentRenderStage()) + oit = push!(pipeline, Makie.OITStage()) + fxaa = push!(pipeline, Makie.FXAAStage(filter_in_shader = false)) + render3 = push!(pipeline, Makie.RenderStage(transparency = false, fxaa = false)) + color_tint = push!(pipeline, Makie.Stage(:Tint, + inputs = [:color => Makie.BufferFormat()], # defaults to 4x N0f8, i.e. 32Bit color + outputs = [:color => Makie.BufferFormat()], + color_transform = Observable(Makie.Mat3f( + # sepia filter + 0.393, 0.349, 0.272, + 0.769, 0.686, 0.534, + 0.189, 0.168, 0.131 + )) + )) + display_stage = push!(pipeline, Makie.DisplayStage()) + + connect!(pipeline, render1, fxaa) + connect!(pipeline, render1, display_stage, :objectid) + connect!(pipeline, render2, oit) + connect!(pipeline, render2, display_stage, :objectid) + connect!(pipeline, oit, fxaa, :color) + connect!(pipeline, fxaa, color_tint, :color) + connect!(pipeline, render3, color_tint, :color) + connect!(pipeline, render3, display_stage, :objectid) + connect!(pipeline, color_tint, display_stage, :color) + end + + + GLMakie.activate!(render_pipeline = pipeline) + cow = load(Makie.assetpath("cow.png")) + f,a,p = image(rotr90(cow)) end \ No newline at end of file diff --git a/GLMakie/test/runtests.jl b/GLMakie/test/runtests.jl index b6c4be8d2d2..de027dff1e3 100644 --- a/GLMakie/test/runtests.jl +++ b/GLMakie/test/runtests.jl @@ -109,15 +109,23 @@ end sleep(0.1) GLMakie.closeall() - # Why does it start with a skipped tick? + # Screen initalizes with `requires_update = false`, so it may skip renders i = 1 while tick_record[i].state == Makie.SkippedRenderTick check_tick(tick_record[1], Makie.SkippedRenderTick, i) i += 1 end + # apply_config!() and scene insertion cause renders + # (number of ticks is probably performance sensitive here so require 1, allow 1..3) check_tick(tick_record[i], Makie.RegularRenderTick, i) i += 1 + for j in 1:2 + if tick_record[i].state == Makie.RegularRenderTick + check_tick(tick_record[i], Makie.RegularRenderTick, i) + i += 1 + end + end while tick_record[i].state == Makie.SkippedRenderTick check_tick(tick_record[i], Makie.SkippedRenderTick, i) @@ -140,18 +148,26 @@ end # texture atlas is triggered by text # include SSAO to make sure its cleanup works too f,a,p = image(rand(4,4)) + + # make sure switching the render pipeline doesn't cause errors + screen = display(f, ssao = true, visible = false) + colorbuffer(screen) + screen = display(f, ssao = false, visible = false) + colorbuffer(screen) screen = display(f, ssao = true, visible = false) colorbuffer(screen) # verify that SSAO is active - @test screen.postprocessors[1] != GLMakie.empty_postprocessor() + @test :SSAO1 in map(x -> x.name, screen.render_pipeline.parent.stages) - framebuffer = screen.framebuffer - framebuffer_textures = copy(screen.framebuffer.buffers) + framebuffer = screen.framebuffer_factory + framebuffer_depth = GLMakie.get_buffer(screen.framebuffer_factory.fb, :depth_stencil) + framebuffer_textures = copy(screen.framebuffer_factory.buffers) + framebuffer_children = copy(screen.framebuffer_factory.children) atlas_textures = first.(values(GLMakie.atlas_texture_cache)) shaders = vcat([[shader for shader in values(shaders)] for shaders in values(screen.shader_cache.shader_cache)]...) programs = [program for program in values(screen.shader_cache.program_cache)] - postprocessors = copy(screen.postprocessors) + pipeline = copy(screen.render_pipeline.steps) robjs = last.(screen.renderlist) GLMakie.destroy!(screen) @@ -164,11 +180,26 @@ end end @testset "Framebuffer" begin - @test framebuffer.id == 0 + # GLFramebuffer object + @test framebuffer.fb.id == 0 + @test all(x -> x.id == 0, framebuffer.fb.buffers) + + # FramebufferFactory object + @test isempty(framebuffer.children) + @test isempty(framebuffer.buffers) + @test !isempty(framebuffer_textures) - for (k, tex) in framebuffer_textures + for tex in framebuffer_textures @test tex.id == 0 end + + @test !isempty(framebuffer_children) + for fb in framebuffer_children + @test fb.id == 0 + @test framebuffer.fb.id == 0 + # should automatically be true if all framebuffer_textures == 0 + @test all(x -> x.id == 0, framebuffer.fb.buffers) + end end @testset "ShaderCache" begin @@ -202,10 +233,10 @@ end end @testset "PostProcessors" begin - @test map(pp -> length(pp.robjs), postprocessors) == [2,1,2,1] - for pp in postprocessors - for robj in pp.robjs - validate_robj(robj) + @test length(pipeline) == 10 + for step in pipeline + if hasfield(typeof(step), :robj) + validate_robj(getfield(step, :robj)) end end end @@ -218,5 +249,6 @@ end end # Check that no finalizers triggered on un-freed objects throughout all tests + GC.gc(true) @test GLMakie.GLAbstraction.FAILED_FREE_COUNTER[] == 0 end diff --git a/GLMakie/test/unit_tests.jl b/GLMakie/test/unit_tests.jl index 306503f6e74..5b3fe8f47b7 100644 --- a/GLMakie/test/unit_tests.jl +++ b/GLMakie/test/unit_tests.jl @@ -11,39 +11,39 @@ end screen = display(GLMakie.Screen(visible = false), Figure()) cache = screen.shader_cache; # Postprocessing shaders - @test length(cache.shader_cache) == 5 - @test length(cache.template_cache) == 5 - @test length(cache.program_cache) == 4 + @test length(cache.shader_cache) == 4 + @test length(cache.template_cache) == 4 + @test length(cache.program_cache) == 3 # Shaders for scatter + linesegments + poly etc (axis) display(screen, scatter(1:4)) - @test length(cache.shader_cache) == 18 - @test length(cache.template_cache) == 18 - @test length(cache.program_cache) == 11 + @test length(cache.shader_cache) == 17 + @test length(cache.template_cache) == 17 + @test length(cache.program_cache) == 10 # No new shaders should be added: display(screen, scatter(1:4)) - @test length(cache.shader_cache) == 18 - @test length(cache.template_cache) == 18 - @test length(cache.program_cache) == 11 + @test length(cache.shader_cache) == 17 + @test length(cache.template_cache) == 17 + @test length(cache.program_cache) == 10 # Same for linesegments display(screen, linesegments(1:4)) - @test length(cache.shader_cache) == 18 - @test length(cache.template_cache) == 18 - @test length(cache.program_cache) == 11 + @test length(cache.shader_cache) == 17 + @test length(cache.template_cache) == 17 + @test length(cache.program_cache) == 10 # heatmap hasn't been compiled so one new program should be added display(screen, heatmap([1,2,2.5,3], [1,2,2.5,3], rand(4,4))) - @test length(cache.shader_cache) == 20 - @test length(cache.template_cache) == 20 - @test length(cache.program_cache) == 12 + @test length(cache.shader_cache) == 19 + @test length(cache.template_cache) == 19 + @test length(cache.program_cache) == 11 # For second time no new shaders should be added display(screen, heatmap([1,2,2.5,3], [1,2,2.5,3], rand(4,4))) - @test length(cache.shader_cache) == 20 - @test length(cache.template_cache) == 20 - @test length(cache.program_cache) == 12 + @test length(cache.shader_cache) == 19 + @test length(cache.template_cache) == 19 + @test length(cache.program_cache) == 11 end @testset "unit tests" begin @@ -326,7 +326,7 @@ end screen = display(GLMakie.Screen(visible = true, scalefactor = 2), fig) @test screen.scalefactor[] === 2f0 @test screen.px_per_unit[] === 2f0 # inherited from scale factor - @test size(screen.framebuffer) == (2W, 2H) + @test size(screen.framebuffer_factory) == (2W, 2H) @test GLMakie.window_size(screen.glscreen) == scaled(screen, (W, H)) # check that picking works through the resized GL buffers @@ -353,7 +353,7 @@ end screen = display(GLMakie.Screen(visible = false, scalefactor = 2, px_per_unit = 1), fig) @test screen.scalefactor[] === 2f0 @test screen.px_per_unit[] === 1f0 - @test size(screen.framebuffer) == (W, H) + @test size(screen.framebuffer_factory) == (W, H) # decrease the scale factor after-the-fact screen.scalefactor[] = 1 diff --git a/docs/makedocs.jl b/docs/makedocs.jl index f28f4345c28..7e738d6a742 100644 --- a/docs/makedocs.jl +++ b/docs/makedocs.jl @@ -189,6 +189,7 @@ pages = [ "explanations/theming/predefined_themes.md", ], "explanations/transparency.md", + "explanations/render_pipeline.md" ], "How-Tos" => [ "how-to/match-figure-size-font-sizes-and-dpi.md", diff --git a/docs/src/explanations/render_pipeline.md b/docs/src/explanations/render_pipeline.md new file mode 100644 index 00000000000..c4c781c216e --- /dev/null +++ b/docs/src/explanations/render_pipeline.md @@ -0,0 +1,287 @@ +# Render Pipeline + +The render pipeline determines in what order plots are rendered and how they get composed into the final image you see. +Currently each backend has its own pipeline which can cause them to behave differently in some cases. +This section aims to summarize how they work and what you can influence to potentially deal with those issues. +Note that this is a fairly in-depth topic that you should usually not need to deal with. +Note as well that we may change pipelines in the future to improve the quality of a backend or the consistency between backends. + +## CairoMakie + +CairoMakie's pipeline is the simplest out of the backends. +It first draws the backgrounds of each scene with `scene.clear[] == true` recursively, starting with the root scene of the scene tree. +Then it gathers all plots in the scene tree for rendering. +They get sorted based on `Makie.zvalue2d()` which queries `plot.transformation.model` and then rendered if they are visible. +In pseudocode it boils down to: + +```julia +function cairo_draw(root_scene) + draw_background_rec(root_scene) + + all_plots = collect_all_plots_recursively(root_scene) + for plot in sort!(all_plots, by = Makie.zvalue2d) + if plot.visible[] && parent_scene(plot).visible[] + clip_to_parent_scene(plot) + draw_plot(plot) + end + end +end + +function draw_background_rec(scene) + if scene.clear[] + draw_background(scene) + end + foreach(draw_background_rec, scene.children) +end +``` + +Note that Cairo or more specifically svg and pdf do not support per-pixel z order. +Therefore z order is generally limited to be per-plot. +Some mesh plots additionally sort triangles based on their average depth to improve on this. +This however is limited to each plot, i.e. another plot will either be completely below or above. + +## GLMakie + +In GLMakie each frame begins by clearing a color buffer, depth buffer and objectid buffer. +Next we go through all scenes in the scene tree recursively and draw their backgrounds to the initial color buffer if `scene.clear[] == true`. +After that rendering is determined by the `Makie.Renderpipeline` that is used by the screen. + +### Default RenderPipeline + +The default pipeline looks like this: + +```@figure +scene = Scene(size = (800, 300), camera = cam2d!) +Makie.pipeline_gui!(scene, Makie.default_pipeline()) +center!(scene) +scene +``` + +It begins with `ZSort` which sorts all plots based on `Makie.zvalue2d`. +Next is the `Render` stage which renders all plots with `plot.transparency[] = false` to the initial color buffer. +Then we get to two Order Independent Transparency Stages. +We render every plot with `plot.transparency[] = true` in `OIT Render`, weighing their colors based on depth and alpha values and then blend them with the opaque color buffer from the first `Render` stage. +The result continues through two anti-aliasing stages `FXAA1` and `FXAA2`. +The first calculates luma (brightness) values and the second uses them to find aliased edges and blur them. +The result moves on to the `Display` stage which copies the color buffer to the screen. + +### Plot and Scene Insertion + +There are two ways plots and scenes can be added to a GLMakie screen. +The first is by displaying a root scene, i.e. (implicitly) calling `display(fig)` or `display(scene)`. +In this case plots and scenes are added to the screen like in CairoMakie. +In pseudocode we effectively do: + +```julia +function add_scene!(screen, scene) + add_scene!(screen, scene) + foreach(plot -> add_plot!(screen, plot, scene), scene.plots) + foreach(scene -> add_scene!(screen, scene), scene.children) +end +``` + +The second way is to add them interactively, i.e. while the root figure or scene is already being displayed. +Adding a plot to a scene will effectively call the `add_plot!(screen, plot, scene)` function above. +Adding a scene however does not call `add_scene!(screen, scene)`. +It is only added once a plot is added to the new scene. +This is usually not a problem, but can cause differences in clearing order. +Deletion works the same for the plots and scenes - both immediately get removed from the screen. +For scenes all child scenes and child plots get deleted as well. + +Note that plot (insertion) order also plays a role in rendering. +When two plots are not separated due to render passes or z order, they will get rendered in the order they were inserted. + +### Overdraw Attribute + +The `overdraw` attribute does not affect when a plot gets rendered in terms of render order or render passes. +Instead it affects depth testing. +A plot with `overdraw = true` ignores depth values when drawing and does not write out its own depth values. +This means that such a plot will draw over everything that has been render, but not stop later plots from drawing over it. + +### SSAO RenderPipeline + +Screen Space Ambient Occlusion is an optional post-processing effect you can turn on in GLMakie. +It modifies the render loop to include 3 more stages - one Render stage and two SSAO stages: + +```@figure +scene = Scene(size = (1000, 300), camera = cam2d!) +Makie.pipeline_gui!(scene, Makie.default_pipeline(ssao = true)) +center!(scene) +scene +``` + +The `Render` stage includes options for filtering plots based on their values for `ssao`, `fxaa` and `transparency`. +The single Render in the default pipeline only filtered based on `transparency == false`. +Now we also filter based on `ssao`. +The first Render stage requires `ssao == true && transparency == false`, the second `ssao = false && transparency = false`. +OIT Render requires `transparency == true`, so that all combinations are covered. +`fxaa` is handled per-pixel in the FXAA stages in this pipeline, but can also be handled per plot. + +After the first Render Stage there are two SSAO stages. +The first calculates an occlusion factor based on the surrounding geometry using position and normal data recorded in the render stage. +The second smooths the occlusion factor and applies it to the recorded colors. +The next Render stage then continues to add to those colors before merging with OIT and applying FXAA. + +### Custom RenderPipeline + +It is possible to define your own pipeline and render stages though the latter requires knowledge of some GLMakie internals and OpenGL. +Explaining those is outside the scope of this section. +Instead we provide a small example you can use as a starting point. +You may also want to look at "GLMakie/src/postprocessing.jl" and "GLMakie/assets/postprocessors" to reference the existing implementations of pipeline stages. + +As an example we will create a stage that applies a Sepia effect to our figure. +Lets begin by setting up a pipeline that includes the stage. + +```@example RenderPipeline +using Makie + +# Create an empty Pipeline +pipeline = Makie.RenderPipeline() + +# Add stages in order +render1 = push!(pipeline, Makie.RenderStage(transparency = false)) +render2 = push!(pipeline, Makie.TransparentRenderStage()) +oit = push!(pipeline, Makie.OITStage()) +fxaa = push!(pipeline, Makie.FXAAStage()) # includes FXAA1 & FXAA2 with color_luma connection + +# Our new stage takes a 32bit color and produces a new 32 bit color +color_tint = Makie.Stage(:Tint, + # BufferFormat defaults to 4x N0f8, i.e. 32Bit color + inputs = [:color => Makie.BufferFormat()], + outputs = [:color => Makie.BufferFormat()], + color_transform = Observable(Makie.Mat3f( + # sepia filter + 0.393, 0.349, 0.272, + 0.769, 0.686, 0.534, + 0.189, 0.168, 0.131 + )) +) +push!(pipeline, color_tint) +display_stage = push!(pipeline, Makie.DisplayStage()) + +# Connect stages +connect!(pipeline, render1, fxaa) # connect everything from render1 to FXAA1, FXAA2 +connect!(pipeline, render2, oit) # everything from OIT Render -> OIT +connect!(pipeline, :objectid) # connect all :objectid inputs and outputs +connect!(pipeline, oit, fxaa, :color) # OIT color output -> fxaa color input +connect!(pipeline, fxaa, color_tint, :color) # fxaa color output -> color tint input +connect!(pipeline, color_tint, display_stage, :color) # color tint -> display +``` + +Note that you can also create connections with `Makie.pipeline_gui!(ax, pipeline)`. + +When passing this pipeline to GLMakie, it will determine all the necessary buffers it needs to allocate. +Each stage will get a framebuffer based on the connected outputs of the pipeline and an `inputs` Dict based on the connected inputs of the pipeline. +The names match the names of the pipeline stage with a `_buffer` postfix. +What we now need to do is define a constructor for a `<: GLMakie.AbstractRenderStep` object which represents the stage in the `GLRenderPipeline`, and a `run_step()` method that executes the stage. + +```@example RenderPipeline +using GLMakie + +function GLMakie.construct(::Val{:Tint}, screen, framebuffer, inputs, parent) + # Create the shader that applies the Sepia effect. + # For the vertex shader we can reuse "fullscreen.vert", which sets up frag_uv. + # For the fragment shader we add a new shader here, which reads a color + # from an input texture, applies a transformation and writes it to the + # output color buffer. + frag_shader = """ + {{GLSL_VERSION}} + + in vec2 frag_uv; + out vec4 fragment_color; + + uniform sampler2D color_buffer; // \$(name of input)_buffer + uniform mat3 color_transform; // from Stage attributes + + void main(void) { + vec4 c = texture(color_buffer, frag_uv).rgba; + fragment_color = vec4(color_transform * c.rgb, c.a); + } + """ + + # Create a GLMakie Shader that will get compiled later + shader = GLMakie.LazyShader( + screen.shader_cache, + GLMakie.loadshader("postprocessing/fullscreen.vert"), + GLMakie.ShaderSource(frag_shader, :frag) + ) + + # `inputs` are unique to each stage, so we can directly pass them as inputs + # of the RenderObject. On top of the `color_buffer` generated from the + # stage input we add a `color_transform` which connects to the color_transform + # in our stage: + inputs[:color_transform] = parent.attributes[:color_transform] + + # We then create a RenderObject: + robj = GLMakie.RenderObject( + inputs, # will lower to uniforms (and vertex attributes/vertex shader inputs) + shader, # will be compiled to a shader program + GLMakie.PostprocessPrerender(), # turn of depth testing, blending and culling before rendering + nothing, # postrender function - set later + screen.glscreen # OpenGL context of the RenderObject + ) + # postrenderfunction is the "nothing" from before + # this produces a draw call with two triangles representing th full framebuffer + robj.postrenderfunction = () -> GLMakie.draw_fullscreen(robj.vertexarray.id) + + # Finally we create a "RenderPass" which simply contains the render object and framebuffer + return GLMakie.RenderPass{:Tint}(framebuffer, robj) +end + +function GLMakie.run_step(screen, glscene, step::GLMakie.RenderPass{:Tint}) + wh = size(step.framebuffer) + # This binds all color buffers in the framebuffer (i.e. our one color output) + GLMakie.set_draw_buffers(step.framebuffer) + GLMakie.glViewport(0, 0, wh[1], wh[2]) + GLMakie.GLAbstraction.render(step.robj) + return +end +``` + +With that done we can now use our new pipeline for rendering: + +```@figure RenderPipeline backend=GLMakie +using FileIO, GLMakie + +GLMakie.activate!(render_pipeline = pipeline) +cow = load(Makie.assetpath("cow.png")) +image(rotr90(cow)) +``` + +## WGLMakie + +WGLMakie does not (yet) support the RenderPipeline. +Instead it relies on Three.js for rendering and adjusts a few settings. +It sets `antialias = true` for all plots, ignoring `fxaa` attributes. +It keeps `sortObjects = true` which enables sorting of plots with transparency. +The `transparency` attribute only affects depth writes. +The renderloop boils down to: + +```julia +function render_scene(scene) + if !scene.visible; return end + set_region(scene.viewport) + if scene.clear; clear(color, depth) end + foreach(render, scene.plots) + foreach(render_scene, scene.children) +end +``` + +Note that unlike CairoMakie and GLMakie, WGLMakie renders scene by scene. +I.e. if a child scene clears, nothing from the parent scene will be visible in that area. + +### Plot & Scene Insertion + +WGLMakie follows the same plot and scene insertion logic as GLMakie. +This includes a new scene only getting added when a plot gets added to it. +The main difference to GLMakie is that plots and scenes get serialized so they can be transferred to JavaScript instead of getting added to a screen directly. + +```julia +function serialize_scene(scene) + return Dict( + :plots => map(plot -> serialize_plot(scene, plot), scene.plots) + :children => map(serialize_scene, scene.children), + # scene data... + ) +end +``` \ No newline at end of file diff --git a/src/Makie.jl b/src/Makie.jl index 5ae8e337191..4787b23e29a 100644 --- a/src/Makie.jl +++ b/src/Makie.jl @@ -123,6 +123,7 @@ end include("documentation/docstringextension.jl") include("utilities/quaternions.jl") include("utilities/stable-hashing.jl") +include("utilities/RenderPipeline.jl") include("bezier.jl") include("types.jl") include("utilities/Plane.jl") diff --git a/src/display.jl b/src/display.jl index 336c6028922..c2d05f26085 100644 --- a/src/display.jl +++ b/src/display.jl @@ -66,10 +66,27 @@ function set_screen_config!(backend::Module, new_values) return backend_defaults end -function merge_screen_config(::Type{Config}, config::Dict) where Config +function merge_screen_config(::Type{Config}, _config::Dict) where Config backend = parentmodule(Config) key = nameof(backend) backend_defaults = CURRENT_DEFAULT_THEME[key] + # To not deprecate ssao, fxaa, oit: + if key == :GLMakie + config = Dict{Symbol, Any}(_config) + get!(config, :render_pipeline) do + if any(in(keys(config)), [:ssao, :fxaa, :oit]) || + (to_value(backend_defaults[:render_pipeline]) == automatic) + ssao = to_value(get(config, :ssao, backend_defaults[:ssao])) + fxaa = to_value(get(config, :fxaa, backend_defaults[:fxaa])) + oit = to_value(get(config, :oit, backend_defaults[:oit])) + return default_pipeline(; ssao, fxaa, oit) + else + return to_value(backend_defaults[:render_pipeline]) + end + end + else + config = _config + end arguments = map(fieldnames(Config)) do name if haskey(config, name) return config[name] diff --git a/src/precompiles.jl b/src/precompiles.jl index c76599dafe5..b716e0f2859 100644 --- a/src/precompiles.jl +++ b/src/precompiles.jl @@ -37,6 +37,7 @@ let empty!(DEFAULT_FONT) empty!(ALTERNATIVE_FONTS) Makie.CURRENT_FIGURE[] = nothing + generate_buffers(default_pipeline()) end nothing end diff --git a/src/theming.jl b/src/theming.jl index a4aebd95c33..bfefa5fbc68 100644 --- a/src/theming.jl +++ b/src/theming.jl @@ -118,6 +118,7 @@ const MAKIE_DEFAULT_THEME = Attributes( oit = true, fxaa = true, ssao = false, + render_pipeline = automatic, # This adjusts a factor in the rendering shaders for order independent # transparency. This should be the same for all of them (within one rendering # pipeline) otherwise depth "order" will be broken. diff --git a/src/utilities/RenderPipeline.jl b/src/utilities/RenderPipeline.jl new file mode 100644 index 00000000000..da8060df6c1 --- /dev/null +++ b/src/utilities/RenderPipeline.jl @@ -0,0 +1,1049 @@ +# Note: This file could easily be moved out into a mini-package. + +import FixedPointNumbers: N0f8 + +module BFT + import FixedPointNumbers: N0f8 + + @enum BufferFormatType::UInt8 begin + float8 = 0; float16 = 1; float32 = 3; + int8 = 4; int16 = 5; int32 = 7; + uint8 = 8; uint16 = 9; uint32 = 11; + end + + # lowest 2 bits do variation between 8, 16 and 32 bit types, others do variation of base type + is_compatible(a::BufferFormatType, b::BufferFormatType) = (UInt8(a) & 0b11111100) == (UInt8(b) & 0b11111100) + + # assuming compatible types (max is a bitwise thing here btw) + _promote(a::BufferFormatType, b::BufferFormatType) = BufferFormatType(max(UInt8(a), UInt8(b))) + + # we matched the lowest 2 bits to bytesize + bytesize(x::BufferFormatType) = Int((UInt8(x) & 0b11) + 1) + + const type_lookup = (N0f8, Float16, Nothing, Float32, Int8, Int16, Nothing, Int32, UInt8, UInt16, Nothing, UInt32) + to_type(t::BufferFormatType) = type_lookup[Int(t) + 1] +end + + +struct BufferFormat + dims::Int + type::BFT.BufferFormatType + + minfilter::Symbol + magfilter::Symbol + repeat::NTuple{2, Symbol} + mipmap::Bool + # anisotropy::Float32 # useless for this context? + # local_read::Bool # flag so stage can mark that it only reads the pixel it will write to, i.e. allows using input as output + # multisample # for MSAA +end + +""" + BufferFormat([dims = 4, type = N0f8]; [texture_parameters...]) + +Creates a `BufferFormat` which encodes requirements for an input or output of a +`Stage`. For example, a color output may require 3 (RGB) N0f8's (8 bit "float" +normalized to a 0..1 range). + +The `BufferFormat` may also specify texture parameters for the buffer: +- `minfilter = :any`: How are pixels combined (:linear, :nearest, :nearest_mipmap_nearest, :linear_mipmap_nearest, :nearest_mipmap_linear, :linear_mipmap_linear) +- `magfilter = :any`: How are pixels interpolated (:linear, :nearest) +- `repeat = :clamp_to_edge`: How are pixels sampled beyond the boundary? (:clamp_to_edge, :mirrored_repeat, :repeat) +- `mipmap = false`: Should mipmaps be used? +""" +function BufferFormat( + dims::Integer, type::BFT.BufferFormatType; + minfilter = :any, magfilter = :any, + repeat = (:clamp_to_edge, :clamp_to_edge), + mipmap = false + ) + _repeat = ifelse(repeat isa Symbol, (repeat, repeat), repeat) + return BufferFormat(dims, type, minfilter, magfilter, _repeat, mipmap) +end +BufferFormat(dims = 4, type = N0f8; kwargs...) = BufferFormat(dims, type; kwargs...) +@generated function BufferFormat(dims::Integer, ::Type{T}; kwargs...) where {T} + type = BFT.BufferFormatType(UInt8(findfirst(x -> x === T, BFT.type_lookup)) - 0x01) + return :(BufferFormat(dims, $type; kwargs...)) +end + +function Base.:(==)(f1::BufferFormat, f2::BufferFormat) + return (f1.dims == f2.dims) && (f1.type == f2.type) && + (f1.minfilter == f2.minfilter) && (f1.magfilter == f2.magfilter) && + (f1.repeat == f2.repeat) && (f1.mipmap == f2.mipmap) +end + +""" + BufferFormat(f1::BufferFormat, f2::BufferFormat) + +Creates a new `BufferFormat` combining two given formats. For this the formats +need to be compatible, but not the same. + +Rules: +- The output size is `dims = max(f1.dims, f2.dims)` +- Types must come from the same base type (AbstractFloat, Signed, Unsigned) where N0f8 counts as a float. +- The output type is the larger one of the two +- minfilter and magfilter must match (if one is :any, the other is used) +- repeat must match +- mipmap = true takes precedence +""" +function BufferFormat(f1::BufferFormat, f2::BufferFormat) + if is_compatible(f1, f2) + dims = max(f1.dims, f2.dims) + type = BFT._promote(f1.type, f2.type) + # currently don't allow different min/magfilters + minfilter = ifelse(f1.minfilter == :any, f2.minfilter, f1.minfilter) + magfilter = ifelse(f1.magfilter == :any, f2.magfilter, f1.magfilter) + repeat = f1.repeat + mipmap = f1.mipmap || f2.mipmap + return BufferFormat(dims, type, minfilter, magfilter, repeat, mipmap) + else + error("Failed to merge BufferFormat: $f1 and $f2 are not compatible.") + end +end + +function is_compatible(f1::BufferFormat, f2::BufferFormat) + return BFT.is_compatible(f1.type, f2.type) && + (f1.minfilter == :any || f2.minfilter == :any || f1.minfilter == f2.minfilter) && + (f1.magfilter == :any || f2.magfilter == :any || f1.magfilter == f2.magfilter) && + (f1.repeat == f2.repeat) +end + +function format_to_type(format::BufferFormat) + eltype = BFT.to_type(format.type) + return format.dims == 1 ? eltype : Vec{format.dims, eltype} +end + + +struct Stage + name::Symbol + + # order matters for outputs + # these are "const" after init + inputs::Dict{Symbol, Int} + outputs::Dict{Symbol, Int} + input_formats::Vector{BufferFormat} + output_formats::Vector{BufferFormat} + + # const for caching (which does quite a lot actually) + attributes::Dict{Symbol, Any} # TODO: rename -> settings? +end + +""" + Stage(name::Symbol[; inputs = Pair{Symbol, BufferFormat}[], outputs = Pair{Symbol, BufferFormat}[]) + +Creates a new `Stage` from the given `name`, `inputs` and `outputs`. A `Stage` +represents an action taken during rendering, e.g. rendering (a subset of) render +objects, running a post processor or sorting render objects. +""" +function Stage(name; inputs = Pair{Symbol, BufferFormat}[], outputs = Pair{Symbol, BufferFormat}[], kwargs...) + return Stage(Symbol(name), + Dict{Symbol, Int}([k => idx for (idx, (k, v)) in enumerate(inputs)]), + BufferFormat[v for (k, v) in inputs], + Dict{Symbol, Int}([k => idx for (idx, (k, v)) in enumerate(outputs)]), + BufferFormat[v for (k, v) in outputs]; + kwargs... + ) +end +function Stage(name, inputs, input_formats, outputs, output_formats; kwargs...) + return Stage( + Symbol(name), + inputs, outputs, + input_formats, output_formats, + Dict{Symbol, Any}(pairs(kwargs)) + ) +end + +get_input_format(stage::Stage, key::Symbol) = stage.input_formats[stage.inputs[key]] +get_output_format(stage::Stage, key::Symbol) = stage.output_formats[stage.outputs[key]] + +function Base.:(==)(s1::Stage, s2::Stage) + # For two stages to be equal: + # name & format must match as they define the backend implementation and required buffers + # outputs don't strictly need to match as they are just names. + # inputs do as they are used as uniform names (could change) + # attributes probably should match because they change uniforms and compile time constants + # (this may allow us to merge pipelines later) + return (s1.name == s2.name) && (s1.input_formats == s2.input_formats) && + (s1.output_formats == s2.output_formats) && + (s1.inputs == s2.inputs) && (s1.outputs == s2.outputs) && + (s1.attributes == s2.attributes) +end + +struct RenderPipeline + stages::Vector{Stage} + + # (stage_idx, negative input index or positive output index) -> connection format index + stageio2idx::Dict{Tuple{Int, Int}, Int} + formats::Vector{BufferFormat} # of connections + # TODO: consider adding: + # endpoints::Vector{Tuple{Int, Int}} +end + +""" + RenderPipeline([stages::Stage...]) + +Creates a `RenderPipeline` from the given `stages` or an empty pipeline if none are +given. The pipeline represents a series of actions (stages) executed during +rendering. +""" +function RenderPipeline() + return RenderPipeline(Stage[], Dict{Tuple{Int, Int}, Int}(), BufferFormat[]) +end +function RenderPipeline(stages::Stage...) + pipeline = RenderPipeline() + foreach(stage -> push!(pipeline, stage), stages) + return pipeline +end + + +# Stages are allowed to be duplicates. E.g. you could have something like this: +# render -> effect 1 -. +# |-> combine +# render -> effect 2 -' +# where render is the same (name/task, inputs, outputs) +function Base.push!(pipeline::RenderPipeline, stage::Stage) + push!(pipeline.stages, stage) + return stage # for convenience +end +function Base.push!(pipeline::RenderPipeline, other::RenderPipeline) + N = length(pipeline.stages); M = length(pipeline.formats) + append!(pipeline.stages, other.stages) + for ((stage_idx, io_idx), format_idx) in other.stageio2idx + pipeline.stageio2idx[(stage_idx + N, io_idx)] = M + format_idx + end + append!(pipeline.formats, other.formats) + return other # for convenience +end + + +""" + connect!(pipeline::RenderPipeline, source::Union{RenderPipeline, Stage}, target::Union{RenderPipeline, Stage}) + +Connects every output in `source` to every input in `target` that shares the +same name. For example, if `:a, :b, :c, :d` exist in source and `:b, :d, :e` +exist in target, `:b, :d` will get connected. +""" +function Observables.connect!(pipeline::RenderPipeline, src::Union{RenderPipeline, Stage}, trg::Union{RenderPipeline, Stage}) + stages(pipeline::RenderPipeline) = pipeline.stages + stages(stage::Stage) = [stage] + + outputs = Set(mapreduce(stage -> keys(stage.outputs), union, stages(src))) + inputs = Set(mapreduce(stage -> keys(stage.inputs), union, stages(trg))) + for key in intersect(outputs, inputs) + connect!(pipeline, src, trg, key) + end + return +end + +""" + connect!(pipeline::RenderPipeline, [source = pipeline, target = pipeline], name::Symbol) + +Connects every output in `source` that uses the given `name` to every input in +`target` with the same `name`. `source` and `target` can be a pipeline, stage +or integer referring to stage in `pipeline`. If both are omitted inputs and +outputs from `pipeline` get connected. +""" +function Observables.connect!(pipeline::RenderPipeline, src::Union{RenderPipeline, Stage, Integer}, trg::Union{RenderPipeline, Stage, Integer}, key::Symbol) + return connect!(pipeline, src, key, trg, key) +end +Observables.connect!(pipeline::RenderPipeline, key::Symbol) = connect!(pipeline, pipeline, key, pipeline, key) + +# TODO: Not sure about this... Maybe it should be first/last instead? But what +# then it wouldn't really work with e.g. SSAO, which needs color as an +# input in step 2. +""" + connect!(pipeline, source, output, target, input) + +Connects an `output::Union{Symbol, Integer}` from `source::Union{Stage, Integer}` +to an `input` of `target`. If either already has a connection the new connection +will be merged with the old. The source and target stage as well as the pipeline +will be updated appropriately. + +`source` and `target` can also be `RenderPipeline`s if both output and input are +`Symbol`s. In this case every stage in source with an appropriately named output +is connected to every stage in target with an appropriately named input. Use with +caution. +""" +function Observables.connect!(pipeline::RenderPipeline, + src::Union{RenderPipeline, Stage}, output::Symbol, + trg::Union{RenderPipeline, Stage}, input::Symbol + ) + + iterable(pipeline::RenderPipeline) = pipeline.stages + iterable(stage::Stage) = Ref(stage) + + for source in iterable(src) + if haskey(source.outputs, output) + output_idx = source.outputs[output] + for target in iterable(trg) + if haskey(target.inputs, input) + input_idx = target.inputs[input] + connect!(pipeline, source, output_idx, target, input_idx) + end + end + end + end + + return +end + +function Observables.connect!(pipeline::RenderPipeline, src::Integer, output::Symbol, trg::Integer, input::Symbol) + return connect!(pipeline, src, output, trg, input) +end +function Observables.connect!(pipeline::RenderPipeline, source::Stage, output::Integer, target::Stage, input::Integer) + src = findfirst(x -> x === source, pipeline.stages) + trg = findfirst(x -> x === target, pipeline.stages) + return connect!(pipeline, src, output, trg, input) +end + +function Observables.connect!(pipeline::RenderPipeline, source::Stage, output::Symbol, target::Stage, input::Symbol) + haskey(source.outputs, output) || error("output $output does not exist in source stage") + haskey(target.inputs, input) || error("input $input does not exist in target stage") + output_idx = source.outputs[output] + input_idx = target.inputs[input] + return connect!(pipeline, source, output_idx, target, input_idx) +end + +function Observables.connect!(pipeline::RenderPipeline, src::Integer, output::Integer, trg::Integer, input::Integer) + @boundscheck begin + checkbounds(pipeline.stages, src) + checkbounds(pipeline.stages, trg) + end + + @boundscheck begin + checkbounds(pipeline.stages[src].output_formats, output) + checkbounds(pipeline.stages[trg].input_formats, input) + end + + # Don't make a new connection if the connection already exists + # (the format must be correct if it exists) + if get(pipeline.stageio2idx, (src, output), 0) === + get(pipeline.stageio2idx, (trg, -input), -1) + return + end + + # format for the requested connection + format = BufferFormat(pipeline.stages[src].output_formats[output], pipeline.stages[trg].input_formats[input]) + + # Resolve format and update existing connection of src & trg + if haskey(pipeline.stageio2idx, (src, output)) + # at least src exists, update format + format_idx = pipeline.stageio2idx[(src, output)] + format = BufferFormat(pipeline.formats[format_idx], format) + pipeline.formats[format_idx] = format + + if haskey(pipeline.stageio2idx, (trg, -input)) + # both exist - update + other_idx = pipeline.stageio2idx[(trg, -input)] + format = BufferFormat(pipeline.formats[other_idx], format) + # replace format of lower index + format_idx, other_idx = minmax(format_idx, other_idx) + pipeline.formats[format_idx] = format + # connect higher index to format and adjust later indices + for (k, v) in pipeline.stageio2idx + pipeline.stageio2idx[k] = ifelse(v == other_idx, format_idx, ifelse(v > other_idx, v - 1, v)) + end + # remove orphaned format + deleteat!(pipeline.formats, other_idx) + else + # src exists, trg doesn't -> connect trg + pipeline.stageio2idx[(trg, -input)] = format_idx + end + + elseif haskey(pipeline.stageio2idx, (trg, -input)) + + # src doesn't exist, trg does, modify target and connect src + format_idx = pipeline.stageio2idx[(trg, -input)] + format = BufferFormat(pipeline.formats[format_idx], format) + pipeline.formats[format_idx] = format + pipeline.stageio2idx[(src, output)] = format_idx + + else + + # neither exists, add new and connect both + push!(pipeline.formats, format) + pipeline.stageio2idx[(src, output)] = length(pipeline.formats) + pipeline.stageio2idx[(trg, -input)] = length(pipeline.formats) + + end + + return +end + +format_complexity(f::BufferFormat) = format_complexity(f.dims, f.type) +format_complexity(dims, type) = dims * BFT.bytesize(type) +# complexity of merged, not max of either +format_complexity(f1::BufferFormat, f2::BufferFormat) = max(f1.dims, f2.dims) * max(BFT.bytesize(f1.type), BFT.bytesize(f2.type)) + +""" + generate_buffers(pipeline) + +Maps the connections in the given pipeline to a vector of buffer formats and +returns them together with a connection-to-index map. This will attempt to +optimize buffers for the lowest memory overhead. I.e. it will reuse buffers for +multiple connections and upgrade them if it is cheaper than creating a new one. +""" +function generate_buffers(pipeline::RenderPipeline) + # Verify that outputs are continuously connected (i.e. if N then 1..N-1 as well) + output_max = zeros(Int, length(pipeline.stages)) + output_sum = zeros(Int, length(pipeline.stages)) + for (stage_idx, io_idx) in keys(pipeline.stageio2idx) + io_idx > 0 || continue # inputs irrelevant + output_max[stage_idx] = max(output_max[stage_idx], io_idx) + output_sum[stage_idx] = output_sum[stage_idx] + io_idx # or use max(0, io_idx) and skip continue? + end + # sum must be 1 + 2 + ... + n = n(n+1)/2 + for i in eachindex(output_sum) + s = output_sum[i]; m = output_max[i] + if s != div(m*(m+1), 2) + error("Stage $i has an incomplete set of output connections.") + end + end + + # Group connections that exist between stages + endpoints = [(999_999, 0) for _ in pipeline.formats] + for ((stage_idx, io_idx), format_idx) in pipeline.stageio2idx + start, stop = endpoints[format_idx] + start = min(start, stage_idx + (io_idx < 0) * 999_999) # always pick start if stage_idx is input + stop = max(stop, (io_idx < 0) * stage_idx - 1) # pick stop if io_idx is output + endpoints[format_idx] = (start, stop) + end + filter!(x -> x != (999_999, 0), endpoints) + + usage_per_transfer = [Int[] for _ in 1:length(pipeline.stages)-1] + for (conn_idx, (start, stop)) in enumerate(endpoints) + start <= stop || error("Connection $conn_idx is read before it is written to. $start $stop") + for i in start:stop + push!(usage_per_transfer[i], conn_idx) + end + end + + buffers = BufferFormat[] + conn2merged = fill(-1, length(pipeline.formats)) + needs_buffer = Int[] + available = Int[] + + # Let's simplify to get correct behavior first... + + for i in eachindex(usage_per_transfer) + # prepare: + # - collect connections without buffers + # - collect available buffers (not in use now or last iteration) + copyto!(resize!(available, length(buffers)), eachindex(buffers)) + empty!(needs_buffer) + for j in max(1, i-1):i + # for j in i:min(length(usage_per_transfer), i+1) # reverse + for conn_idx in usage_per_transfer[j] + if conn2merged[conn_idx] != -1 + idx = conn2merged[conn_idx] + filter!(!=(idx), available) + elseif j == i + push!(needs_buffer, conn_idx) + end + end + end + + # Handle most expensive connections first + sort!(needs_buffer, by = i -> format_complexity(pipeline.formats[i]), rev = true) + + for conn_idx in needs_buffer + # search for most compatible buffer + best_match = 0 + prev_comp = 999999 + prev_delta = 999999 + conn_format = pipeline.formats[conn_idx] + conn_comp = format_complexity(conn_format) + for i in available + if buffers[i] == conn_format # exact match + best_match = i + break + elseif is_compatible(buffers[i], conn_format) + # found compatible buffer, but we only use it if + # - using it is cheaper than using the last + # - using it is cheaper than creating a new buffer + # - it is more compatible than the last when both are 0 cost + # (i.e prefer 3, Float16 over 3 Float8 for 3 Float16 target) + updated_comp = format_complexity(buffers[i], conn_format) + buffer_comp = format_complexity(buffers[i]) + delta = updated_comp - buffer_comp + is_cheaper = (delta < prev_delta) && (delta <= conn_comp) + more_compatible = (delta == prev_delta == 0) && (buffer_comp < prev_comp) + if is_cheaper || more_compatible + best_match = i + prev_comp = updated_comp + prev_delta = delta + end + end + end + + if best_match == 0 + # nothing compatible found/available, add format + push!(buffers, conn_format) + best_match = length(buffers) + elseif buffers[best_match] != conn_format + # found upgradeable format, upgrade it (or use it) + new_format = BufferFormat(buffers[best_match], conn_format) + buffers[best_match] = new_format + end + + # Link to new found or upgraded format + conn2merged[conn_idx] = best_match + + # Can't use a buffer twice in one transfer + filter!(!=(best_match), available) + end + end + + return buffers, conn2merged +end + + +################################################################################ +### show +################################################################################ + + +function Base.show(io::IO, format::BufferFormat) + print(io, "BufferFormat($(format.dims), $(format.type)") + for (k, v) in format.extras + print(io, ", :", k, " => ", v) + end + print(io, ")") +end + +Base.show(io::IO, stage::Stage) = print(io, "Stage($(stage.name))") +function Base.show(io::IO, ::MIME"text/plain", stage::Stage) + print(io, "Stage($(stage.name))") + + if !isempty(stage.inputs) + print(io, "\ninputs:") + ks = collect(keys(stage.inputs)) + sort!(ks, by = k -> stage.inputs[k]) + pad = mapreduce(k -> length(string(k)), max, ks) + for (i, k) in enumerate(ks) + print(io, "\n [?] ", lpad(string(k), pad), "::", stage.input_formats[i]) + end + end + + if !isempty(stage.outputs) + print(io, "\noutputs:") + ks = collect(keys(stage.outputs)) + sort!(ks, by = k -> stage.outputs[k]) + pad = mapreduce(k -> length(string(k)), max, ks) + for (i, k) in enumerate(ks) + print(io, "\n [?] ", lpad(string(k), pad), "::", stage.output_formats[i]) + end + end + + if !isempty(stage.attributes) + print(io, "\nattributes:") + pad = mapreduce(k -> length(":$k"), max, keys(stage.attributes)) + for (k, v) in pairs(stage.attributes) + print(io, "\n ", lpad(":$k", pad), " = ", v) + end + end + + return +end + +function Base.show(io::IO, ::MIME"text/plain", pipeline::RenderPipeline) + return show_resolved(io, pipeline, pipeline.formats, collect(eachindex(pipeline.formats))) +end + +function show_resolved(pipeline::RenderPipeline, buffers, remap) + return show_resolved(stdout, pipeline, buffers, remap) +end + +function show_resolved(io::IO, pipeline::RenderPipeline, buffers, remap) + println(io, "RenderPipeline():") + print(io, "Stages:") + pad = isempty(buffers) ? 0 : 1 + floor(Int, log10(length(buffers))) + + for (stage_idx, stage) in enumerate(pipeline.stages) + print(io, "\n Stage($(stage.name))") + + if !isempty(stage.input_formats) + print(io, "\n inputs: ") + strs = map(eachindex(stage.input_formats)) do i + k = findfirst(==(i), stage.inputs) + if haskey(pipeline.stageio2idx, (stage_idx, -i)) + conn = string(remap[pipeline.stageio2idx[(stage_idx, -i)]]) + else + conn = " " + end + return "[$conn] $k" + end + while length(strs) > 0 && startswith(last(strs), "[ ]") + pop!(strs) + end + join(io, strs, ", ") + end + + if !isempty(stage.output_formats) + print(io, "\n outputs: ") + strs = map(eachindex(stage.output_formats)) do i + k = findfirst(==(i), stage.outputs) + if haskey(pipeline.stageio2idx, (stage_idx, i)) + conn = string(remap[pipeline.stageio2idx[(stage_idx, i)]]) + else + conn = "#undef" + end + return "[$conn] $k" + end + while length(strs) > 0 && startswith(last(strs), "[#undef]") + pop!(strs) + end + join(io, strs, ", ") + end + end + + println(io, "\nConnection Formats:") + for (i, c) in enumerate(buffers) + s = lpad("$i", pad) + println(io, " [$s] ", c) + end + + return +end + + +################################################################################ +### Defaults +################################################################################ + + +SortStage() = Stage(:ZSort) + +function RenderStage(; kwargs...) + outputs = Dict(:color => 1, :objectid => 2, :position => 3, :normal => 4) + output_formats = [BufferFormat(4, N0f8), BufferFormat(2, UInt32), BufferFormat(3, Float16), BufferFormat(3, Float16)] + return Stage(:Render, Dict{Symbol, Int}(), BufferFormat[], outputs, output_formats; kwargs...) +end + +function TransparentRenderStage() + outputs = Dict(:color_sum => 1, :objectid => 2, :transmittance => 3) + output_formats = [BufferFormat(4, Float16), BufferFormat(2, UInt32), BufferFormat(1, N0f8)] + return Stage(Symbol("OIT Render"), Dict{Symbol, Int}(), BufferFormat[], outputs, output_formats) +end + +function SSAOStage(; kwargs...) + inputs = Dict(:position => 1, :normal => 2) + input_formats = [BufferFormat(3, Float32), BufferFormat(3, Float16)] + stage1 = Stage(:SSAO1, inputs, input_formats, Dict(:occlusion => 1), [BufferFormat(1, N0f8)]; kwargs...) + + inputs = Dict(:occlusion => 1, :color => 2, :objectid => 3) + input_formats = [BufferFormat(1, N0f8), BufferFormat(4, N0f8), BufferFormat(2, UInt32)] + stage2 = Stage(:SSAO2, inputs, input_formats, Dict(:color => 1), [BufferFormat()]; kwargs...) + + pipeline = RenderPipeline(stage1, stage2) + connect!(pipeline, stage1, 1, stage2, 1) + + return pipeline +end + +function OITStage(; kwargs...) + inputs = Dict(:color_sum => 1, :transmittance => 2) + input_formats = [BufferFormat(4, Float16), BufferFormat(1, N0f8)] + outputs = Dict(:color => 1) + output_formats = [BufferFormat(4, N0f8)] + return Stage(:OIT, inputs, input_formats, outputs, output_formats; kwargs...) +end + +function FXAAStage(; kwargs...) + stage1 = Stage(:FXAA1, + Dict(:color => 1, :objectid => 2), [BufferFormat(4, N0f8), BufferFormat(2, UInt32)], + Dict(:color_luma => 1), [BufferFormat(4, N0f8)]; kwargs... + ) + + stage2 = Stage(:FXAA2, + Dict(:color_luma => 1), [BufferFormat(4, N0f8, minfilter = :linear)], + Dict(:color => 1), [BufferFormat(4, N0f8)]; kwargs... + ) + + pipeline = RenderPipeline(stage1, stage2) + connect!(pipeline, stage1, 1, stage2, 1) + + return pipeline +end + +function DisplayStage() + return Stage(:Display, + Dict(:color => 1, :objectid => 2), [BufferFormat(4, N0f8), BufferFormat(2, UInt32)], + Dict{Symbol, Int}(), BufferFormat[]) +end + + +function default_pipeline(; ssao = false, fxaa = true, oit = true) + pipeline = RenderPipeline() + push!(pipeline, SortStage()) + + # Note - order important! + # TODO: maybe add insert!()? + if ssao + render1 = push!(pipeline, RenderStage(ssao = true, transparency = false)) + _ssao = push!(pipeline, SSAOStage()) + render2 = push!(pipeline, RenderStage(ssao = false, transparency = false)) + else + render2 = push!(pipeline, RenderStage(transparency = false)) + end + if oit + render3 = push!(pipeline, TransparentRenderStage()) + _oit = push!(pipeline, OITStage()) + else + render3 = push!(pipeline, RenderStage(transparency = true)) + end + if fxaa + _fxaa = push!(pipeline, FXAAStage(filter_in_shader = true)) + end + display = push!(pipeline, DisplayStage()) + + + if ssao + connect!(pipeline, render1, _ssao) + connect!(pipeline, _ssao, fxaa ? _fxaa : display, :color) + end + connect!(pipeline, render2, fxaa ? _fxaa : display) + if oit + connect!(pipeline, render3, _oit) + connect!(pipeline, _oit, fxaa ? _fxaa : display, :color) + else + connect!(pipeline, render3, fxaa ? _fxaa : display, :color) + end + if fxaa + connect!(pipeline, _fxaa, display, :color) + end + connect!(pipeline, :objectid) + + return pipeline +end + +function test_pipeline_3D() + pipeline = RenderPipeline() + + render1 = push!(pipeline, RenderStage(ssao = true, transparency = false, fxaa = true)) + ssao = push!(pipeline, SSAOStage()) + render2 = push!(pipeline, RenderStage(ssao = false, transparency = false, fxaa = true)) + render3 = push!(pipeline, TransparentRenderStage()) + oit = push!(pipeline, OITStage()) + fxaa = push!(pipeline, FXAAStage(filter_in_shader = false)) + # dedicated fxaa = false render improves sdf (lines, scatter, text) intersections with FXAA + render4 = push!(pipeline, RenderStage(transparency = false, fxaa = false)) + display = push!(pipeline, DisplayStage()) + + connect!(pipeline, render1, ssao) + connect!(pipeline, ssao, fxaa, :color) + connect!(pipeline, render2, fxaa, :color) + connect!(pipeline, render3, oit) + connect!(pipeline, oit, fxaa, :color) + connect!(pipeline, fxaa, display, :color) + connect!(pipeline, render4, display) + connect!(pipeline, :objectid) + + return pipeline +end + +function test_pipeline_2D() + pipeline = RenderPipeline() + + # dedicated fxaa = false pass + no SSAO + render1 = push!(pipeline, RenderStage(transparency = false, fxaa = true)) + render2 = push!(pipeline, TransparentRenderStage()) + oit = push!(pipeline, OITStage()) + fxaa = push!(pipeline, FXAAStage(filter_in_shader = false)) + render3 = push!(pipeline, RenderStage(transparency = false, fxaa = false)) + display = push!(pipeline, DisplayStage()) + + connect!(pipeline, render1, fxaa) + connect!(pipeline, render2, oit) + connect!(pipeline, oit, fxaa, :color) + connect!(pipeline, fxaa, display, :color) + connect!(pipeline, render3, display) + connect!(pipeline, :objectid) + + return pipeline +end + +function test_pipeline_GUI() + pipeline = RenderPipeline() + + # GUI elements don't need OIT because they are (usually?) layered plot by + # plot, rather than element by element. Do need FXAA occasionally, e.g. Toggle + render1 = push!(pipeline, RenderStage(fxaa = true)) + fxaa = push!(pipeline, FXAAStage(filter_in_shader = false)) + render2 = push!(pipeline, RenderStage(fxaa = false)) + display = push!(pipeline, DisplayStage()) + + connect!(pipeline, render1, display) + connect!(pipeline, render1, fxaa) + connect!(pipeline, fxaa, display) + connect!(pipeline, render2, display) + + return pipeline +end + +function test_pipeline_minimal() + pipeline = RenderPipeline() + + render = push!(pipeline, RenderStage()) + display = push!(pipeline, DisplayStage()) + connect!(pipeline, render, display) + + return pipeline +end + +################################################################################ +## Experimental GUI for RenderPipeline +################################################################################ + +function pipeline_gui!(ax, pipeline) + width = 5 + + rects = Rect2f[] + header_line = Point2f[] + header = Tuple{String, Point2f}[] + marker_pos = Vector{Point2f}[] + input = Vector{Tuple{String, Point2f}}[] + output = Vector{Tuple{String, Point2f}}[] + stageio_lookup = Tuple{Int, Int}[] + + max_size = 0 + max_size2 = 0 + + for (idx, stage) in enumerate(pipeline.stages) + output_height = length(stage.outputs) + height = length(stage.inputs) + output_height + + r = Rect2f(-width/2, -height, width, height+2) + push!(rects, r) + push!(header_line, Point2f(-0.5width, 0), Point2f(0.5width, 0)) + push!(header, (string(stage.name), Point2f(0, 1))) + + ops = sort!([(string(s), Point2f(0.5width, 0.5-y)) for (s, y) in stage.outputs], by = x -> -x[2][2]) + ips = sort!( + [(string(s), Point2f(-0.5width, 0.5 - y - output_height)) for (s, y) in stage.inputs], + by = x -> -x[2][2]) + + push!(input, ips) + push!(output, ops) + push!(marker_pos, vcat(last.(ips), last.(ops))) + append!(stageio_lookup, [(idx, -i) for i in 1:length(stage.inputs)]) + append!(stageio_lookup, [(idx, i) for i in 1:length(stage.outputs)]) + + if max_size < length(stage.inputs) + length(stage.outputs) + max_size2 = max_size + max_size = length(stage.inputs) + length(stage.outputs) + end + end + + + origins = Observable([Point2f(8x, 0) for x in eachindex(pipeline.stages)]) + + # Do something to get better starting layout... + begin + shift = 0.5 * (4 + max_size + max_size2 + 1) + # vector[connection idx] = [(stage idx, input/output index)] (- input, + output) + conn2stageio = [Tuple{Int, Int}[] for _ in eachindex(pipeline.formats)] + for (stageio, conn) in pipeline.stageio2idx + push!(conn2stageio[conn], stageio) + end + + for i in length(pipeline.stages):-1:1 + targets = Int[] + for j in eachindex(pipeline.stages[i].input_formats) + if haskey(pipeline.stageio2idx, (i, -j)) + conn_idx = pipeline.stageio2idx[(i, -j)] + for (stage_idx, io) in conn2stageio[conn_idx] + if io > 0 # is output + push!(targets, stage_idx) + end + end + end + end + + if !isempty(targets) + y0 = 0.5 * (length(targets)+1) + for (j, stage_idx) in enumerate(targets) + origins[][stage_idx] = Point2f(origins[][stage_idx][1], origins[][i][2]) + Point2f(0, shift * (y0 - j)) + end + end + end + end + + rects_obs = Observable(Rect2f[]) + header_line_obs = Observable(Point2f[]) + header_obs = Observable(Tuple{String, Point2f}[]) + marker_pos_obs = Observable(Point2f[]) + input_obs = Observable(Tuple{String, Point2f}[]) + output_obs = Observable(Tuple{String, Point2f}[]) + path_ps_obs = Observable(Point2f[]) + path_cs_obs = Observable(RGBf[]) + + on(origins) do origins + rects_obs[] = rects .+ origins + header_line_obs[] = [origins[i] .+ header_line[2(i-1) + j] for i in eachindex(origins) for j in 1:2] + header_obs[] = map((x, pos) -> (x[1], pos + x[2]), header, origins) + + marker_pos_obs[] = mapreduce(vcat, marker_pos, origins) do ps, pos + return [p + pos for p in ps] + end + input_obs[] = mapreduce(vcat, input, origins) do input, pos + return [(x[1], x[2] + pos) for x in input] + end + output_obs[] = mapreduce(vcat, output, origins) do input, pos + return [(x[1], x[2] + pos) for x in input] + end + end + + function bezier_connect(p0, p1) + x0, y0 = ifelse(p0[1] < p1[1], p0, p1) + x1, y1 = ifelse(p0[1] < p1[1], p1, p0) + mid = 0.5 * (x0+x1) + path = Any[Makie.MoveTo(x0, y0)] + if (x1 - x0) > 10 + push!(path, + Makie.LineTo(Point2f(mid - 5, y0)), + Makie.CurveTo(Point2f(mid + 1, y0), Point2f(mid - 1, y1), Point2f(mid + 5, y1)), + Makie.LineTo(Point2f(x1, y1)) + ) + else + push!(path, Makie.CurveTo(Point2f(mid + 1, y0), Point2f(mid - 1, y1), Point2f(x1, y1))) + end + path = Makie.BezierPath(path) + return Makie.convert_arguments(PointBased(), path)[1] + end + + on(origins) do origins + # vector[connection idx] = [(stage idx, input/output index)] (- input, + output) + conn2stageio = [Tuple{Int, Int}[] for _ in eachindex(pipeline.formats)] + for (stageio, conn) in pipeline.stageio2idx + push!(conn2stageio[conn], stageio) + end + + paths = Point2f[] + color_pool = Makie.to_colormap(:seaborn_bright) + cs = RGBf[] + for (idx, stageio) in enumerate(conn2stageio) + sort!(stageio, by = first) + N = length(stageio) + for i in 1:N-1 + start_stage, start_idx = stageio[i] + start_idx < 0 && continue # is a stage input + for j in i+1:N + stop_stage, stop_idx = stageio[j] + stop_idx > 0 && continue # is a stage output + p0 = output[start_stage][start_idx][2] + origins[start_stage] + p1 = input[stop_stage][-stop_idx][2] + origins[stop_stage] + ps = bezier_connect(p0, p1) + append!(paths, ps) + push!(paths, Point2f(NaN)) + append!(cs, [color_pool[mod1(idx, end)] for _ in ps]) + push!(cs, RGBf(0,0,0)) + end + end + end + path_ps_obs.val = paths + path_cs_obs[] = cs + notify(path_ps_obs) + return + end + + notify(origins) + + scale = map(pv -> max(0.5, 10*min(pv[1,1], pv[2,2])), get_scene(ax).camera.projectionview) + + poly!(ax, rects_obs, strokewidth = scale, strokecolor = :black, fxaa = false, + shading = NoShading, color = :lightgray, transparency = false) + linesegments!(ax, header_line_obs, linewidth = scale, color = :black) + text!(ax, header_obs, markerspace = :data, fontsize = 0.8, color = :black, + align = (:center, :center)) + + text!(ax, output_obs, markerspace = :data, fontsize = 0.75, color = :black, + align = (:right, :center), offset = (-0.25, 0)) + text!(ax, input_obs, markerspace = :data, fontsize = 0.75, color = :black, + align = (:left, :center), offset = (0.25, 0)) + + p = scatter!(ax, marker_pos_obs, color = :black, markerspace = :data, marker = Circle, markersize = 0.3) + translate!(p, 0, 0, 1) + + p = lines!(ax, path_ps_obs, color = path_cs_obs) + translate!(p, 0, 0, -1) + + new_conn_plot = lines!(ax, Point2f[], color = :black, visible = false) + + # Drag Stages around & connect inputs/outputs + selected_idx = Ref(-1) + connecting = Ref(false) # true = drawing line - false = moving stage + drag_offset = Ref(Point2f(0)) + start_pos = Ref(Point2f(0)) + io_range = 0.4 + on(events(ax).mousebutton, priority = 100) do event + if event.button == Mouse.left + if event.action == Mouse.press + pos = mouseposition(ax) + for (i, p) in enumerate(marker_pos_obs[]) + if norm(pos - p) < io_range + selected_idx[] = i + connecting[] = true + start_pos[] = p + return Consume(true) + end + end + for (i, rect) in enumerate(rects_obs[]) + if pos in rect + selected_idx[] = i + connecting[] = false + drag_offset[] = origins[][i] - pos + return Consume(true) + end + end + elseif (event.action == Mouse.release) && (selected_idx[] != -1) + if connecting[] + pos = mouseposition(ax) + for (i, p) in enumerate(marker_pos_obs[]) + if norm(pos - p) < io_range + start_stage, start_io = stageio_lookup[selected_idx[]] + stop_stage, stop_io = stageio_lookup[i] + @info start_stage, start_io, stop_stage, stop_io + if (start_io > 0) && (stop_io < 0) && (start_stage < stop_stage) # output to input + Makie.connect!(pipeline, start_stage, start_io, stop_stage, -stop_io) + notify(origins) # trigger redraw of connections + elseif (start_io < 0) && (stop_io > 0) && (stop_stage < start_stage) # input to output + Makie.connect!(pipeline, stop_stage, stop_io, start_stage, -start_io) + notify(origins) # trigger redraw of connections + end + selected_idx[] = -1 + new_conn_plot.visible[] = false + return Consume(true) + end + end + end + new_conn_plot.visible[] = false + selected_idx[] = -1 + return Consume(true) + end + end + return Consume(false) + end + + on(events(ax).mouseposition, priority = 100) do event + if selected_idx[] != -1 + curr = mouseposition(ax) + if connecting[] + new_conn_plot[1][] = bezier_connect(start_pos[], curr) + new_conn_plot.visible[] = true + else + origins[][selected_idx[]] = curr + drag_offset[] + notify(origins) + end + return Consume(true) + end + return Consume(false) + end + + on(events(ax).keyboardbutton, priority = 100) do event + if (event.key == Keyboard.o) && (event.action == Keyboard.press) + @info origins[] + end + end + +end \ No newline at end of file diff --git a/test/render_pipeline.jl b/test/render_pipeline.jl new file mode 100644 index 00000000000..ef02590731b --- /dev/null +++ b/test/render_pipeline.jl @@ -0,0 +1,309 @@ +using Makie +using Makie: BufferFormat, N0f8, is_compatible, BFT +using Makie: Stage, get_input_format, get_output_format +using Makie: RenderPipeline, connect! +using Makie: generate_buffers, default_pipeline + +@testset "Render RenderPipeline" begin + + @testset "BufferFormat" begin + @testset "Constructors" begin + f = BufferFormat() + @test f.dims == 4 + @test f.type == BFT.float8 + @test f.minfilter == :any + @test f.magfilter == :any + @test f.repeat == (:clamp_to_edge, :clamp_to_edge) + @test f.mipmap == false + + f = BufferFormat(1, Float16, minfilter = :linear, magfilter = :nearest, mipmap = true, repeat = :repeat) + @test f.dims == 1 + @test f.type == BFT.float16 + @test f.minfilter == :linear + @test f.magfilter == :nearest + @test f.repeat == (:repeat, :repeat) + @test f.mipmap == true + end + + types = [(N0f8, Float16, Float32), (Int8, Int16, Int32), (UInt8, UInt16, UInt32)] + groups = [[BufferFormat(rand(1:4), T) for T in types[i]] for i in 1:3] + + @testset "is_compatible" begin + # All types that should or should not be compatible + for i in 1:3, j in 1:3 + for a in groups[i], b in groups[j] + @test (i == j) == is_compatible(a, b) + @test (i == j) == is_compatible(b, a) + end + end + + # extras + @test is_compatible(BufferFormat(mipmap = true), BufferFormat(mipmap = false)) + + @test is_compatible(BufferFormat(repeat = :repeat), BufferFormat(repeat = :repeat)) + @test !is_compatible(BufferFormat(repeat = :repeat), BufferFormat(repeat = :clamp_to_egde)) + + @test is_compatible(BufferFormat(minfilter = :any), BufferFormat(minfilter = :any)) + @test is_compatible(BufferFormat(minfilter = :any), BufferFormat(minfilter = :linear)) + @test is_compatible(BufferFormat(minfilter = :linear), BufferFormat(minfilter = :linear)) + @test !is_compatible(BufferFormat(minfilter = :nearest), BufferFormat(minfilter = :linear)) + + @test is_compatible(BufferFormat(magfilter = :any), BufferFormat(magfilter = :any)) + @test is_compatible(BufferFormat(magfilter = :any), BufferFormat(magfilter = :linear)) + @test is_compatible(BufferFormat(magfilter = :linear), BufferFormat(magfilter = :linear)) + @test !is_compatible(BufferFormat(magfilter = :nearest), BufferFormat(magfilter = :linear)) + + @test is_compatible(BufferFormat(2, Int8, minfilter = :any, magfilter = :linear), BufferFormat(1, Int32, minfilter = :nearest)) + end + + @testset "BufferFormat merging" begin + for i in 1:3, j in 1:3 + for m in 1:3, n in 1:3 + a = groups[i][m]; b = groups[j][n] + if i == j + expected = BufferFormat(max(a.dims, b.dims), types[i][max(m, n)]) + @test BufferFormat(a, b) == expected + @test BufferFormat(b, a) == expected + else + @test_throws ErrorException BufferFormat(a, b) + @test_throws ErrorException BufferFormat(b, a) + end + end + end + + a = BufferFormat(minfilter = :any, magfilter = :any, mipmap = false) + b = BufferFormat(minfilter = :linear, magfilter = :nearest, mipmap = true) + B = BufferFormat(a, b) + @test B.minfilter == :linear + @test B.magfilter == :nearest + @test B.mipmap + + B = BufferFormat(b, a) + @test B.minfilter == :linear + @test B.magfilter == :nearest + @test B.mipmap + + a = BufferFormat(minfilter = :nearest, magfilter = :linear, mipmap = true, repeat = :repeat) + b = BufferFormat(minfilter = :nearest, magfilter = :linear, mipmap = true, repeat = :repeat) + B = BufferFormat(a, b) + @test B.minfilter == :nearest + @test B.magfilter == :linear + @test B.mipmap + @test B.repeat == (:repeat, :repeat) + + @test_throws ErrorException BufferFormat(BufferFormat(repeat = :a), BufferFormat(repeat = :b)) + @test_throws ErrorException BufferFormat(BufferFormat(minfilter = :a), BufferFormat(minfilter = :b)) + @test_throws ErrorException BufferFormat(BufferFormat(magfilter = :a), BufferFormat(magfilter = :b)) + end + end + + + @testset "Stage" begin + @test_throws MethodError Stage() + @test Stage(:name) == Stage("name") + stage = Stage(:test) + @test stage.name == :test + @test stage.inputs == Dict{Symbol, Int}() + @test stage.outputs == Dict{Symbol, Int}() + @test stage.input_formats == BufferFormat[] + @test stage.output_formats == BufferFormat[] + @test stage.attributes == Dict{Symbol, Any}() + + stage = Stage(:test, + inputs = [:a => BufferFormat(), :b => BufferFormat(2)], + outputs = [:c => BufferFormat(1, Int8)], + attr = 17f0) + @test stage.name == :test + @test stage.inputs == Dict(:a => 1, :b => 2) + @test stage.outputs == Dict(:c => 1) + @test stage.input_formats == [BufferFormat(), BufferFormat(2)] + @test stage.output_formats == [BufferFormat(1, Int8)] + @test stage.attributes == Dict{Symbol, Any}(:attr => 17f0) + + @test get_input_format(stage, :a) == stage.input_formats[stage.inputs[:a]] + @test get_output_format(stage, :c) == stage.output_formats[stage.outputs[:c]] + end + + @testset "RenderPipeline & Connections" begin + pipeline = RenderPipeline() + + @test isempty(pipeline.stages) + @test isempty(pipeline.stageio2idx) + @test isempty(pipeline.formats) + + stage1 = Stage(:stage1, outputs = [:a => BufferFormat(), :b => BufferFormat(2)]) + stage2 = Stage(:stage2, + inputs = [:b => BufferFormat(2)], + outputs = [:c => BufferFormat(1, Int16)], + attr = 17f0) + stage3 = Stage(:stage3, inputs = [:b => BufferFormat(4, Float16), :c => BufferFormat(2, Int8)]) + stage4 = Stage(:stage4, inputs = [:x => BufferFormat()], outputs = [:y => BufferFormat()]) + stage5 = Stage(:stage5, inputs = [:z => BufferFormat()]) + + push!(pipeline, stage1) + push!(pipeline, stage2) + push!(pipeline, stage3) + push!(pipeline, stage4) + push!(pipeline, stage5) + + @test all(pipeline.stages .== [stage1, stage2, stage3, stage4, stage5]) + @test isempty(pipeline.stageio2idx) + @test isempty(pipeline.formats) + + for _ in 1:2 # also verify that double-connect doesn't ruin things + connect!(pipeline, stage1, stage2) + + @test length(pipeline.formats) == 1 + @test length(pipeline.stageio2idx) == 2 + @test pipeline.formats[end] == BufferFormat(2) + @test pipeline.stageio2idx[(1, 2)] == 1 + @test pipeline.stageio2idx[(2, -1)] == 1 + end + + connect!(pipeline, stage2, stage3, :c) + + @test length(pipeline.formats) == 2 + @test length(pipeline.stageio2idx) == 4 + @test pipeline.formats[end] == BufferFormat(2, Int16) + @test pipeline.stageio2idx[(2, 1)] == 2 + @test pipeline.stageio2idx[(3, -2)] == 2 + + connect!(pipeline, stage1, stage3, :b) + + @test length(pipeline.formats) == 2 + @test length(pipeline.stageio2idx) == 5 + @test pipeline.formats[pipeline.stageio2idx[(1, 2)]] == BufferFormat(4, Float16) + @test pipeline.stageio2idx[(1, 2)] == 1 + @test pipeline.stageio2idx[(2, -1)] == 1 + @test pipeline.stageio2idx[(3, -1)] == 1 + + # Stage 1 incomplete - if output 2 is connected all previous outputs must be connected too + @test_throws Exception generate_buffers(pipeline) + + connect!(pipeline, stage1, :a, stage4, :x) + + @test length(pipeline.formats) == 3 + @test length(pipeline.stageio2idx) == 7 + @test pipeline.formats[end] == BufferFormat() + @test pipeline.stageio2idx[(1, 1)] == 3 + @test pipeline.stageio2idx[(4, -1)] == 3 + + connect!(pipeline, stage4, :y, stage5, :z) + + @test length(pipeline.formats) == 4 + @test length(pipeline.stageio2idx) == 9 + @test pipeline.formats[end] == BufferFormat() + @test pipeline.stageio2idx[(4, 1)] == 4 + @test pipeline.stageio2idx[(5, -1)] == 4 + + #= + 1 2 3 4 5 (stages) + a -----------------> x ---> z (inputs outputs) + b -+----------> b + '-> b c ---> c + =# + buffers, remap = generate_buffers(pipeline) + @test length(buffers) == 3 # 3 buffer textures are needed for transfers + @test length(remap) == 4 # 4 connections map to them + @test buffers[remap[1]] == pipeline.formats[1] + @test buffers[remap[2]] == pipeline.formats[2] + @test buffers[remap[3]] == pipeline.formats[3] + # reuse from (4 x --> 5 z): + # (1 a --> 4 x) not yet available for reuse + # (1 b --> 2 b, 3b) available, upgrades + # (2 c --> 3 c) not allowed, incompatible types (int16) + @test buffers[remap[4]] == pipeline.formats[1] + + # text connecting two nodes that each have connections + connect!(pipeline, stage1, :b, stage5, :z) + + @test length(pipeline.formats) == 3 + @test length(pipeline.stageio2idx) == 9 + @test pipeline.formats == [BufferFormat(4, Float16), BufferFormat(2, Int16), BufferFormat(4, N0f8)] + for (k, v) in [(3, -1) => 1, (1, 2) => 1, (1, 1) => 3, (4, -1) => 3, (5, -1) => 1, (4, 1) => 1, (2, -1) => 1, (2, 1) => 2, (3, -2) => 2] + @test pipeline.stageio2idx[k] == v + end + + buffers, remap = generate_buffers(pipeline) + @test length(buffers) == 3 + @test length(remap) == 3 + @test buffers[remap[1]] == pipeline.formats[1] + @test buffers[remap[2]] == pipeline.formats[2] + @test buffers[remap[3]] == pipeline.formats[3] + end + + @testset "default pipeline" begin + pipeline = default_pipeline() + + # These directly check what the call should construct. (This indirectly + # also checks that connect!() works for pipelines) + @test length(pipeline.stages) == 7 + @test pipeline.stages[1] == Stage(:ZSort) + @test pipeline.stages[2] == Stage(:Render, Dict{Symbol, Int}(), BufferFormat[], + Dict(:color => 1, :objectid => 2, :position => 3, :normal => 4), + [BufferFormat(4, N0f8), BufferFormat(2, UInt32), BufferFormat(3, Float16), BufferFormat(3, Float16)], + transparency = false) + @test pipeline.stages[3] == Stage(Symbol("OIT Render"), Dict{Symbol, Int}(), BufferFormat[], + Dict(:color_sum => 1, :objectid => 2, :transmittance => 3), + [BufferFormat(4, Float16), BufferFormat(2, UInt32), BufferFormat(1, N0f8)]) + @test pipeline.stages[4] == Stage(:OIT, + Dict(:color_sum => 1, :transmittance => 2), [BufferFormat(4, Float16), BufferFormat(1, N0f8)], + Dict(:color => 1), [BufferFormat(4, N0f8)]) + @test pipeline.stages[5] == Stage(:FXAA1, + Dict(:color => 1, :objectid => 2), [BufferFormat(4, N0f8), BufferFormat(2, UInt32)], + Dict(:color_luma => 1), [BufferFormat(4, N0f8)], filter_in_shader = true) + @test pipeline.stages[6] == Stage(:FXAA2, + Dict(:color_luma => 1), [BufferFormat(4, N0f8, minfilter = :linear)], + Dict(:color => 1), [BufferFormat(4, N0f8)], filter_in_shader = true) + @test pipeline.stages[7] == Stage(:Display, + Dict(:color => 1, :objectid => 2), [BufferFormat(4, N0f8), BufferFormat(2, UInt32)], + Dict{Symbol, Int}(), BufferFormat[]) + + # Note: Order technically irrelevant but it's easier to test with order + # Same for inputs and outputs here + @test length(pipeline.formats) == 6 + @test length(pipeline.stageio2idx) == 15 + @test pipeline.formats[pipeline.stageio2idx[(5, 1)]] == BufferFormat(4, N0f8, minfilter = :linear) + @test pipeline.formats[pipeline.stageio2idx[(6, -1)]] == BufferFormat(4, N0f8, minfilter = :linear) + @test pipeline.formats[pipeline.stageio2idx[(3, 1)]] == BufferFormat(4, Float16) + @test pipeline.formats[pipeline.stageio2idx[(4, -1)]] == BufferFormat(4, Float16) + @test pipeline.formats[pipeline.stageio2idx[(3, 3)]] == BufferFormat(1, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(4, -2)]] == BufferFormat(1, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(4, 1)]] == BufferFormat(4, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(2, 1)]] == BufferFormat(4, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(5, -1)]] == BufferFormat(4, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(3, 2)]] == BufferFormat(2, UInt32) + @test pipeline.formats[pipeline.stageio2idx[(2, 2)]] == BufferFormat(2, UInt32) + @test pipeline.formats[pipeline.stageio2idx[(7, -2)]] == BufferFormat(2, UInt32) + @test pipeline.formats[pipeline.stageio2idx[(5, -2)]] == BufferFormat(2, UInt32) + @test pipeline.formats[pipeline.stageio2idx[(6, 1)]] == BufferFormat(4, N0f8) + @test pipeline.formats[pipeline.stageio2idx[(7, -1)]] == BufferFormat(4, N0f8) + + # Verify buffer generation with this more complex example + buffers, remap = generate_buffers(pipeline) + + # Order in buffers irrelevant as long as the mapping works + # Note: all outputs are unique so we don't have to explicitly test + # which one we hit + # Note: Changes to generate_buffers could change how formats get merged + # and cause different correct results + @test length(buffers) == 4 + @test length(remap) == 6 + + @test buffers[remap[pipeline.stageio2idx[(5, 1)]]] == BufferFormat(4, Float16, minfilter = :linear) + @test buffers[remap[pipeline.stageio2idx[(6, -1)]]] == BufferFormat(4, Float16, minfilter = :linear) + @test buffers[remap[pipeline.stageio2idx[(3, 1)]]] == BufferFormat(4, Float16, minfilter = :linear) + @test buffers[remap[pipeline.stageio2idx[(4, -1)]]] == BufferFormat(4, Float16, minfilter = :linear) + @test buffers[remap[pipeline.stageio2idx[(3, 3)]]] == BufferFormat(1, N0f8) + @test buffers[remap[pipeline.stageio2idx[(4, -2)]]] == BufferFormat(1, N0f8) + @test buffers[remap[pipeline.stageio2idx[(4, 1)]]] == BufferFormat(4, N0f8) + @test buffers[remap[pipeline.stageio2idx[(2, 1)]]] == BufferFormat(4, N0f8) + @test buffers[remap[pipeline.stageio2idx[(5, -1)]]] == BufferFormat(4, N0f8) + @test buffers[remap[pipeline.stageio2idx[(3, 2)]]] == BufferFormat(2, UInt32) + @test buffers[remap[pipeline.stageio2idx[(2, 2)]]] == BufferFormat(2, UInt32) + @test buffers[remap[pipeline.stageio2idx[(7, -2)]]] == BufferFormat(2, UInt32) + @test buffers[remap[pipeline.stageio2idx[(5, -2)]]] == BufferFormat(2, UInt32) + @test buffers[remap[pipeline.stageio2idx[(6, 1)]]] == BufferFormat(4, N0f8) + @test buffers[remap[pipeline.stageio2idx[(7, -1)]]] == BufferFormat(4, N0f8) + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index d378d05f175..f1b94dc0ec2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,4 +51,5 @@ using Makie: volume include("float32convert.jl") include("dim-converts.jl") include("Plane.jl") + include("render_pipeline.jl") end