From f56e723526802a8b49dbd17631a14ee6b08fdb70 Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Sat, 1 Jun 2024 13:52:15 +0200 Subject: [PATCH] Wrap JuliaSyntax.GreenNode in Runic.Node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using JuliaSyntax.GreenNode directly have worked for a long time, but at this point it seems that it is easier to re-package the tree in a custom type. This will be used to attach metadata to nodes, for example. Also include the following renames: - s/verified_children/verified_kids/ - s/children/kids/ - s/children′/kids′/ - s/children′′/kids′′/ - s/child/kid/ - s/child′/kid′/ - s/child′′/kid′′/ --- src/Runic.jl | 170 ++++++++++++++-------- src/chisels.jl | 122 +++++----------- src/debug.jl | 34 +++++ src/runestone.jl | 366 +++++++++++++++++++++++------------------------ test/runtests.jl | 14 +- 5 files changed, 379 insertions(+), 327 deletions(-) create mode 100644 src/debug.jl diff --git a/src/Runic.jl b/src/Runic.jl index 7b5c1df..bc8f1c1 100644 --- a/src/Runic.jl +++ b/src/Runic.jl @@ -14,7 +14,55 @@ using JuliaSyntax: end end -# JuliaSyntax extensions and other utilities +# Debug and assert utilities +include("debug.jl") + +######## +# Node # +######## + +# This is essentially just a re-packed `JuliaSyntax.GreenNode`. +struct Node + # The next three fields directly match JuliaSyntax.GreenNode. We can not store a + # GreenNode directly because the type of the children vector should be `Vector{Node}` + # and not `Vector{GreenNode}`. + head::JuliaSyntax.SyntaxHead + span::UInt32 + kids::Union{Tuple{}, Vector{Node}} +end + +# Re-package a GreenNode as a Node +function Node(node::JuliaSyntax.GreenNode) + return Node( + JuliaSyntax.head(node), JuliaSyntax.span(node), + map(Node, JuliaSyntax.children(node)), + ) +end + +# Defining this allow using many duck-typed methods in JuliaSyntax directly without having +# to re-package the Node as a GreenNode. +JuliaSyntax.head(node::Node) = head(node) +JuliaSyntax.span(node::Node) = span(node) + +# Matching JuliaSyntax.(head|span|flags|kind) +head(node::Node) = node.head +span(node::Node) = node.span +flags(node::Node) = JuliaSyntax.flags(node) +kind(node::Node) = JuliaSyntax.kind(node) + +function is_leaf(node::Node) + return node.kids === () +end + +# This function must only be be called after verifying that the node is not a leaf. We can +# then type-assert the return value to narrow it down from `Union{Tuple{}, Vector{Node}}` to +# `Vector{Node}`. +function verified_kids(node::Node) + @assert !is_leaf(node) + return node.kids::Vector{Node} +end + +# Node utilities and JuliaSyntax extensions include("chisels.jl") # Return the result of expr if it doesn't evaluate to `nothing` @@ -24,14 +72,18 @@ macro return_something(expr) end) end +####################################################### +# Main drivers for traversing and formatting the tree # +####################################################### + mutable struct Context # Input @const src_str::String - @const src_tree::JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead} + @const src_tree::Node @const src_io::IOBuffer # Output @const fmt_io::IOBuffer - fmt_tree::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing} + fmt_tree::Union{Node, Nothing} # User settings quiet::Bool verbose::Bool @@ -40,10 +92,10 @@ mutable struct Context check::Bool diff::Bool # Current state - # node::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing} - prev_sibling::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing} - next_sibling::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing} - # parent::Union{JuliaSyntax.GreenNode{JuliaSyntax.SyntaxHead}, Nothing} + # node::Union{Node, Nothing} + prev_sibling::Union{Node, Nothing} + next_sibling::Union{Node, Nothing} + # parent::Union{Node, Nothing} end function Context( @@ -51,7 +103,9 @@ function Context( diff::Bool = false, check::Bool = false, quiet::Bool = false, ) src_io = IOBuffer(src_str) - src_tree = JuliaSyntax.parseall(JuliaSyntax.GreenNode, src_str; ignore_warnings = true) + src_tree = Node( + JuliaSyntax.parseall(JuliaSyntax.GreenNode, src_str; ignore_warnings = true), + ) fmt_io = IOBuffer() fmt_tree = nothing # Debug mode enforces verbose and assert @@ -71,17 +125,17 @@ end # Read the bytes of the current node from the output io function read_bytes(ctx, node) pos = position(ctx.fmt_io) - bytes = read(ctx.fmt_io, JuliaSyntax.span(node)) - @assert length(bytes) == JuliaSyntax.span(node) + bytes = read(ctx.fmt_io, span(node)) + @assert length(bytes) == span(node) seek(ctx.fmt_io, pos) @assert position(ctx.fmt_io) == pos return bytes end -function accept_node!(ctx::Context, node::JuliaSyntax.GreenNode) +function accept_node!(ctx::Context, node::Node) # Accept the string representation of the current node by advancing the # output IO to the start of the next node - pos = position(ctx.fmt_io) + JuliaSyntax.span(node) + pos = position(ctx.fmt_io) + span(node) seek(ctx.fmt_io, pos) return end @@ -93,9 +147,9 @@ end struct NullNode end const nullnode = NullNode() -function format_node_with_children!(ctx::Context, node::JuliaSyntax.GreenNode) - # If the node doesn't have children there is nothing to do here - if !JuliaSyntax.haschildren(node) +function format_node_with_kids!(ctx::Context, node::Node) + # If the node doesn't have kids there is nothing to do here + if is_leaf(node) return nothing end @@ -105,63 +159,63 @@ function format_node_with_children!(ctx::Context, node::JuliaSyntax.GreenNode) ctx.prev_sibling = nothing ctx.next_sibling = nothing - # The new node parts. `children′` aliases `children` and only copied below if any of the + # The new node parts. `kids′` aliases `kids` and only copied below if any of the # nodes change ("copy-on-write"). - children = verified_children(node) - children′ = children - any_child_changed = false - - # Loop over all the children - for (i, child) in pairs(children) - # Set the siblings: previous from children′, next from children - ctx.prev_sibling = get(children′, i - 1, nothing) - ctx.next_sibling = get(children, i + 1, nothing) - child′ = child - this_child_changed = false + kids = verified_kids(node) + kids′ = kids + any_kid_changed = false + + # Loop over all the kids + for (i, kid) in pairs(kids) + # Set the siblings: previous from kids′, next from kids + ctx.prev_sibling = get(kids′, i - 1, nothing) + ctx.next_sibling = get(kids, i + 1, nothing) + kid′ = kid + this_kid_changed = false itr = 0 # Loop until this node reaches a steady state and is accepted while true # Keep track of the stream position and reset it below if the node is changed fmt_pos = position(ctx.fmt_io) - # Format the child - child′′ = format_node!(ctx, child′) - if child′′ === nullnode + # Format the kid + kid′′ = format_node!(ctx, kid′) + if kid′′ === nullnode # This node should be deleted from the tree # TODO: When this is fixed the sibling setting above needs to be modified to # handle this too - this_child_changed = true - error("TODO: handle removed children") - elseif child′′ === nothing + this_kid_changed = true + error("TODO: handle removed kids") + elseif kid′′ === nothing # The node was accepted, continue to next sibling - @assert position(ctx.fmt_io) == fmt_pos + JuliaSyntax.span(child′) + @assert position(ctx.fmt_io) == fmt_pos + span(kid′) break else # The node should be replaced with the new one. Reset the stream and try # again until it is accepted. - @assert child′′ isa JuliaSyntax.GreenNode - this_child_changed = true + @assert kid′′ isa Node + this_kid_changed = true seek(ctx.fmt_io, fmt_pos) - child′ = child′′ + kid′ = kid′′ end if (itr += 1) == 1000 error("infinite loop?") end end - any_child_changed |= this_child_changed - if any_child_changed - # De-alias the children if not already done - if children′ === children - children′ = eltype(children)[children[j] for j in 1:(i - 1)] + any_kid_changed |= this_kid_changed + if any_kid_changed + # De-alias the kids if not already done + if kids′ === kids + kids′ = eltype(kids)[kids[j] for j in 1:(i - 1)] end - push!(children′, child′) + push!(kids′, kid′) end end # Reset the siblings ctx.prev_sibling = prev_sibling ctx.next_sibling = next_sibling - # Return a new node if any of the children changed - if any_child_changed - return make_node(node, children′) + # Return a new node if any of the kids changed + if any_kid_changed + return make_node(node, kids′) else return nothing end @@ -175,8 +229,8 @@ Format a node. Return values: - `nullnode::NullNode`: The node should be deleted from the tree - `node::JuliaSyntax.GreenNode`: The node should be replaced with the new node """ -function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyntax.GreenNode, Nothing, NullNode} - node_kind = JuliaSyntax.kind(node) +function format_node!(ctx::Context, node::Node)::Union{Node, Nothing, NullNode} + node_kind = kind(node) # Go through the runestone and apply transformations. @return_something trim_trailing_whitespace(ctx, node) @@ -226,7 +280,7 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn node_kind === K"vect" ) @assert !JuliaSyntax.is_trivia(node) - node′ = format_node_with_children!(ctx, node) + node′ = format_node_with_kids!(ctx, node) @assert node′ !== nullnode return node′ @@ -265,16 +319,16 @@ function format_node!(ctx::Context, node::JuliaSyntax.GreenNode)::Union{JuliaSyn node_kind === K"where" || node_kind === K"while" ) - node′ = format_node_with_children!(ctx, node) + node′ = format_node_with_kids!(ctx, node) @assert node′ !== nullnode return node′ - # Nodes that should recurse if they have children (all??) - elseif JuliaSyntax.haschildren(node) && ( + # Nodes that should recurse if they have kids (all??) + elseif !is_leaf(node) && ( JuliaSyntax.is_operator(node) || node_kind === K"else" # try-(catch|finally)-else ) - node′ = format_node_with_children!(ctx, node) + node′ = format_node_with_kids!(ctx, node) @assert node′ !== nullnode return node′ @@ -374,8 +428,8 @@ function format_tree!(ctx::Context) @assert src_pos == 0 fmt_pos = position(ctx.fmt_io) @assert fmt_pos == 0 - nb = write(ctx.fmt_io, read(ctx.src_io, JuliaSyntax.span(root))) - @assert nb == JuliaSyntax.span(root) + nb = write(ctx.fmt_io, read(ctx.src_io, span(root))) + @assert nb == span(root) # Reset IOs so that the offsets are correct seek(ctx.src_io, src_pos) seek(ctx.fmt_io, fmt_pos) @@ -391,10 +445,10 @@ function format_tree!(ctx::Context) error("root node deleted") elseif root′′ === nothing # root′ = root′′ - @assert position(ctx.fmt_io) == fmt_pos + JuliaSyntax.span(root′) + @assert position(ctx.fmt_io) == fmt_pos + span(root′) break else - @assert root′′ isa JuliaSyntax.GreenNode + @assert root′′ isa Node # The node was changed, reset the output stream and try again seek(ctx.fmt_io, fmt_pos) root′ = root′′ @@ -405,7 +459,7 @@ function format_tree!(ctx::Context) end end # Truncate the output at the root span - truncate(ctx.fmt_io, JuliaSyntax.span(root′)) + truncate(ctx.fmt_io, span(root′)) # Set the final tree ctx.fmt_tree = root′ return nothing diff --git a/src/chisels.jl b/src/chisels.jl index f225f13..45eb345 100644 --- a/src/chisels.jl +++ b/src/chisels.jl @@ -1,132 +1,88 @@ # SPDX-License-Identifier: MIT -############## -# Debug info # -############## +######################################################## +# Node utilities extensions and JuliaSyntax extensions # +######################################################## -# @lock is defined but not exported in older Julia versions -if VERSION < v"1.7.0" - using Base: @lock +# Create a new node with the same head but new kids +function make_node(node::Node, kids′::Vector{Node}) + span′ = mapreduce(span, +, kids′; init = 0) + return Node(head(node), span′, kids′) end -# Code derived from ToggleableAsserts.jl kept in a separate file -include("ToggleableAsserts.jl") - -abstract type RunicException <: Exception end - -struct AssertionError <: RunicException - msg::String -end - -function Base.showerror(io::IO, err::AssertionError) - print( - io, - "Runic.AssertionError: `", err.msg, "`. This is unexpected, " * - "please file an issue with a reproducible example at " * - "https://github.com/fredrikekre/Runic.jl/issues/new.", - ) -end - -function macroexpand_assert(expr) - msg = string(expr) - return :($(esc(expr)) || throw(AssertionError($msg))) -end - - -########################## -# JuliaSyntax extensions # -########################## - -# Create a new node with the same head but new children -function make_node(node::JuliaSyntax.GreenNode, children′::AbstractVector{<:JuliaSyntax.GreenNode}) - span′ = mapreduce(JuliaSyntax.span, +, children′; init = 0) - return JuliaSyntax.GreenNode(JuliaSyntax.head(node), span′, children′) -end - -function is_leaf(node::JuliaSyntax.GreenNode) - return !JuliaSyntax.haschildren(node) -end - -function first_leaf(node::JuliaSyntax.GreenNode) +function first_leaf(node::Node) if is_leaf(node) return node else - return first_leaf(first(verified_children(node))) + return first_leaf(first(verified_kids(node))) end end -# Return number of non-whitespace children -function n_children(node::JuliaSyntax.GreenNode) - return is_leaf(node) ? 0 : count(!JuliaSyntax.is_whitespace, verified_children(node)) -end - -# This function exist so that we can type-assert the return value to narrow it down from -# `Union{Tuple{}, Vector{JuliaSyntax.GreenNode}}` to `Vector{JuliaSyntax.GreenNode}`. Must -# only be called after verifying that the node has children. -function verified_children(node::JuliaSyntax.GreenNode) - @assert JuliaSyntax.haschildren(node) - return JuliaSyntax.children(node)::AbstractVector +# Return number of non-whitespace kids, basically the length the equivalent +# (expr::Expr).args +function meta_nargs(node::Node) + return is_leaf(node) ? 0 : count(!JuliaSyntax.is_whitespace, verified_kids(node)) end -function replace_first_leaf(node::JuliaSyntax.GreenNode, child′::JuliaSyntax.GreenNode) +function replace_first_leaf(node::Node, kid′::Node) if is_leaf(node) - return child′ + return kid′ else - children′ = copy(verified_children(node)) - children′[1] = replace_first_leaf(children′[1], child′) - @assert length(children′) > 0 - return make_node(node, children′) + kids′ = copy(verified_kids(node)) + kids′[1] = replace_first_leaf(kids′[1], kid′) + @assert length(kids′) > 0 + return make_node(node, kids′) end end -function last_leaf(node::JuliaSyntax.GreenNode) +function last_leaf(node::Node) if is_leaf(node) return node else - return last_leaf(last(verified_children(node))) + return last_leaf(last(verified_kids(node))) end end -function is_assignment(node::JuliaSyntax.GreenNode) +function is_assignment(node::Node) return JuliaSyntax.is_prec_assignment(node) - return !is_leaf(node) && JuliaSyntax.is_prec_assignment(node) + # return !is_leaf(node) && JuliaSyntax.is_prec_assignment(node) end # Just like `JuliaSyntax.is_infix_op_call`, but also check that the node is K"call" -function is_infix_op_call(node::JuliaSyntax.GreenNode) - return JuliaSyntax.kind(node) === K"call" && - JuliaSyntax.is_infix_op_call(node) +function is_infix_op_call(node::Node) + return kind(node) === K"call" && JuliaSyntax.is_infix_op_call(node) end -function infix_op_call_op(node::JuliaSyntax.GreenNode) +# Extract the operator of an infix op call node +function infix_op_call_op(node::Node) @assert is_infix_op_call(node) - children = verified_children(node) - first_operand_index = findfirst(!JuliaSyntax.is_whitespace, children) - op_index = findnext(JuliaSyntax.is_operator, children, first_operand_index + 1) - return children[op_index] + kids = verified_kids(node) + first_operand_index = findfirst(!JuliaSyntax.is_whitespace, kids) + op_index = findnext(JuliaSyntax.is_operator, kids, first_operand_index + 1) + return kids[op_index] end # Comparison leaf or a dotted comparison leaf (.<) -function is_comparison_leaf(node::JuliaSyntax.GreenNode) +function is_comparison_leaf(node::Node) if is_leaf(node) && JuliaSyntax.is_prec_comparison(node) return true - elseif !is_leaf(node) && JuliaSyntax.kind(node) === K"." && - n_children(node) == 2 && is_comparison_leaf(verified_children(node)[2]) + elseif !is_leaf(node) && kind(node) === K"." && + meta_nargs(node) == 2 && is_comparison_leaf(verified_kids(node)[2]) return true else return false end end -function is_operator_leaf(node::JuliaSyntax.GreenNode) +function is_operator_leaf(node::Node) return is_leaf(node) && JuliaSyntax.is_operator(node) end -function first_non_whitespace_child(node::JuliaSyntax.GreenNode) +function first_non_whitespace_kid(node::Node) @assert !is_leaf(node) - children = verified_children(node) - idx = findfirst(!JuliaSyntax.is_whitespace, children)::Int - return children[idx] + kids = verified_kids(node) + idx = findfirst(!JuliaSyntax.is_whitespace, kids)::Int + return kids[idx] end ########################## diff --git a/src/debug.jl b/src/debug.jl new file mode 100644 index 0000000..008f3bd --- /dev/null +++ b/src/debug.jl @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: MIT + +############## +# Debug info # +############## + +# @lock is defined but not exported in older Julia versions +if VERSION < v"1.7.0" + using Base: @lock +end + +# Code derived from ToggleableAsserts.jl kept in a separate file +include("ToggleableAsserts.jl") + +abstract type RunicException <: Exception end + +struct AssertionError <: RunicException + msg::String +end + +function Base.showerror(io::IO, err::AssertionError) + print( + io, + "Runic.AssertionError: `", err.msg, "`. This is unexpected, " * + "please file an issue with a reproducible example at " * + "https://github.com/fredrikekre/Runic.jl/issues/new.", + ) +end + +function macroexpand_assert(expr) + msg = string(expr) + return :($(esc(expr)) || throw(AssertionError($msg))) +end + diff --git a/src/runestone.jl b/src/runestone.jl index a1e0f64..decbd30 100644 --- a/src/runestone.jl +++ b/src/runestone.jl @@ -1,10 +1,14 @@ # SPDX-License-Identifier: MIT +function dumpnode(node) + println("node: {kind: $(kind(node)), span: $(span(node)), flags: $(flags(node)), nkids: $(length(verified_kids(node)))}") +end + # This is the runestone where all the formatting transformations are implemented. -function trim_trailing_whitespace(ctx::Context, node::JuliaSyntax.GreenNode) - JuliaSyntax.kind(node) === K"NewlineWs" || return nothing - @assert !JuliaSyntax.haschildren(node) +function trim_trailing_whitespace(ctx::Context, node::Node) + kind(node) === K"NewlineWs" || return nothing + @assert is_leaf(node) str = String(read_bytes(ctx, node)) str′ = replace(str, r"\h*(\r\n|\r|\n)" => '\n') # If the next sibling is also a NewlineWs we can trim trailing @@ -18,44 +22,44 @@ function trim_trailing_whitespace(ctx::Context, node::JuliaSyntax.GreenNode) return nothing end # Write new bytes and reset the stream - nb = replace_bytes!(ctx, str′, JuliaSyntax.span(node)) - @assert nb != JuliaSyntax.span(node) + nb = replace_bytes!(ctx, str′, span(node)) + @assert nb != span(node) # Create new node and return it - node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ()) + node′ = Node(head(node), nb, ()) return node′ end -function format_hex_literals(ctx::Context, node::JuliaSyntax.GreenNode) - JuliaSyntax.kind(node) === K"HexInt" || return nothing - @assert JuliaSyntax.flags(node) == 0 - @assert !JuliaSyntax.haschildren(node) - span = JuliaSyntax.span(node) - @assert span > 2 # 0x prefix + something more +function format_hex_literals(ctx::Context, node::Node) + kind(node) === K"HexInt" || return nothing + @assert flags(node) == 0 + @assert is_leaf(node) + spn = span(node) + @assert spn > 2 # 0x prefix + something more # Target spans(0x + maximum chars for formatted UInt8, UInt16, UInt32, UInt64, UInt128) target_spans = 2 .+ (2, 4, 8, 16, 32) - if span >= 34 || span in target_spans + if spn >= 34 || spn in target_spans # Do nothing: correctly formatted or a BigInt hex literal return nothing end # Insert leading zeros - i = findfirst(x -> x > span, target_spans)::Int + i = findfirst(x -> x > spn, target_spans)::Int bytes = read_bytes(ctx, node) while length(bytes) < target_spans[i] insert!(bytes, 3, '0') end - nb = replace_bytes!(ctx, bytes, span) + nb = replace_bytes!(ctx, bytes, spn) @assert nb == length(bytes) == target_spans[i] # Create new node and return it - node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ()) + node′ = Node(head(node), nb, ()) return node′ end -function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode) - JuliaSyntax.kind(node) === K"OctInt" || return nothing - @assert JuliaSyntax.flags(node) == 0 - @assert !JuliaSyntax.haschildren(node) - span = JuliaSyntax.span(node) - @assert span > 2 # 0o prefix + something more +function format_oct_literals(ctx::Context, node::Node) + kind(node) === K"OctInt" || return nothing + @assert flags(node) == 0 + @assert is_leaf(node) + spn = span(node) + @assert spn > 2 # 0o prefix + something more # Padding depends on the value of the literal... str = String(read_bytes(ctx, node)) n = tryparse(UInt128, str) @@ -69,10 +73,10 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode) n <= typemax(UInt32) ? 13 : n <= typemax(UInt64) ? 24 : n <= typemax(UInt128) ? 45 : error("unreachable") target_spans = (5, 8, 13, 24, 45) - i = findfirst(x -> x >= span, target_spans)::Int + i = findfirst(x -> x >= spn, target_spans)::Int target_span_from_source = target_spans[i] target_span = max(target_span_from_value, target_span_from_source) - if span == target_span + if spn == target_span # Do nothing: correctly formatted oct literal return nothing end @@ -81,17 +85,17 @@ function format_oct_literals(ctx::Context, node::JuliaSyntax.GreenNode) while length(bytes) < target_span insert!(bytes, 3, '0') end - nb = replace_bytes!(ctx, bytes, span) + nb = replace_bytes!(ctx, bytes, spn) @assert nb == length(bytes) == target_span # Create new node and return it - node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ()) + node′ = Node(head(node), nb, ()) return node′ end -function format_float_literals(ctx::Context, node::JuliaSyntax.GreenNode) - JuliaSyntax.kind(node) in KSet"Float Float32" || return nothing - @assert JuliaSyntax.flags(node) == 0 - @assert !JuliaSyntax.haschildren(node) +function format_float_literals(ctx::Context, node::Node) + kind(node) in KSet"Float Float32" || return nothing + @assert flags(node) == 0 + @assert is_leaf(node) str = String(read_bytes(ctx, node)) # Check and shortcut the happy path first r = r""" @@ -138,129 +142,125 @@ function format_float_literals(ctx::Context, node::JuliaSyntax.GreenNode) write(io, exp_part) end bytes = take!(io) - nb = replace_bytes!(ctx, bytes, JuliaSyntax.span(node)) + nb = replace_bytes!(ctx, bytes, span(node)) @assert nb == length(bytes) # Create new node and return it - node′ = JuliaSyntax.GreenNode(JuliaSyntax.head(node), nb, ()) + node′ = Node(head(node), nb, ()) return node′ end # Insert space around `x`, where `x` can be operators, assignments, etc. with the pattern: # ``, for example the spaces around `+` and `=` in # `a = x + y`. -function spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) where F +function spaces_around_x(ctx::Context, node::Node, is_x::F) where F # TODO: So much boilerplate here... - @assert JuliaSyntax.haschildren(node) + @assert !is_leaf(node) - children = verified_children(node) - children′ = children + kids = verified_kids(node) + kids′ = kids any_changes = false pos = position(ctx.fmt_io) - ws = JuliaSyntax.GreenNode( - JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1, (), - ) + ws = Node(JuliaSyntax.SyntaxHead(K"Whitespace", JuliaSyntax.TRIVIA_FLAG), 1, ()) # Toggle for whether we are currently looking for whitespace or not looking_for_whitespace = false looking_for_x = false - for (i, child) in pairs(children) - if JuliaSyntax.kind(child) === K"NewlineWs" || - (i == 1 && JuliaSyntax.kind(child) === K"Whitespace") + for (i, kid) in pairs(kids) + if kind(kid) === K"NewlineWs" || + (i == 1 && kind(kid) === K"Whitespace") # NewlineWs are accepted as is by this pass. - # Whitespace is accepted as is if this is the first child even if the span is + # Whitespace is accepted as is if this is the first kid even if the span is # larger than we expect since we don't look backwards. It should be cleaned up # by some other pass. - accept_node!(ctx, child) - any_changes && push!(children′, child) + accept_node!(ctx, kid) + any_changes && push!(kids′, kid) looking_for_whitespace = false elseif looking_for_whitespace - if JuliaSyntax.kind(child) === K"Whitespace" && JuliaSyntax.span(child) == 1 + if kind(kid) === K"Whitespace" && span(kid) == 1 # All good, just advance the IO - accept_node!(ctx, child) - any_changes && push!(children′, child) + accept_node!(ctx, kid) + any_changes && push!(kids′, kid) looking_for_whitespace = false - elseif JuliaSyntax.kind(child) === K"Whitespace" + elseif kind(kid) === K"Whitespace" # Whitespace node but replace since not single space any_changes = true - if children′ === children - children′ = children[1:i - 1] + if kids′ === kids + kids′ = kids[1:i - 1] end - push!(children′, ws) - replace_bytes!(ctx, " ", JuliaSyntax.span(child)) + push!(kids′, ws) + replace_bytes!(ctx, " ", span(kid)) accept_node!(ctx, ws) looking_for_whitespace = false - elseif JuliaSyntax.haschildren(child) && - JuliaSyntax.kind(first_leaf(child)) === K"Whitespace" - # Whitespace found at the beginning of next child. - child_ws = first_leaf(child) - looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace" - @assert !is_x(child)::Bool + elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"Whitespace" + # Whitespace found at the beginning of next kid. + kid_ws = first_leaf(kid) + looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace" + @assert !is_x(kid)::Bool looking_for_x = true - if JuliaSyntax.span(child_ws) == 1 + if span(kid_ws) == 1 # Accept the node - accept_node!(ctx, child) - any_changes && push!(children′, child) + accept_node!(ctx, kid) + any_changes && push!(kids′, kid) else - # Replace the whitespace node of the child - child′ = replace_first_leaf(child, ws) - @assert JuliaSyntax.span(child′) == JuliaSyntax.span(child) - JuliaSyntax.span(child_ws) + 1 - bytes_to_skip = JuliaSyntax.span(child) - JuliaSyntax.span(child′) + # Replace the whitespace node of the kid + kid′ = replace_first_leaf(kid, ws) + @assert span(kid′) == span(kid) - span(kid_ws) + 1 + bytes_to_skip = span(kid) - span(kid′) @assert bytes_to_skip > 0 replace_bytes!(ctx, "", bytes_to_skip) - accept_node!(ctx, child′) + accept_node!(ctx, kid′) any_changes = true - if children′ === children - children′ = children[1:i - 1] + if kids′ === kids + kids′ = kids[1:i - 1] end - push!(children′, child′) + push!(kids′, kid′) end - elseif JuliaSyntax.haschildren(child) && - JuliaSyntax.kind(first_leaf(child)) === K"NewlineWs" + elseif !is_leaf(kid) && kind(first_leaf(kid)) === K"NewlineWs" # NewlineWs have to be accepted as is - # @info " ... childs first leaf is NewlineWs I'll take it" - accept_node!(ctx, child) - any_changes && push!(children′, child) - looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace" - @assert !is_x(child)::Bool + # @info " ... kids first leaf is NewlineWs I'll take it" + accept_node!(ctx, kid) + any_changes && push!(kids′, kid) + looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace" + @assert !is_x(kid)::Bool looking_for_x = true else - # @info " ... no whitespace, inserting" JuliaSyntax.kind(child) + # @info " ... no whitespace, inserting" kind(kid) # Not a whitespace node, insert one any_changes = true - if children′ === children - children′ = children[1:i - 1] + if kids′ === kids + kids′ = kids[1:i - 1] end - push!(children′, ws) + push!(kids′, ws) replace_bytes!(ctx, " ", 0) accept_node!(ctx, ws) # Write and accept the node - push!(children′, child) - accept_node!(ctx, child) - looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace" + push!(kids′, kid) + accept_node!(ctx, kid) + looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace" if looking_for_x - @assert is_x(child)::Bool + @assert is_x(kid)::Bool end - # Flip the switch, unless child is a comment - looking_for_x = JuliaSyntax.kind(child) === K"Comment" ? looking_for_x : !looking_for_x + # Flip the switch, unless kid is a comment + looking_for_x = kind(kid) === K"Comment" ? looking_for_x : !looking_for_x end else # !expect_ws if looking_for_x - @assert is_x(child)::Bool + @assert is_x(kid)::Bool end - @assert JuliaSyntax.kind(child) !== K"Whitespace" # This would be weird, I think? - any_changes && push!(children′, child) - accept_node!(ctx, child) - looking_for_whitespace = JuliaSyntax.kind(last_leaf(child)) !== K"Whitespace" - # Flip the switch, unless child is a comment - looking_for_x = JuliaSyntax.kind(child) === K"Comment" ? looking_for_x : !looking_for_x + @assert kind(kid) !== K"Whitespace" # This would be weird, I think? + any_changes && push!(kids′, kid) + accept_node!(ctx, kid) + looking_for_whitespace = kind(last_leaf(kid)) !== K"Whitespace" + # Flip the switch, unless kid is a comment + looking_for_x = kind(kid) === K"Comment" ? looking_for_x : !looking_for_x end end # Reset stream seek(ctx.fmt_io, pos) if any_changes # Create new node and return it - return make_node(node, children′) + return make_node(node, kids′) else return nothing end @@ -268,39 +268,39 @@ end # This pass handles spaces around infix operator calls, comparison chains, and # <: and >: operators. -function spaces_around_operators(ctx::Context, node::JuliaSyntax.GreenNode) +function spaces_around_operators(ctx::Context, node::Node) if !( - (is_infix_op_call(node) && !(JuliaSyntax.kind(infix_op_call_op(node)) in KSet": ^")) || - (JuliaSyntax.kind(node) in KSet"<: >:" && n_children(node) == 3) || - (JuliaSyntax.kind(node) === K"comparison" && !JuliaSyntax.is_trivia(node)) + (is_infix_op_call(node) && !(kind(infix_op_call_op(node)) in KSet": ^")) || + (kind(node) in KSet"<: >:" && meta_nargs(node) == 3) || + (kind(node) === K"comparison" && !JuliaSyntax.is_trivia(node)) ) return nothing end - @assert JuliaSyntax.kind(node) in KSet"call comparison <: >:" + @assert kind(node) in KSet"call comparison <: >:" is_x = x -> is_operator_leaf(x) || is_comparison_leaf(x) return spaces_around_x(ctx, node, is_x) end -function spaces_around_assignments(ctx::Context, node::JuliaSyntax.GreenNode) +function spaces_around_assignments(ctx::Context, node::Node) if !(is_assignment(node) && !is_leaf(node) ) return nothing end # for-loop nodes are of kind K"=" even when `in` or `∈` is used so we need to # include these kinds in the predicate too. - is_x = x -> is_assignment(x) || JuliaSyntax.kind(x) in KSet"in ∈" + is_x = x -> is_assignment(x) || kind(x) in KSet"in ∈" return spaces_around_x(ctx, node, is_x) end # Opposite of `spaces_around_x`: remove spaces around `x` -function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) where F - @assert JuliaSyntax.haschildren(node) +function no_spaces_around_x(ctx::Context, node::Node, is_x::F) where F + @assert !is_leaf(node) # TODO: Can't handle NewlineWs here right now - if any(JuliaSyntax.kind(c) === K"NewlineWs" for c in JuliaSyntax.children(node)) + if any(kind(c) === K"NewlineWs" for c in verified_kids(node)) return nothing end - children = verified_children(node) - children′ = children + kids = verified_kids(node) + kids′ = kids any_changes = false pos = position(ctx.fmt_io) @@ -308,28 +308,28 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) # K"::", K"<:", and K">:" are special cases here since they can be used without an LHS # in e.g. `f(::Int) = ...` and `Vector{<:Real}`. - if JuliaSyntax.kind(node) in KSet":: <: >:" - looking_for_x = is_x(first_non_whitespace_child(node))::Bool + if kind(node) in KSet":: <: >:" + looking_for_x = is_x(first_non_whitespace_kid(node))::Bool end - for (i, child) in pairs(children) - if (i == 1 || i == length(children)) && JuliaSyntax.kind(child) === K"Whitespace" - accept_node!(ctx, child) - any_changes && push!(children′, child) - elseif JuliaSyntax.kind(child) === K"Whitespace" - # Ignore it but need to copy children and re-write bytes + for (i, kid) in pairs(kids) + if (i == 1 || i == length(kids)) && kind(kid) === K"Whitespace" + accept_node!(ctx, kid) + any_changes && push!(kids′, kid) + elseif kind(kid) === K"Whitespace" + # Ignore it but need to copy kids and re-write bytes any_changes = true - if children′ === children - children′ = children[1:i - 1] + if kids′ === kids + kids′ = kids[1:i - 1] end - replace_bytes!(ctx, "", JuliaSyntax.span(child)) + replace_bytes!(ctx, "", span(kid)) else - @assert JuliaSyntax.kind(child) !== K"Whitespace" + @assert kind(kid) !== K"Whitespace" if looking_for_x - @assert is_x(child)::Bool + @assert is_x(kid)::Bool end - any_changes && push!(children′, child) - accept_node!(ctx, child) + any_changes && push!(kids′, kid) + accept_node!(ctx, kid) looking_for_x = !looking_for_x end end @@ -337,8 +337,8 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) seek(ctx.fmt_io, pos) if any_changes # Create new node and return it - node′ = make_node(node, children′) - @assert JuliaSyntax.span(node′) < JuliaSyntax.span(node) + node′ = make_node(node, kids′) + @assert span(node′) < span(node) return node′ else return nothing @@ -346,127 +346,127 @@ function no_spaces_around_x(ctx::Context, node::JuliaSyntax.GreenNode, is_x::F) end # no spaces around `:`, `^`, and `::` -function no_spaces_around_colon_etc(ctx::Context, node::JuliaSyntax.GreenNode) +function no_spaces_around_colon_etc(ctx::Context, node::Node) if !( - (is_infix_op_call(node) && JuliaSyntax.kind(infix_op_call_op(node)) in KSet": ^") || - (JuliaSyntax.kind(node) === K"::" && !is_leaf(node)) || - (JuliaSyntax.kind(node) in KSet"<: >:" && n_children(node) == 2) + (is_infix_op_call(node) && kind(infix_op_call_op(node)) in KSet": ^") || + (kind(node) === K"::" && !is_leaf(node)) || + (kind(node) in KSet"<: >:" && meta_nargs(node) == 2) ) return nothing end - @assert JuliaSyntax.kind(node) in KSet"call :: <: >:" - is_x = x -> is_leaf(x) && JuliaSyntax.kind(x) in KSet": ^ :: <: >:" + @assert kind(node) in KSet"call :: <: >:" + is_x = x -> is_leaf(x) && kind(x) in KSet": ^ :: <: >:" return no_spaces_around_x(ctx, node, is_x) end # Replace the K"=" operator with `in` -function replace_with_in(ctx::Context, node::JuliaSyntax.GreenNode) - @assert JuliaSyntax.kind(node) === K"=" && !is_leaf(node) && n_children(node) == 3 - children = verified_children(node) - vars_index = findfirst(!JuliaSyntax.is_whitespace, children) +function replace_with_in(ctx::Context, node::Node) + @assert kind(node) === K"=" && !is_leaf(node) && meta_nargs(node) == 3 + kids = verified_kids(node) + vars_index = findfirst(!JuliaSyntax.is_whitespace, kids) # TODO: Need to insert whitespaces around `in` when replacing e.g. `i=I` with `iinI`. # However, at the moment it looks like the whitespace around operator pass does it's # thing first? I don't really know how though, because the for loop pass should be # happening before... - in_index = findnext(!JuliaSyntax.is_whitespace, children, vars_index + 1) - in_node = children[in_index] - if JuliaSyntax.kind(in_node) === K"in" + in_index = findnext(!JuliaSyntax.is_whitespace, kids, vars_index + 1) + in_node = kids[in_index] + if kind(in_node) === K"in" @assert JuliaSyntax.is_trivia(in_node) @assert is_leaf(in_node) return nothing end - @assert JuliaSyntax.kind(in_node) in KSet"∈ =" + @assert kind(in_node) in KSet"∈ =" @assert JuliaSyntax.is_trivia(in_node) @assert is_leaf(in_node) # Accept nodes to advance the stream for i in 1:(in_index - 1) - accept_node!(ctx, children[i]) + accept_node!(ctx, kids[i]) end # Construct the replacement - nb = replace_bytes!(ctx, "in", JuliaSyntax.span(in_node)) - in_node′ = JuliaSyntax.GreenNode( + nb = replace_bytes!(ctx, "in", span(in_node)) + in_node′ = Node( JuliaSyntax.SyntaxHead(K"in", JuliaSyntax.TRIVIA_FLAG), nb, (), ) accept_node!(ctx, in_node′) - children′ = copy(children) - children′[in_index] = in_node′ - # Accept remaining eq_children - for i in (in_index + 1):length(children′) - accept_node!(ctx, children′[i]) + kids′ = copy(kids) + kids′[in_index] = in_node′ + # Accept remaining kids + for i in (in_index + 1):length(kids′) + accept_node!(ctx, kids′[i]) end - return make_node(node, children′) + return make_node(node, kids′) end -function replace_with_in_cartesian(ctx::Context, node::JuliaSyntax.GreenNode) - @assert JuliaSyntax.kind(node) === K"cartesian_iterator" && !is_leaf(node) - children = verified_children(node) - children′ = children - for (i, child) in pairs(children) - if JuliaSyntax.kind(child) === K"=" - child′ = replace_with_in(ctx, child) - if child′ !== nothing - if children′ === children - children′ = copy(children) +function replace_with_in_cartesian(ctx::Context, node::Node) + @assert kind(node) === K"cartesian_iterator" && !is_leaf(node) + kids = verified_kids(node) + kids′ = kids + for (i, kid) in pairs(kids) + if kind(kid) === K"=" + kid′ = replace_with_in(ctx, kid) + if kid′ !== nothing + if kids′ === kids + kids′ = copy(kids) end - children′[i] = child′ + kids′[i] = kid′ else - children′[i] = child - accept_node!(ctx, child) + kids′[i] = kid + accept_node!(ctx, kid) end else - children′[i] = child - accept_node!(ctx, child) + kids′[i] = kid + accept_node!(ctx, kid) end end - if children === children′ + if kids === kids′ return nothing end - return make_node(node, children′) + return make_node(node, kids′) end # replace `=` and `∈` with `in` in for-loops -function for_loop_use_in(ctx::Context, node::JuliaSyntax.GreenNode) +function for_loop_use_in(ctx::Context, node::Node) if !( - (JuliaSyntax.kind(node) === K"for" && !is_leaf(node) && n_children(node) == 4) || - (JuliaSyntax.kind(node) === K"generator" && n_children(node) == 3) # TODO: Unsure about 3. + (kind(node) === K"for" && !is_leaf(node) && meta_nargs(node) == 4) || + (kind(node) === K"generator" && meta_nargs(node) == 3) # TODO: Unsure about 3. ) return nothing end pos = position(ctx.fmt_io) - children = verified_children(node) - for_index = findfirst(c -> JuliaSyntax.kind(c) === K"for" && is_leaf(c), children)::Int - for_node = children[for_index] - @assert JuliaSyntax.kind(for_node) === K"for" && JuliaSyntax.span(for_node) == 3 && + kids = verified_kids(node) + for_index = findfirst(c -> kind(c) === K"for" && is_leaf(c), kids)::Int + for_node = kids[for_index] + @assert kind(for_node) === K"for" && span(for_node) == 3 && is_leaf(for_node) && JuliaSyntax.is_trivia(for_node) for i in 1:for_index - accept_node!(ctx, children[i]) + accept_node!(ctx, kids[i]) end # The for loop specification node can be either K"=" or K"cartesian_iterator" for_spec_index = for_index + 1 - for_spec_node = children[for_spec_index] - @assert JuliaSyntax.kind(for_spec_node) in KSet"= cartesian_iterator" - if JuliaSyntax.kind(for_spec_node) === K"=" + for_spec_node = kids[for_spec_index] + @assert kind(for_spec_node) in KSet"= cartesian_iterator" + if kind(for_spec_node) === K"=" for_spec_node′ = replace_with_in(ctx, for_spec_node) else - @assert JuliaSyntax.kind(for_spec_node) === K"cartesian_iterator" + @assert kind(for_spec_node) === K"cartesian_iterator" for_spec_node′ = replace_with_in_cartesian(ctx, for_spec_node) end if for_spec_node′ === nothing seek(ctx.fmt_io, pos) return nothing end - @assert position(ctx.fmt_io) == pos + mapreduce(JuliaSyntax.span, +, @view(children[1:for_index])) + JuliaSyntax.span(for_spec_node′) + @assert position(ctx.fmt_io) == pos + mapreduce(span, +, @view(kids[1:for_index])) + span(for_spec_node′) # Insert the new for spec node - children′ = copy(children) - children′[for_spec_index] = for_spec_node′ + kids′ = copy(kids) + kids′[for_spec_index] = for_spec_node′ # At this point the eq node is done, just accept any remaining nodes # TODO: Don't need to do this... - for i in (for_spec_index + 1):length(children′) - accept_node!(ctx, children′[i]) + for i in (for_spec_index + 1):length(kids′) + accept_node!(ctx, kids′[i]) end # Construct the full node and return - node′ = make_node(node, children′) - @assert position(ctx.fmt_io) == pos + JuliaSyntax.span(node′) + node′ = make_node(node, kids′) + @assert position(ctx.fmt_io) == pos + span(node′) seek(ctx.fmt_io, pos) # reset return node′ end diff --git a/test/runtests.jl b/test/runtests.jl index b15ba57..1fd952c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,9 +8,17 @@ using JuliaSyntax: JuliaSyntax @testset "Chisels" begin - # Type stability of verified_children - node = JuliaSyntax.parseall(JuliaSyntax.GreenNode, "a = 1 + b\n") - @test typeof(@inferred Runic.verified_children(node)) <: Vector{<:JuliaSyntax.GreenNode} + # Type stability of verified_kids + node = Runic.Node(JuliaSyntax.parseall(JuliaSyntax.GreenNode, "a = 1 + b\n")) + @test typeof(@inferred Runic.verified_kids(node)) === Vector{Runic.Node} + + # JuliaSyntax duck-typing + for n in (node, node.children...,) + @test Runic.head(n) === JuliaSyntax.head(n) === n.head + @test Runic.kind(n) === JuliaSyntax.kind(n) === n.head.kind + @test Runic.flags(n) === JuliaSyntax.flags(n) === n.head.flags + @test Runic.span(n) === JuliaSyntax.span(n) === n.span + end # replace_bytes!: insert larger io = IOBuffer(); write(io, "abc"); seek(io, 1)