From d9b45afe3359825a9d41ec85bb9f321bdb24de66 Mon Sep 17 00:00:00 2001 From: Koki Fushimi Date: Fri, 21 Jun 2024 20:39:14 +0900 Subject: [PATCH] Rename `get_mark` to `Mark`. --- src/scanner.jl | 146 ++++++++++++++++++++++++------------------------- 1 file changed, 72 insertions(+), 74 deletions(-) diff --git a/src/scanner.jl b/src/scanner.jl index d1d327f..a7d46cd 100644 --- a/src/scanner.jl +++ b/src/scanner.jl @@ -136,9 +136,7 @@ function reset!(stream::TokenStream) end -function get_mark(stream::TokenStream) - Mark(stream.index, stream.line, stream.column) -end +Mark(stream::TokenStream) = Mark(stream.index, stream.line, stream.column) # Advance the stream by k characters. @@ -289,7 +287,7 @@ function stale_possible_simple_keys(stream::TokenStream) if key.mark.line != stream.line || stream.index - key.mark.index > 1024 if key.required throw(ScannerError("while scanning a simple key", key.mark, - "could not find expected ':'", get_mark(stream))) + "could not find expected ':'", Mark(stream))) end delete!(stream.possible_simple_keys, level) end @@ -305,7 +303,7 @@ function save_possible_simple_key(stream::TokenStream) if stream.allow_simple_key remove_possible_simple_key(stream) token_number = stream.tokens_taken + length(stream.token_queue) - key = SimpleKey(token_number, required, get_mark(stream)) + key = SimpleKey(token_number, required, Mark(stream)) stream.possible_simple_keys[stream.flow_level] = key end end @@ -317,7 +315,7 @@ function remove_possible_simple_key(stream::TokenStream) key = stream.possible_simple_keys[stream.flow_level] if key.required throw(ScannerError("while scanning a simple key", key.mark, - "could not find expected ':'", get_mark(stream))) + "could not find expected ':'", Mark(stream))) end delete!(stream.possible_simple_keys, stream.flow_level) end @@ -333,7 +331,7 @@ function unwind_indent(stream::TokenStream, column) # In block context, we may need to issue the BLOCK-END tokens. while stream.indent > column - mark = get_mark(stream) + mark = Mark(stream) stream.indent = pop!(stream.indents) enqueue!(stream.token_queue, BlockEndToken(Span(mark, mark))) end @@ -399,7 +397,7 @@ end # -------- function fetch_stream_start(stream::TokenStream) - mark = get_mark(stream) + mark = Mark(stream) enqueue!(stream.token_queue, StreamStartToken(Span(mark, mark), string(stream.encoding))) end @@ -414,7 +412,7 @@ function fetch_stream_end(stream::TokenStream) stream.allow_simple_key = false empty!(stream.possible_simple_keys) - mark = get_mark(stream) + mark = Mark(stream) enqueue!(stream.token_queue, StreamEndToken(Span(mark, mark))) stream.done = true end @@ -452,9 +450,9 @@ function fetch_document_indicator(stream::TokenStream, ::Type{T}) where {T<:Toke stream.allow_simple_key = false # Add DOCUMENT-START or DOCUMENT-END. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream, 3) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, T(Span(start_mark, end_mark))) end @@ -463,10 +461,10 @@ function fetch_byte_order_mark(stream::TokenStream) # Set the current intendation to -1. unwind_indent(stream, -1) - start_mark = get_mark(stream) + start_mark = Mark(stream) forward!(stream.input) stream.index += 1 - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, ByteOrderMarkToken(Span(start_mark, end_mark))) end @@ -493,9 +491,9 @@ function fetch_flow_collection_start(stream::TokenStream, ::Type{T}) where {T<:T # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, T(Span(start_mark, end_mark))) end @@ -521,9 +519,9 @@ function fetch_flow_collection_end(stream::TokenStream, ::Type{T}) where {T<:Tok stream.allow_simple_key = false # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, T(Span(start_mark, end_mark))) end @@ -536,9 +534,9 @@ function fetch_flow_entry(stream::TokenStream) remove_possible_simple_key(stream) # Add FLOW-ENTRY. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, FlowEntryToken(Span(start_mark, end_mark))) end @@ -550,11 +548,11 @@ function fetch_block_entry(stream::TokenStream) if !stream.allow_simple_key throw(ScannerError(nothing, nothing, "sequence entries not allowed here", - get_mark(stream))) + Mark(stream))) end if add_indent(stream, stream.column) - mark = get_mark(stream) + mark = Mark(stream) enqueue!(stream.token_queue, BlockSequenceStartToken(Span(mark, mark))) end @@ -572,9 +570,9 @@ function fetch_block_entry(stream::TokenStream) remove_possible_simple_key(stream) # Add BLOCK-ENTRY. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, BlockEntryToken(Span(start_mark, end_mark))) end @@ -586,12 +584,12 @@ function fetch_key(stream::TokenStream) if !stream.allow_simple_key throw(ScannerError(nothing, nothing, "mapping keys are not allowed here", - get_mark(stream))) + Mark(stream))) end # We may need to add BLOCK-MAPPING-START. if add_indent(stream, stream.column) - mark = get_mark(stream) + mark = Mark(stream) enqueue!(stream.token_queue, BlockMappingStartToken(Span(mark, mark))) end @@ -604,9 +602,9 @@ function fetch_key(stream::TokenStream) remove_possible_simple_key(stream) # Add KEY. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, KeyToken(Span(start_mark, end_mark))) end @@ -642,7 +640,7 @@ function fetch_value(stream::TokenStream) if !stream.allow_simple_key throw(ScannerError(nothing, nothing, "mapping values are not allowed here", - get_mark(stream))) + Mark(stream))) end end @@ -650,7 +648,7 @@ function fetch_value(stream::TokenStream) # BLOCK-MAPPING-START. It will be detected as an error later by # the parser. if stream.flow_level == 0 && add_indent(stream, stream.column) - mark = get_mark(stream) + mark = Mark(stream) enqueue!(stream.token_queue, BlockMappingStartToken(Span(mark, mark))) end @@ -663,9 +661,9 @@ function fetch_value(stream::TokenStream) end # Add VALUE. - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) enqueue!(stream.token_queue, ValueToken(Span(start_mark, end_mark))) end @@ -816,22 +814,22 @@ end function scan_directive(stream::TokenStream) - start_mark = get_mark(stream) + start_mark = Mark(stream) forwardchars!(stream) name = scan_directive_name(stream, start_mark) value = nothing if name == "YAML" value = scan_yaml_directive_value(stream, start_mark) - end_mark = get_mark(stream) + end_mark = Mark(stream) elseif name == "TAG" tag_handle = scan_tag_directive_handle(stream, start_mark) tag_prefix = scan_tag_directive_prefix(stream, start_mark) value = (tag_handle, tag_prefix) - end_mark = get_mark(stream) + end_mark = Mark(stream) else # Otherwise we warn and ignore the directive. - end_mark = get_mark(stream) + end_mark = Mark(stream) @warn """unknown directive name: "$name" at $end_mark. We ignore this.""" while !in(peek(stream.input), "\0\r\n\u0085\u2028\u2029") forwardchars!(stream) @@ -854,7 +852,7 @@ function scan_directive_name(stream::TokenStream, start_mark::Mark) if length == 0 throw(ScannerError("while scanning a directive", start_mark, "expected alphanumeric character, but found '$(c)'", - get_mark(stream))) + Mark(stream))) end value = prefix(stream.input, length) @@ -864,7 +862,7 @@ function scan_directive_name(stream::TokenStream, start_mark::Mark) if !in(c, ":\0 \r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a directive", start_mark, "expected alphanumeric character, but found '$(c)'", - get_mark(stream))) + Mark(stream))) end value @@ -880,14 +878,14 @@ function scan_yaml_directive_value(stream::TokenStream, start_mark::Mark) if peek(stream.input) != '.' throw(ScannerError("while scanning a directive", start_mark, "expected '.' but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end forwardchars!(stream) minor = scan_yaml_directive_number(stream, start_mark) if !in(peek(stream.input), "\0 \r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a directive", start_mark, "expected ' ' or a line break, but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end return (major, minor) end @@ -905,7 +903,7 @@ function scan_yaml_directive_number(stream::TokenStream, start_mark::Mark)::Int # throw an error if the input is not decimal digits isdigit(c) || throw(ScannerError( "while scanning a directive", start_mark, - "expected a digit, but found '$c'", get_mark(stream), + "expected a digit, but found '$c'", Mark(stream), )) # ----------------------------------------------------------- # until the end of the decimal digits, increment the position @@ -942,7 +940,7 @@ function scan_tag_directive_handle(stream::TokenStream, start_mark::Mark) if peek(stream.input) != ' ' throw(ScannerError("while scanning a directive", start_mark, "expected ' ', but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end value end @@ -957,7 +955,7 @@ function scan_tag_directive_prefix(stream::TokenStream, start_mark::Mark) if !in(peek(stream.input), "\0 \r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a directive", start_mark, "expected ' ', but found $(peek(stream.input))", - get_mark(stream))) + Mark(stream))) end value end @@ -976,14 +974,14 @@ function scan_directive_ignored_line(stream::TokenStream, start_mark::Mark) if !in(peek(stream.input), "\0\r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a directive", start_mark, "expected a comment or a line break, but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end scan_line_break(stream) end function scan_anchor(stream::TokenStream, ::Type{T}) where {T<:Token} - start_mark = get_mark(stream) + start_mark = Mark(stream) indicator = peek(stream.input) if indicator == '*' name = "alias" @@ -1001,22 +999,22 @@ function scan_anchor(stream::TokenStream, ::Type{T}) where {T<:Token} if length == 0 throw(ScannerError("while scanning an $(name)", start_mark, "expected an alphanumeric character, but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end value = prefix(stream.input, length) forwardchars!(stream, length) if !in(peek(stream.input), "\0 \t\r\n\u0085\u2028\u2029?:,]}%@`") throw(ScannerError("while scanning an $(name)", start_mark, "expected an alphanumeric character, but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end - end_mark = get_mark(stream) + end_mark = Mark(stream) T(Span(start_mark, end_mark), value) end function scan_tag(stream::TokenStream) - start_mark = get_mark(stream) + start_mark = Mark(stream) c = peek(stream.input, 1) if c == '<' handle = nothing @@ -1025,7 +1023,7 @@ function scan_tag(stream::TokenStream) if peek(stream.input) != '>' throw(ScannerError("while parsing a tag", start_mark, "expected '>', but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end forwardchars!(stream) elseif in(c, "\0 \t\r\n\u0085\u2028\u2029") @@ -1056,11 +1054,11 @@ function scan_tag(stream::TokenStream) if !in(c, "\0 \r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a tag", start_mark, "expected ' ' or a line break, but found '$(c)'", - get_mark(stream))) + Mark(stream))) end value = (handle, suffix) - end_mark = get_mark(stream) + end_mark = Mark(stream) TagToken(Span(start_mark, end_mark), value) end @@ -1069,7 +1067,7 @@ function scan_block_scalar(stream::TokenStream, style::Char) folded = style == '>' chunks = Any[] - start_mark = get_mark(stream) + start_mark = Mark(stream) # Scan the header. forwardchars!(stream) @@ -1140,7 +1138,7 @@ function scan_block_scalar_ignored_line(stream::TokenStream, start_mark::Mark) if !in(peek(stream.input), "\0\r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a block scalal", start_mark, "expected a comment or a line break, but found '$(peek(stream.input))'", - get_mark(stream))) + Mark(stream))) end scan_line_break(stream) @@ -1160,7 +1158,7 @@ function scan_block_scalar_indicators(stream::TokenStream, start_mark::Mark) if increment == 0 throw(ScannerError("while scanning a block scalar", start_mark, "expected indentation indicator in the range 1-9, but found 0", - get_mark(stream))) + Mark(stream))) end end elseif in(c, "0123456789") @@ -1168,7 +1166,7 @@ function scan_block_scalar_indicators(stream::TokenStream, start_mark::Mark) if increment == 0 throw(ScannerError("while scanning a block scalar", start_mark, "expected indentation indicator in the range 1-9, but found 0", - get_mark(stream))) + Mark(stream))) end forwardchars!(stream) @@ -1183,7 +1181,7 @@ function scan_block_scalar_indicators(stream::TokenStream, start_mark::Mark) if !in(c, "\0 \r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a block scalar", start_mark, "expected chomping or indentation indicators, but found '$(c)'", - get_mark(stream))) + Mark(stream))) end chomping, increment @@ -1193,11 +1191,11 @@ end function scan_block_scalar_indentation(stream::TokenStream) chunks = Any[] max_indent = 0 - end_mark = get_mark(stream) + end_mark = Mark(stream) while in(peek(stream.input), " \r\n\u0085\u2028\u2029") if peek(stream.input) != ' ' push!(chunks, scan_line_break(stream)) - end_mark = get_mark(stream) + end_mark = Mark(stream) else forwardchars!(stream) if stream.column > max_indent @@ -1212,14 +1210,14 @@ end function scan_block_scalar_breaks(stream::TokenStream, indent) chunks = Any[] - end_mark = get_mark(stream) + end_mark = Mark(stream) while stream.column < indent && peek(stream.input) == ' ' forwardchars!(stream) end while yaml_1_1_is_b_char(peek(stream.input)) push!(chunks, scan_line_break(stream)) - end_mark = get_mark(stream) + end_mark = Mark(stream) while stream.column < indent && peek(stream.input) == ' ' forwardchars!(stream) end @@ -1232,7 +1230,7 @@ end function scan_flow_scalar(stream::TokenStream, style::Char) double = style == '"' chunks = Any[] - start_mark = get_mark(stream) + start_mark = Mark(stream) q = peek(stream.input) # quote forwardchars!(stream) @@ -1242,7 +1240,7 @@ function scan_flow_scalar(stream::TokenStream, style::Char) end forwardchars!(stream) - end_mark = get_mark(stream) + end_mark = Mark(stream) ScalarToken(Span(start_mark, end_mark), string(chunks...), false, style) end @@ -1312,7 +1310,7 @@ function scan_flow_scalar_non_spaces(stream::TokenStream, double::Bool, string("expected escape sequence of", " $(length) hexadecimal", "digits, but found '$(c)'"), - get_mark(stream))) + Mark(stream))) end end push!(chunks, Char(parse(Int, prefix(stream.input, length), base = 16))) @@ -1324,7 +1322,7 @@ function scan_flow_scalar_non_spaces(stream::TokenStream, double::Bool, throw(ScannerError("while scanning a double-quoted scalar", start_mark, "found unknown escape character '$(c)'", - get_mark(stream))) + Mark(stream))) end else return chunks @@ -1346,7 +1344,7 @@ function scan_flow_scalar_spaces(stream::TokenStream, double::Bool, c = peek(stream.input) if c == '\0' throw(ScannerError("while scanning a quoted scalar", start_mark, - "found unexpected end of stream", get_mark(stream))) + "found unexpected end of stream", Mark(stream))) elseif yaml_1_1_is_b_char(c) line_break = scan_line_break(stream) breaks = scan_flow_scalar_breaks(stream, double, start_mark) @@ -1373,7 +1371,7 @@ function scan_flow_scalar_breaks(stream::TokenStream, double::Bool, in(peek(stream.input, 3), "\0 \t\r\n\u0085\u2028\u2029") throw(ScannerError("while scanning a quoted scalar", start_mark, "found unexpected document seperator", - get_mark(stream))) + Mark(stream))) end while in(peek(stream.input), " \t") @@ -1396,7 +1394,7 @@ function scan_plain(stream::TokenStream) # We also keep track of the `allow_simple_key` flag here. # Indentation rules are loosed for the flow context. chunks = Any[] - start_mark = get_mark(stream) + start_mark = Mark(stream) end_mark = start_mark indent = stream.indent + 1 @@ -1430,7 +1428,7 @@ function scan_plain(stream::TokenStream) !in(peek(stream.input, length + 1), "\0 \t\r\n\u0085\u2028\u2029,[]{}") forwardchars!(stream, length) throw(ScannerError("while scanning a plain scalar", start_mark, - "found unexpected ':'", get_mark(stream))) + "found unexpected ':'", Mark(stream))) end if length == 0 @@ -1441,7 +1439,7 @@ function scan_plain(stream::TokenStream) append!(chunks, spaces) push!(chunks, prefix(stream.input, length)) forwardchars!(stream, length) - end_mark = get_mark(stream) + end_mark = Mark(stream) spaces = scan_plain_spaces(stream, indent, start_mark) if isempty(spaces) || peek(stream.input) == '#' || (stream.flow_level == 0 && stream.column < indent) @@ -1510,7 +1508,7 @@ function scan_tag_handle(stream::TokenStream, name::String, start_mark::Mark) c = peek(stream.input) if c != '!' throw(ScannerError("while scanning a $(name)", start_mark, - "expected '!', but found '$(c)'", get_mark(stream))) + "expected '!', but found '$(c)'", Mark(stream))) end length = 1 c = peek(stream.input, length) @@ -1524,7 +1522,7 @@ function scan_tag_handle(stream::TokenStream, name::String, start_mark::Mark) forwardchars!(stream, length) throw(ScannerError("while scanning a $(name)", start_mark, "expected '!', but found '$(c)'", - get_mark(stream))) + Mark(stream))) end length += 1 end @@ -1560,7 +1558,7 @@ function scan_tag_uri(stream::TokenStream, name::String, start_mark::Mark) if isempty(chunks) throw(ScannerError("while parsing a $(name)", start_mark, "expected URI, but found '$(c)'", - get_mark(stream))) + Mark(stream))) end string(chunks...) @@ -1569,7 +1567,7 @@ end function scan_uri_escapes(stream::TokenStream, name::String, start_mark::Mark) bytes = Any[] - mark = get_mark(stream) + mark = Mark(stream) while peek(stream.input) == '%' forward!(stream.input) for k in 0:1 @@ -1578,7 +1576,7 @@ function scan_uri_escapes(stream::TokenStream, name::String, start_mark::Mark) string("expected URI escape sequence of", " 2 hexadecimal digits, but found", " '$(peek(stream.input, k))'"), - get_mark(stream))) + Mark(stream))) end end push!(bytes, Char(parse(Int, prefix(stream.input, 2), base=16)))