diff --git a/Project.toml b/Project.toml index 5667558..a84abe4 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "YAML" uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" -version = "0.4.10" +version = "0.4.11" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/YAML.jl b/src/YAML.jl index 1a1eb05..880c8f0 100644 --- a/src/YAML.jl +++ b/src/YAML.jl @@ -1,5 +1,22 @@ -__precompile__(true) +""" + YAML +A package to treat YAML. +https://github.com/JuliaData/YAML.jl + +Reading: + +* `YAML.load` parses the first YAML document of a YAML file as a Julia object. +* `YAML.load_all` parses the all YAML documents of a YAML file. +* `YAML.load_file` is same with `YAML.load` except it reads from a file. +* `YAML.load_all_file` is same with `YAML.load_all` except it reads from a file. + +Writing: + +* `YAML.write` prints a Julia object as a YAML file. +* `YAML.write_file` is same with `YAML.write` except it writes to a file. +* `YAML.yaml` converts a given Julia object to a YAML-formatted string. +""" module YAML import Base: isempty, length, show, peek @@ -10,8 +27,14 @@ using Dates using Printf using StringEncodings +include("queue.jl") +include("buffered_input.jl") +include("tokens.jl") include("scanner.jl") +include("events.jl") include("parser.jl") +include("nodes.jl") +include("resolver.jl") include("composer.jl") include("constructor.jl") include("writer.jl") # write Julia dictionaries to YAML files @@ -21,7 +44,7 @@ const _dicttype = Union{Type,Function} # add a dicttype-aware version of construct_mapping to the constructors function _patch_constructors(more_constructors::_constructor, dicttype::_dicttype) - if more_constructors == nothing + if more_constructors === nothing more_constructors = Dict{String,Function}() else more_constructors = copy(more_constructors) # do not change the outside world @@ -35,6 +58,12 @@ function _patch_constructors(more_constructors::_constructor, dicttype::_dicttyp end +""" + load(x::Union{AbstractString, IO}) + +Parse the string or stream `x` as a YAML file, and return the first YAML document as a +Julia object. +""" load(ts::TokenStream, constructor::Constructor) = construct_document(constructor, compose(EventStream(ts))) @@ -47,6 +76,12 @@ load(ts::TokenStream, more_constructors::_constructor = nothing, multi_construct load(input::IO, more_constructors::_constructor = nothing, multi_constructors::Dict = Dict(); kwargs...) = load(TokenStream(input), more_constructors, multi_constructors ; kwargs...) +""" + YAMLDocIterator + +An iterator type to represent multiple YAML documents. You can retrieve each YAML document +as a Julia object by iterating. +""" mutable struct YAMLDocIterator input::IO ts::TokenStream @@ -85,6 +120,11 @@ iterate(it::YAMLDocIterator, s) = done(it, s) ? nothing : next(it, s) Base.IteratorSize(::Type{YAMLDocIterator}) = Base.SizeUnknown() Base.IteratorEltype(::Type{YAMLDocIterator}) = Base.EltypeUnknown() +""" + load_all(x::Union{AbstractString, IO}) -> YAMLDocIterator + +Parse the string or stream `x` as a YAML file, and return corresponding YAML documents. +""" load_all(input::IO, args...; kwargs...) = YAMLDocIterator(input, args...; kwargs...) @@ -94,11 +134,21 @@ load(input::AbstractString, args...; kwargs...) = load_all(input::AbstractString, args...; kwargs...) = load_all(IOBuffer(input), args...; kwargs...) +""" + load_file(filename::AbstractString) + +Parse the YAML file `filename`, and return the first YAML document as a Julia object. +""" load_file(filename::AbstractString, args...; kwargs...) = open(filename, "r") do input load(input, args...; kwargs...) end +""" + load_all_file(filename::AbstractString) -> YAMLDocIterator + +Parse the YAML file `filename`, and return corresponding YAML documents. +""" load_all_file(filename::AbstractString, args...; kwargs...) = load_all(open(filename, "r"), args...; kwargs...) end # module diff --git a/src/composer.jl b/src/composer.jl index b410396..135c44b 100644 --- a/src/composer.jl +++ b/src/composer.jl @@ -1,8 +1,4 @@ -include("nodes.jl") -include("resolver.jl") - - struct ComposerError context::Union{String, Nothing} context_mark::Union{Mark, Nothing} @@ -18,7 +14,7 @@ struct ComposerError end function show(io::IO, error::ComposerError) - if error.context != nothing + if error.context !== nothing print(io, error.context, " at ", error.context_mark, ": ") end print(io, error.problem, " at ", error.problem_mark) @@ -34,21 +30,21 @@ end function compose(events) composer = Composer(events, Dict{String, Node}(), Resolver()) - @assert typeof(forward!(composer.input)) == StreamStartEvent + @assert forward!(composer.input) isa StreamStartEvent node = compose_document(composer) - if typeof(peek(composer.input)) == StreamEndEvent + if peek(composer.input) isa StreamEndEvent forward!(composer.input) else - @assert typeof(peek(composer.input)) == DocumentStartEvent + @assert peek(composer.input) isa DocumentStartEvent end node end function compose_document(composer::Composer) - @assert typeof(forward!(composer.input)) == DocumentStartEvent + @assert forward!(composer.input) isa DocumentStartEvent node = compose_node(composer) - @assert typeof(forward!(composer.input)) == DocumentEndEvent + @assert forward!(composer.input) isa DocumentEndEvent empty!(composer.anchors) node end @@ -134,7 +130,9 @@ function _compose_sequence_node(start_event::SequenceStartEvent, composer, ancho composer.anchors[anchor] = node end - while (event = peek(composer.input)) !== nothing + while true + event = peek(composer.input) + event === nothing && break __compose_sequence_node(event, composer, node) || break end diff --git a/src/constructor.jl b/src/constructor.jl index 71d8a69..711ced7 100644 --- a/src/constructor.jl +++ b/src/constructor.jl @@ -15,7 +15,7 @@ struct ConstructorError end function show(io::IO, error::ConstructorError) - if error.context != nothing + if error.context !== nothing print(io, error.context, " at ", error.context_mark, ": ") end print(io, error.problem, " at ", error.problem_mark) @@ -147,7 +147,7 @@ function flatten_mapping(node::MappingNode) elseif value_node isa SequenceNode submerge = [] for subnode in value_node.value - if typeof(subnode) != MappingNode + if !(subnode isa MappingNode) throw(ConstructorError("while constructing a mapping", node.start_mark, "expected a mapping node, but found $(typeof(subnode))", diff --git a/src/parser.jl b/src/parser.jl index 182b861..ece2432 100644 --- a/src/parser.jl +++ b/src/parser.jl @@ -1,6 +1,4 @@ -include("events.jl") - const DEFAULT_TAGS = Dict{String,String}("!" => "!", "!!" => "tag:yaml.org,2002:") @@ -19,7 +17,7 @@ struct ParserError end function show(io::IO, error::ParserError) - if error.context != nothing + if error.context !== nothing print(io, error.context, " at ", error.context_mark, ": ") end print(io, error.problem, " at ", error.problem_mark) @@ -83,7 +81,7 @@ end function process_directives(stream::EventStream) stream.yaml_version = nothing stream.tag_handles = Dict{String, String}() - while typeof(peek(stream.input)) == DirectiveToken + while peek(stream.input) isa DirectiveToken token = forward!(stream.input) if token.name == "YAML" if stream.yaml_version !== nothing @@ -138,11 +136,11 @@ end function parse_implicit_document_start(stream::EventStream) token = peek(stream.input) # Parse a byte order mark - if typeof(token) == ByteOrderMarkToken + if token isa ByteOrderMarkToken forward!(stream.input) token = peek(stream.input) end - if !in(typeof(token), [DirectiveToken, DocumentStartToken, StreamEndToken]) + if !(token isa Union{DirectiveToken, DocumentStartToken, StreamEndToken}) stream.tag_handles = DEFAULT_TAGS event = DocumentStartEvent(token.span.start_mark, token.span.start_mark, false) @@ -159,22 +157,22 @@ end function parse_document_start(stream::EventStream) # Parse any extra document end indicators. - while typeof(peek(stream.input)) == DocumentEndToken + while peek(stream.input) isa DocumentEndToken stream.input = Iterators.rest(stream.input) end token = peek(stream.input) # Parse a byte order mark if it exists - if typeof(token) == ByteOrderMarkToken + if token isa ByteOrderMarkToken forward!(stream.input) token = peek(stream.input) end # Parse explicit document. - if typeof(token) != StreamEndToken + if !(token isa StreamEndToken) start_mark = token.span.start_mark version, tags = process_directives(stream) - if typeof(peek(stream.input)) != DocumentStartToken + if !(peek(stream.input) isa DocumentStartToken) throw(ParserError(nothing, nothing, "expected '' but found $(typeof(token))")) end @@ -200,7 +198,7 @@ function parse_document_end(stream::EventStream) token = peek(stream.input) start_mark = end_mark = token.span.start_mark explicit = false - if typeof(token) == DocumentEndToken + if token isa DocumentEndToken forward!(stream.input) end_mark = token.span.end_mark explicit = true @@ -214,7 +212,7 @@ end function parse_document_content(stream::EventStream) - if in(peek(stream.input), [DirectiveToken, DocumentStartToken, DocumentEndToken,StreamEndToken]) + if peek(stream.input) isa Union{DirectiveToken, DocumentStartToken, DocumentEndToken, StreamEndToken} event = process_empty_scalar(stream, peek(stream.input).span.start_mark) stream.state = pop!(stream.states) event @@ -248,9 +246,9 @@ end function __parse_node(token::ScalarToken, stream::EventStream, block, start_mark, end_mark, anchor, tag, implicit) forward!(stream.input) end_mark = token.span.end_mark - if (token.plain && tag == nothing) || tag == "!" + if (token.plain && tag === nothing) || tag == "!" implicit = true, false - elseif tag == nothing + elseif tag === nothing implicit = false, true else implicit = false, false @@ -307,25 +305,25 @@ function _parse_node(token, stream::EventStream, block, indentless_sequence) anchor = nothing tag = nothing start_mark = end_mark = tag_mark = nothing - if typeof(token) == AnchorToken + if token isa AnchorToken forward!(stream.input) start_mark = token.span.start_mark end_mark = token.span.end_mark anchor = token.value token = peek(stream.input) - if typeof(token) == TagToken + if token isa TagToken forward!(stream.input) tag_mark = token.span.start_mark end_mark = token.span.end_mark tag = token.value end - elseif typeof(token) == TagToken + elseif token isa TagToken forward!(stream.input) start_mark = token.span.start_mark end_mark = token.span.end_mark tag = token.value token = peek(stream.input) - if typeof(token) == AnchorToken + if token isa AnchorToken forward!(stream.input) end_mark = token.end_mark anchor = token.value @@ -347,13 +345,13 @@ function _parse_node(token, stream::EventStream, block, indentless_sequence) end token = peek(stream.input) - if start_mark == nothing + if start_mark === nothing start_mark = end_mark = token.span.start_mark end event = nothing implicit = tag === nothing || tag == "!" - if indentless_sequence && typeof(token) == BlockEntryToken + if indentless_sequence && token isa BlockEntryToken end_mark = token.span.end_mark stream.state = parse_indentless_sequence_entry event = SequenceStartEvent(start_mark, end_mark, anchor, tag, implicit, @@ -379,9 +377,9 @@ end function parse_block_sequence_entry(stream::EventStream) token = peek(stream.input) - if typeof(token) == BlockEntryToken + if token isa BlockEntryToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [BlockEntryToken, BlockEndToken]) + if !(peek(stream.input) isa Union{BlockEntryToken, BlockEndToken}) push!(stream.states, parse_block_sequence_entry) return parse_block_node(stream) else @@ -390,7 +388,7 @@ function parse_block_sequence_entry(stream::EventStream) end end - if typeof(token) != BlockEndToken + if !(token isa BlockEndToken) throw(ParserError("while parsing a block collection", stream.marks[end], "expected , but found $(typeof(token))", token.span.start_mark)) @@ -405,9 +403,9 @@ end function parse_indentless_sequence_entry(stream::EventStream) token = peek(stream.input) - if typeof(token) == BlockEntryToken + if token isa BlockEntryToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [BlockEntryToken, KeyToken, ValueToken, BlockEndToken]) + if !(peek(stream.input) isa Union{BlockEntryToken, KeyToken, ValueToken, BlockEndToken}) push!(stream.states, parse_indentless_sequence_entry) return parse_block_node(stream) else @@ -430,9 +428,9 @@ end function parse_block_mapping_key(stream::EventStream) token = peek(stream.input) - if typeof(token) == KeyToken + if token isa KeyToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [KeyToken, ValueToken, BlockEndToken]) + if !(peek(stream.input) isa Union{KeyToken, ValueToken, BlockEndToken}) push!(stream.states, parse_block_mapping_value) return parse_block_node_or_indentless_sequence(stream) else @@ -441,7 +439,7 @@ function parse_block_mapping_key(stream::EventStream) end end - if typeof(token) != BlockEndToken + if !(token isa BlockEndToken) throw(ParserError("while parsing a block mapping", stream.marks[end], "expected , but found $(typeof(token))", token.span.start_mark)) @@ -456,9 +454,9 @@ end function parse_block_mapping_value(stream::EventStream) token = peek(stream.input) - if typeof(token) == ValueToken + if token isa ValueToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [KeyToken, ValueToken, BlockEndToken]) + if !(peek(stream.input) isa Union{KeyToken, ValueToken, BlockEndToken}) push!(stream.states, parse_block_mapping_key) parse_block_node_or_indentless_sequence(stream) else @@ -487,7 +485,7 @@ end function _parse_flow_sequence_entry(token::Any, stream::EventStream, first_entry=false) if !first_entry - if typeof(token) == FlowEntryToken + if token isa FlowEntryToken forward!(stream.input) else throw(ParserError("while parsing a flow sequence", @@ -517,7 +515,7 @@ end function parse_flow_sequence_entry_mapping_key(stream::EventStream) token = forward!(stream.input) - if !in(typeof(token), [ValueToken, FlowEntryToken, FlowSequenceEndToken]) + if !(token isa Union{ValueToken, FlowEntryToken, FlowSequenceEndToken}) push!(stream.states, parse_flow_sequence_entry_mapping_value) parse_flow_node(stream) else @@ -529,9 +527,9 @@ end function parse_flow_sequence_entry_mapping_value(stream::EventStream) token = peek(stream.input) - if typeof(token) == ValueToken + if token isa ValueToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [FlowEntryToken, FlowSequenceEndToken]) + if !(peek(stream.input) isa Union{FlowEntryToken, FlowSequenceEndToken}) push!(stream.states, parse_flow_sequence_entry_mapping_end) parse_flow_node(stream) else @@ -561,9 +559,9 @@ end function parse_flow_mapping_key(stream::EventStream, first_entry=false) token = peek(stream.input) - if typeof(token) != FlowMappingEndToken + if !(token isa FlowMappingEndToken) if !first_entry - if typeof(token) == FlowEntryToken + if token isa FlowEntryToken forward!(stream.input) else throw(ParserError("while parsing a flow mapping", @@ -574,16 +572,16 @@ function parse_flow_mapping_key(stream::EventStream, first_entry=false) end token = peek(stream.input) - if typeof(token) == KeyToken + if token isa KeyToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [ValueToken, FlowEntryToken, FlowMappingEndToken]) + if !(peek(stream.input) isa Union{ValueToken, FlowEntryToken, FlowMappingEndToken}) push!(stream.states, parse_flow_mapping_value) return parse_flow_node(stream) else stream.state = parse_flow_mapping_value return process_empty_scalar(stream, token.span.end_mark) end - elseif typeof(token) != FlowMappingEndToken + elseif !(token isa FlowMappingEndToken) push!(stream.states, parse_flow_mapping_empty_value) return parse_flow_node(stream) end @@ -598,9 +596,9 @@ end function parse_flow_mapping_value(stream::EventStream) token = peek(stream.input) - if typeof(token) == ValueToken + if token isa ValueToken forward!(stream.input) - if !in(typeof(peek(stream.input)), [FlowEntryToken, FlowMappingEndToken]) + if !(peek(stream.input) isa Union{FlowEntryToken, FlowMappingEndToken}) push!(stream.states, parse_flow_mapping_key) parse_flow_node(stream) else diff --git a/src/resolver.jl b/src/resolver.jl index 9296d39..ff42f05 100644 --- a/src/resolver.jl +++ b/src/resolver.jl @@ -58,7 +58,7 @@ const default_implicit_resolvers = ] -mutable struct Resolver +struct Resolver implicit_resolvers::Vector{Tuple{String,Regex}} function Resolver() diff --git a/src/scanner.jl b/src/scanner.jl index 718095a..c92b60d 100644 --- a/src/scanner.jl +++ b/src/scanner.jl @@ -1,27 +1,4 @@ -include("queue.jl") -include("buffered_input.jl") - -# Position within the document being parsed -struct Mark - index::UInt64 - line::UInt64 - column::UInt64 -end - - -function show(io::IO, mark::Mark) - @printf(io, "line %d, column %d", mark.line, mark.column) -end - - -# Where in the stream a particular token lies. -struct Span - start_mark::Mark - end_mark::Mark -end - - struct SimpleKey token_number::UInt64 required::Bool @@ -45,9 +22,6 @@ function show(io::IO, error::ScannerError) end -include("tokens.jl") - - function detect_encoding(input::IO)::Encoding pos = position(input) start_bytes = Array{UInt8}(undef, 4) @@ -850,7 +824,9 @@ function scan_directive(stream::TokenStream) value = (tag_handle, tag_prefix) end_mark = get_mark(stream) else + # Otherwise we warn and ignore the directive. end_mark = get_mark(stream) + @warn """unknown directive name: "$name" at $end_mark. We ignore this.""" while !in(peek(stream.input), "\0\r\n\u0085\u2028\u2029") forwardchars!(stream) end @@ -1575,7 +1551,7 @@ function scan_uri_escapes(stream::TokenStream, name::String, start_mark::Mark) get_mark(stream))) end end - push!(bytes, char(parse_hex(prefix(stream.input, 2)))) + push!(bytes, Char(parse(Int, prefix(stream.input, 2), base=16))) forwardchars!(stream, 2) end diff --git a/src/tokens.jl b/src/tokens.jl index eabd55b..0b279de 100644 --- a/src/tokens.jl +++ b/src/tokens.jl @@ -1,4 +1,24 @@ +# Position within the document being parsed +struct Mark + index::UInt64 + line::UInt64 + column::UInt64 +end + + +function show(io::IO, mark::Mark) + @printf(io, "line %d, column %d", mark.line, mark.column) +end + + +# Where in the stream a particular token lies. +struct Span + start_mark::Mark + end_mark::Mark +end + + # YAML Tokens. # Each token must include at minimum member "span::Span". abstract type Token end diff --git a/test/runtests.jl b/test/runtests.jl index 2100fbd..d76119f 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -477,4 +477,15 @@ end @test_throws YAML.ScannerError YAML.load(""" '''a'' """) end +# issue #148 - warn unknown directives +@testset "issue #148" begin + @test (@test_logs (:warn, """unknown directive name: "FOO" at line 1, column 4. We ignore this.""") YAML.load("""%FOO bar baz\n\n--- "foo\"""")) == "foo" + @test (@test_logs (:warn, """unknown directive name: "FOO" at line 1, column 4. We ignore this.""") (:warn, """unknown directive name: "BAR" at line 2, column 4. We ignore this.""") YAML.load("""%FOO\n%BAR\n--- foo""")) == "foo" +end + +# issue #144 +@testset "issue #144" begin + @test YAML.load("---") === nothing +end + end # module