From 2cbe65b0a13c8a2ce91bd4799f5b96b8e1953097 Mon Sep 17 00:00:00 2001 From: Dani Miba <43004687+danimiba@users.noreply.github.com> Date: Tue, 5 May 2020 11:28:54 -0300 Subject: [PATCH 001/263] Remove Bountysource badge and reference in README (#9225) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3dac5a0b596f..4c073d5b15bd 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![Travis CI Build Status](https://travis-ci.org/crystal-lang/crystal.svg)](https://travis-ci.org/crystal-lang/crystal) [![CircleCI Build Status](https://circleci.com/gh/crystal-lang/crystal/tree/master.svg?style=shield)](https://circleci.com/gh/crystal-lang/crystal) [![Join the chat at https://gitter.im/crystal-lang/crystal](https://badges.gitter.im/crystal-lang/crystal.svg)](https://gitter.im/crystal-lang/crystal) -[![Bountysource](https://api.bountysource.com/badge/team?team_id=89730&style=raised)](https://salt.bountysource.com/teams/crystal-lang) [![Code Triagers Badge](https://www.codetriage.com/crystal-lang/crystal/badges/users.svg)](https://www.codetriage.com/crystal-lang/crystal) --- @@ -38,7 +37,7 @@ Project Status Crystal is still under heavy development. There can be breaking changes but we're trying to keep them as minimum as possible. -The development is possible thanks to the community's effort, [84codes](https://www.84codes.com/)' support, and every [BountySource supporter](https://crystal-lang.org/sponsors). +The development is possible thanks to the community's effort and the continued support of [84codes](https://www.84codes.com/), [Nikola Motor Company](https://nikolamotor.com/) and every other [sponsor](https://crystal-lang.org/sponsors). Installing ---------- From 2f8195b21a9a33ef7f887b881a9dbc56ee51cd83 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 7 May 2020 13:37:23 -0300 Subject: [PATCH 002/263] Parser: fix parsing of capitalized named argument (#9232) --- spec/compiler/parser/parser_spec.cr | 2 ++ src/compiler/crystal/syntax/parser.cr | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index eef540a69702..e3123f5e3947 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -412,6 +412,8 @@ module Crystal it_parses %(foo("foo bar": 1, "baz": 2)), Call.new(nil, "foo", named_args: [NamedArgument.new("foo bar", 1.int32), NamedArgument.new("baz", 2.int32)]) it_parses %(foo "foo bar": 1, "baz": 2), Call.new(nil, "foo", named_args: [NamedArgument.new("foo bar", 1.int32), NamedArgument.new("baz", 2.int32)]) + it_parses %(foo(Foo: 1, Bar: 2)), Call.new(nil, "foo", named_args: [NamedArgument.new("Foo", 1.int32), NamedArgument.new("Bar", 2.int32)]) + it_parses "x.foo(a: 1, b: 2)", Call.new("x".call, "foo", named_args: [NamedArgument.new("a", 1.int32), NamedArgument.new("b", 2.int32)]) it_parses "x.foo a: 1, b: 2 ", Call.new("x".call, "foo", named_args: [NamedArgument.new("a", 1.int32), NamedArgument.new("b", 2.int32)]) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index fe40c93985b7..412a7445ef01 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -4301,7 +4301,7 @@ module Crystal return parse_call_block_arg(args, true) end - if @token.type == :IDENT && current_char == ':' + if named_tuple_start? return parse_call_args_named_args(@token.location, args, first_name: nil, allow_newline: true) else arg = parse_call_arg(found_double_splat) From d27f52486544b7b94f277e966855469051624b1a Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 7 May 2020 12:38:25 -0400 Subject: [PATCH 003/263] Support `TypeNode.name(generic_args: false)` for generic instances (#9224) --- spec/compiler/macro/macro_methods_spec.cr | 12 +++++++++ src/compiler/crystal/types.cr | 32 ++++++++++++----------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index b590b15341aa..7579a8596205 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1423,6 +1423,10 @@ module Crystal [TypeNode.new(GenericClassType.new(program, program, "SomeType", program.object, ["A", "B"]))] of ASTNode end end + + it "includes the generic_args of the instantiated type by default" do + assert_macro("", "{{Array(Int32).name}}", [] of ASTNode, "Array(Int32)") + end end describe :generic_args do @@ -1432,6 +1436,10 @@ module Crystal [TypeNode.new(GenericClassType.new(program, program, "SomeType", program.object, ["A", "B"]))] of ASTNode end end + + it "includes the generic_args of the instantiated type" do + assert_macro("", "{{Array(Int32).name(generic_args: true)}}", [] of ASTNode, "Array(Int32)") + end end describe false do @@ -1440,6 +1448,10 @@ module Crystal [TypeNode.new(GenericClassType.new(program, program, "SomeType", program.object, ["A", "B"]))] of ASTNode end end + + it "does not include the generic_args of the instantiated type" do + assert_macro("", "{{Array(Int32).name(generic_args: false)}}", [] of ASTNode, "Array") + end end describe "with an invalid type argument" do diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 18f56bafaf45..84ad3e1f3159 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -1980,26 +1980,28 @@ module Crystal def to_s_with_options(io : IO, skip_union_parens : Bool = false, generic_args : Bool = true, codegen : Bool = false) : Nil generic_type.append_full_name(io) - io << '(' - type_vars.each_value.with_index do |type_var, i| - io << ", " if i > 0 - if type_var.is_a?(Var) - if i == splat_index - tuple = type_var.type.as(TupleInstanceType) - tuple.tuple_types.join(", ", io) do |tuple_type| - tuple_type = tuple_type.devirtualize unless codegen - tuple_type.to_s_with_options(io, codegen: codegen) + if generic_args + io << '(' + type_vars.each_value.with_index do |type_var, i| + io << ", " if i > 0 + if type_var.is_a?(Var) + if i == splat_index + tuple = type_var.type.as(TupleInstanceType) + tuple.tuple_types.join(", ", io) do |tuple_type| + tuple_type = tuple_type.devirtualize unless codegen + tuple_type.to_s_with_options(io, codegen: codegen) + end + else + type_var_type = type_var.type + type_var_type = type_var_type.devirtualize unless codegen + type_var_type.to_s_with_options(io, skip_union_parens: true, codegen: codegen) end else - type_var_type = type_var.type - type_var_type = type_var_type.devirtualize unless codegen - type_var_type.to_s_with_options(io, skip_union_parens: true, codegen: codegen) + type_var.to_s(io) end - else - type_var.to_s(io) end + io << ')' end - io << ')' end end From f22ab9f6e3991dd6d7e2d0b9b8ca9d822e2490d2 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Thu, 7 May 2020 14:19:17 -0300 Subject: [PATCH 004/263] Force linking of `pthread_once` when compiling static binaries in musl (#9238) --- src/crystal/system/unix/pthread.cr | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 67e7979cbd70..005d71cfc17f 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -178,3 +178,19 @@ class Thread @th end end + +# In musl (alpine) the calls to unwind API segfaults +# when the binary is statically linked. This is because +# some symbols like `pthread_once` are defined as "weak" +# and, for some reason, not linked into the final binary. +# Adding an explicit reference to the symbol ensures it's +# included in the statically linked binary. +{% if flag?(:musl) && flag?(:static) %} + lib LibC + fun pthread_once(Void*, Void*) + end + + fun __crystal_static_musl_workaround + LibC.pthread_once(nil, nil) + end +{% end %} From 9efe317379358323ab14cb7299fda6d8c22565cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 7 May 2020 19:27:06 +0200 Subject: [PATCH 005/263] Refactor via private Path#empty? (#9137) * Add Path#empty? * Remove method from public API --- src/path.cr | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/path.cr b/src/path.cr index 9f4afcc6cf2d..03b817cde050 100644 --- a/src/path.cr +++ b/src/path.cr @@ -9,8 +9,8 @@ # A `Path` can represent a root, a root and a sequence of names, or simply one or # more name elements. # A `Path` is considered to be an empty path if it consists solely of one name -# element that is empty. Accessing a file using an empty path is equivalent -# to accessing the default directory of the process. +# element that is empty or equal to `"."`. Accessing a file using an empty path +# is equivalent to accessing the default directory of the process. # # # Examples # @@ -230,7 +230,7 @@ struct Path # Returns the parent path of this path. # - # If the path is empty or `"."`, it returns `"."`. If the path is rooted + # If the path is empty, it returns `"."`. If the path is rooted # and in the top-most hierarchy, the root path is returned. # # ``` @@ -267,7 +267,7 @@ struct Path # # Path["foo/bar"] # ``` def each_parent(&block : Path ->) - return if @name.empty? || @name == "." + return if empty? first_char = @name.char_at(0) unless separators.includes?(first_char) || (first_char == '.' && separators.includes?(@name.byte_at?(1).try &.unsafe_chr)) || (windows? && (windows_drive? || unc_share?)) @@ -398,7 +398,7 @@ struct Path # # See also Rob Pike: *[Lexical File Names in Plan 9 or Getting Dot-Dot Right](https://9p.io/sys/doc/lexnames.html)* def normalize(*, remove_final_separator : Bool = true) : Path - return new_instance "." if @name.empty? + return new_instance "." if empty? drive, root = drive_and_root reader = Char::Reader.new(@name) @@ -946,6 +946,10 @@ struct Path end end + private def empty? + @name.empty? || @name == "." + end + # Returns a relative path that is lexically equivalent to `self` when joined # to *base* with an intervening separator. # From e68767e4937a542b65bcec6f5d12b52dff8c701b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 7 May 2020 16:33:17 -0300 Subject: [PATCH 006/263] Log: Avoid hard breaking change in Log.setup_from_env (#9240) It covers all the usages found publicly * crystal-lang/shards (v0.10.0) * crystalshards/crystalshards * jessedoyle/duktape.cr * RomainFranceschini/quartz --- src/log/setup.cr | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/log/setup.cr b/src/log/setup.cr index 7371f8876b40..cb26c44d5962 100644 --- a/src/log/setup.cr +++ b/src/log/setup.cr @@ -26,6 +26,14 @@ class Log end end + @[Deprecated("Use default_level, default_sources named arguments")] + def self.setup_from_env(*, builder : Log::Builder = Log.builder, + level : String, + sources : String, + backend = Log::IOBackend.new) + Log.setup(sources, Log::Severity.parse(level), backend, builder: builder) + end + # Setups logging based on `LOG_LEVEL` environment variable. def self.setup_from_env(*, builder : Log::Builder = Log.builder, default_level : Log::Severity = Log::Severity::Info, From a10db96409e3474d6c1fbaf29df8086855890d8e Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Fri, 8 May 2020 06:05:14 -0700 Subject: [PATCH 007/263] Fix `crystal spec` file paths on Windows (#9234) The entry point file is being generated with the paths being pasted directly into source code (`require "string"`) but `require` always needs forward slashes, while backslashes actually end up being interpreted as escape sequences. --- src/compiler/crystal/command/spec.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/compiler/crystal/command/spec.cr b/src/compiler/crystal/command/spec.cr index 3a1137450478..41785ccc6af3 100644 --- a/src/compiler/crystal/command/spec.cr +++ b/src/compiler/crystal/command/spec.cr @@ -66,7 +66,9 @@ class Crystal::Command source_filename = File.expand_path("spec") - source = target_filenames.map { |filename| %(require "./#{filename}") }.join('\n') + source = target_filenames.map { |filename| + %(require "./#{::Path[filename].to_posix}") + }.join('\n') sources = [Compiler::Source.new(source_filename, source)] output_filename = Crystal.temp_executable "spec" From cfec5691e59a958fe2786100e7ff6f33f5dbc6ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 8 May 2020 15:06:37 +0200 Subject: [PATCH 008/263] Fix version selector in API docs shows current version twice (#9187) --- src/compiler/crystal/tools/doc/html/js/_versions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/doc/html/js/_versions.js b/src/compiler/crystal/tools/doc/html/js/_versions.js index 6e36aa6b49cc..bc09a98d6fb0 100644 --- a/src/compiler/crystal/tools/doc/html/js/_versions.js +++ b/src/compiler/crystal/tools/doc/html/js/_versions.js @@ -30,7 +30,7 @@ CrystalDocs.loadConfig = function (config) { var currentVersion = document.querySelector("html > head > meta[name=\"crystal_docs.project_version\"]").getAttribute("content") var currentVersionInList = projectVersions.find(function (element) { - element.version == currentVersion + element.name == currentVersion }) if (!currentVersionInList) { From 37063893393e935e3572d813b4a62f658168c73a Mon Sep 17 00:00:00 2001 From: Kubo Takehiro Date: Fri, 8 May 2020 22:10:02 +0900 Subject: [PATCH 009/263] Set `sync` or `flush_on_newline` for standard I/O on Windows. (#9207) * Extract platform-specific code in IO::FileDescriptor.from_stdio * Set `sync` or `flush_on_newline` for standard I/O on Windows. If starndard I/O is a handle to the console, set `sync` to true. Otherwise, set `flush_on_newline` to true as described in STDOUT and STDERR docs. --- src/crystal/system/unix/file_descriptor.cr | 20 +++++++++++++++++ src/crystal/system/win32/file_descriptor.cr | 21 ++++++++++++++++++ src/io/file_descriptor.cr | 22 +------------------ src/lib_c/x86_64-windows-msvc/c/consoleapi.cr | 5 +++++ 4 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 src/lib_c/x86_64-windows-msvc/c/consoleapi.cr diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 08f21e25ad26..40de66b85dbc 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -161,4 +161,24 @@ module Crystal::System::FileDescriptor bytes_read end + + def self.from_stdio(fd) + # If we have a TTY for stdin/out/err, it is possibly a shared terminal. + # We need to reopen it to use O_NONBLOCK without causing other programs to break + + # Figure out the terminal TTY name. If ttyname fails we have a non-tty, or something strange. + # For non-tty we set flush_on_newline to true for reasons described in STDOUT and STDERR docs. + path = uninitialized UInt8[256] + ret = LibC.ttyname_r(fd, path, 256) + return IO::FileDescriptor.new(fd).tap(&.flush_on_newline=(true)) unless ret == 0 + + clone_fd = LibC.open(path, LibC::O_RDWR) + return IO::FileDescriptor.new(fd).tap(&.flush_on_newline=(true)) if clone_fd == -1 + + # We don't buffer output for TTY devices to see their output right away + io = IO::FileDescriptor.new(clone_fd) + io.close_on_exec = true + io.sync = true + io + end end diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 12d85d6c6054..62fa4332a35d 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -1,4 +1,5 @@ require "c/io" +require "c/consoleapi" module Crystal::System::FileDescriptor @volatile_fd : Atomic(LibC::Int) @@ -162,4 +163,24 @@ module Crystal::System::FileDescriptor bytes_read end + + def self.from_stdio(fd) + console_handle = false + handle = LibC._get_osfhandle(fd) + if handle != -1 + if LibC.GetConsoleMode(LibC::HANDLE.new(handle), out _) != 0 + console_handle = true + end + end + + io = IO::FileDescriptor.new(fd) + # Set sync or flush_on_newline as described in STDOUT and STDERR docs. + # See https://crystal-lang.org/api/toplevel.html#STDERR + if console_handle + io.sync = true + else + io.flush_on_newline = true + end + io + end end diff --git a/src/io/file_descriptor.cr b/src/io/file_descriptor.cr index 64e30c155811..35754c9ec97b 100644 --- a/src/io/file_descriptor.cr +++ b/src/io/file_descriptor.cr @@ -32,27 +32,7 @@ class IO::FileDescriptor < IO # :nodoc: def self.from_stdio(fd) - {% if flag?(:win32) %} - new(fd) - {% else %} - # If we have a TTY for stdin/out/err, it is possibly a shared terminal. - # We need to reopen it to use O_NONBLOCK without causing other programs to break - - # Figure out the terminal TTY name. If ttyname fails we have a non-tty, or something strange. - # For non-tty we set flush_on_newline to true for reasons described in STDOUT and STDERR docs. - path = uninitialized UInt8[256] - ret = LibC.ttyname_r(fd, path, 256) - return new(fd).tap(&.flush_on_newline=(true)) unless ret == 0 - - clone_fd = LibC.open(path, LibC::O_RDWR) - return new(fd).tap(&.flush_on_newline=(true)) if clone_fd == -1 - - # We don't buffer output for TTY devices to see their output right away - io = new(clone_fd) - io.close_on_exec = true - io.sync = true - io - {% end %} + Crystal::System::FileDescriptor.from_stdio(fd) end def blocking diff --git a/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr new file mode 100644 index 000000000000..44cfeab11261 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/consoleapi.cr @@ -0,0 +1,5 @@ +require "c/winnt" + +lib LibC + fun GetConsoleMode(hConsoleHandle : HANDLE, lpMode : DWORD*) : BOOL +end From 807e219668c845a1359975f40bfaca3581e1d02d Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 8 May 2020 17:23:12 -0300 Subject: [PATCH 010/263] Fix version selector in API docs shows current version twice (#9250) --- src/compiler/crystal/tools/doc/html/js/_versions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/tools/doc/html/js/_versions.js b/src/compiler/crystal/tools/doc/html/js/_versions.js index bc09a98d6fb0..9d0a390c9028 100644 --- a/src/compiler/crystal/tools/doc/html/js/_versions.js +++ b/src/compiler/crystal/tools/doc/html/js/_versions.js @@ -30,7 +30,7 @@ CrystalDocs.loadConfig = function (config) { var currentVersion = document.querySelector("html > head > meta[name=\"crystal_docs.project_version\"]").getAttribute("content") var currentVersionInList = projectVersions.find(function (element) { - element.name == currentVersion + return element.name == currentVersion }) if (!currentVersionInList) { From 0eecc97dd982c59169fda172fea31f8e8cf5e9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 8 May 2020 22:24:24 +0200 Subject: [PATCH 011/263] [Docs] Fix line-height in version selector (#9252) --- src/compiler/crystal/tools/doc/html/css/style.css | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compiler/crystal/tools/doc/html/css/style.css b/src/compiler/crystal/tools/doc/html/css/style.css index 72506d4ab21a..eaa0059ccc27 100644 --- a/src/compiler/crystal/tools/doc/html/css/style.css +++ b/src/compiler/crystal/tools/doc/html/css/style.css @@ -146,6 +146,7 @@ body { color: inherit; font-family: inherit; font-size: inherit; + line-height: inherit; } .project-versions-nav:focus { outline: none; From 8705717eb1bb280744138f9462c47c230e06d782 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 8 May 2020 18:08:20 -0300 Subject: [PATCH 012/263] Update distributions-script (#9242) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index adfcd2f10aa2..c3067787c916 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - run: | git clone https://github.com/crystal-lang/distribution-scripts.git ~/distribution-scripts cd ~/distribution-scripts - git checkout cfe843ff938697174afea976f4499fef806b5286 + git checkout 1f2d5d937d431ff86355a276fc259478b3df73ac # persist relevant information for build process - run: | cd ~/distribution-scripts From 3dfc133ef4aade11f1fe8455d5c0f93bd0241a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 11 May 2020 14:54:03 +0200 Subject: [PATCH 013/263] Rename CLI argument to json-config-url (#9254) All other CLI arguments use kebab-case, this keeps it consistent --- man/crystal.1 | 2 +- src/compiler/crystal/command/docs.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/crystal.1 b/man/crystal.1 index 48d50894fbb7..5a69bb5b6807 100644 --- a/man/crystal.1 +++ b/man/crystal.1 @@ -180,7 +180,7 @@ In case no default can be found, this option is mandatory. .It Fl -project-version Ar VERSION Set the project version. The default value is extracted from current git commit or shard.yml if available. In case no default can be found, this option is mandatory. -.It Fl -json_config_url Ar URL +.It Fl -json-config-url Ar URL Set the URL pointing to a config file (used for discovering versions). .It Fl o Ar DIR, Fl -output Ar DIR Set the output directory (default: ./docs). diff --git a/src/compiler/crystal/command/docs.cr b/src/compiler/crystal/command/docs.cr index 24dc922e7a13..c0b49be1cf69 100644 --- a/src/compiler/crystal/command/docs.cr +++ b/src/compiler/crystal/command/docs.cr @@ -44,7 +44,7 @@ class Crystal::Command output_format = value end - opts.on("--json_config_url=URL", "Set the URL pointing to a config file (used for discovering versions)") do |value| + opts.on("--json-config-url=URL", "Set the URL pointing to a config file (used for discovering versions)") do |value| project_info.json_config_url = value end From c1cd2005491c5d4f9a97d5d2f061ab0c88ba15f0 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 11 May 2020 10:23:30 -0300 Subject: [PATCH 014/263] Cleanup File & FileUtils (#9175) * Add File.copy Only FileUtils.cp treats the dest as a potential directory * Add File/IO.same_content?. Refactor FileUtils.cmp Drop misplaced FileUtils.cmp(IO, IO) * Deprecate Dir.rmdir in favor of Dir.delete Allow Dir.path with Path argument Co-Authored-By: Julien Reichardt * Avoid looking up the path twice * Ensure permissions are overwritten if copy destination exists In Ruby a preserve argument can be used to mimic cp -p The default in cp and in Ruby is to not preserve the existing permissions of the copy destination. * Ignore copies permissions specs on windows * Fix FileUtils.cp_r with dest_path exists Fixes #9170 Co-authored-by: Julien Reichardt --- spec/std/dir_spec.cr | 21 +++++++---- spec/std/file_spec.cr | 56 ++++++++++++++++++++++++++++- spec/std/file_utils_spec.cr | 62 +++++++++----------------------- spec/std/io/io_spec.cr | 47 ++++++++++++++++++++++++ src/compiler/crystal/compiler.cr | 2 +- src/dir.cr | 10 ++++-- src/file.cr | 38 ++++++++++++++++++++ src/file_utils.cr | 51 +++++--------------------- src/io.cr | 23 ++++++++++++ 9 files changed, 213 insertions(+), 97 deletions(-) diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 5a0394d0599a..4bf7067f527f 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -75,6 +75,15 @@ describe "Dir" do end end + it "tests mkdir and delete with a new path" do + with_tempfile("mkdir") do |path| + Dir.mkdir(path, 0o700) + Dir.exists?(path).should be_true + Dir.delete(path) + Dir.exists?(path).should be_false + end + end + it "tests mkdir and rmdir with a new path" do with_tempfile("mkdir") do |path| Dir.mkdir(path, 0o700) @@ -116,17 +125,17 @@ describe "Dir" do end end - it "tests rmdir with an nonexistent path" do + it "tests delete with an nonexistent path" do with_tempfile("nonexistant") do |path| expect_raises(File::NotFoundError, "Unable to remove directory: '#{path.inspect_unquoted}'") do - Dir.rmdir(path) + Dir.delete(path) end end end - it "tests rmdir with a path that cannot be removed" do + it "tests delete with a path that cannot be removed" do expect_raises(File::Error, "Unable to remove directory: '#{datapath.inspect_unquoted}'") do - Dir.rmdir(datapath) + Dir.delete(datapath) end end @@ -536,8 +545,8 @@ describe "Dir" do Dir.mkdir_p("foo\0bar") end - it_raises_on_null_byte "rmdir" do - Dir.rmdir("foo\0bar") + it_raises_on_null_byte "delete" do + Dir.delete("foo\0bar") end end end diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 812891c66248..456121c15fed 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -359,7 +359,7 @@ describe "File" do File.chmod(path, 0o664) File.info(path).permissions.should eq(normalize_permissions(0o664, directory: true)) ensure - Dir.rmdir(path) if Dir.exists?(path) + Dir.delete(path) if Dir.exists?(path) end end @@ -1224,6 +1224,60 @@ describe "File" do end end + describe ".same_content?" do + it "compares two equal files" do + File.same_content?( + datapath("test_file.txt"), + datapath("test_file.txt") + ).should be_true + end + + it "compares two different files" do + File.same_content?( + datapath("test_file.txt"), + datapath("test_file.ini") + ).should be_false + end + end + + describe ".copy" do + it "copies a file" do + src_path = datapath("test_file.txt") + with_tempfile("cp.txt") do |out_path| + File.copy(src_path, out_path) + File.exists?(out_path).should be_true + File.same_content?(src_path, out_path).should be_true + end + end + + pending_win32 "copies permissions" do + with_tempfile("cp-permissions-src.txt", "cp-permissions-out.txt") do |src_path, out_path| + File.write(src_path, "foo") + File.chmod(src_path, 0o700) + + File.copy(src_path, out_path) + + File.info(out_path).permissions.should eq(File::Permissions.new(0o700)) + File.same_content?(src_path, out_path).should be_true + end + end + + pending_win32 "overwrites existing destination and permissions" do + with_tempfile("cp-permissions-src.txt", "cp-permissions-out.txt") do |src_path, out_path| + File.write(src_path, "foo") + File.chmod(src_path, 0o700) + + File.write(out_path, "bar") + File.chmod(out_path, 0o777) + + File.copy(src_path, out_path) + + File.info(out_path).permissions.should eq(File::Permissions.new(0o700)) + File.same_content?(src_path, out_path).should be_true + end + end + end + describe ".match?" do it "matches basics" do File.match?("abc", "abc").should be_true diff --git a/spec/std/file_utils_spec.cr b/spec/std/file_utils_spec.cr index e94444f71691..f3c2110bb937 100644 --- a/spec/std/file_utils_spec.cr +++ b/spec/std/file_utils_spec.cr @@ -1,27 +1,6 @@ require "./spec_helper" require "file_utils" -private class OneByOneIO < IO - @bytes : Bytes - - def initialize(string) - @bytes = string.to_slice - @pos = 0 - end - - def read(slice : Bytes) - return 0 if slice.empty? - return 0 if @pos >= @bytes.size - - slice[0] = @bytes[@pos] - @pos += 1 - 1 - end - - def write(slice : Bytes) : Nil - end -end - describe "FileUtils" do describe "cd" do it "should work" do @@ -69,30 +48,6 @@ describe "FileUtils" do datapath("test_file.ini") ).should be_false end - - it "compares two ios, one way (true)" do - io1 = OneByOneIO.new("hello") - io2 = IO::Memory.new("hello") - FileUtils.cmp(io1, io2).should be_true - end - - it "compares two ios, second way (true)" do - io1 = OneByOneIO.new("hello") - io2 = IO::Memory.new("hello") - FileUtils.cmp(io2, io1).should be_true - end - - it "compares two ios, one way (false)" do - io1 = OneByOneIO.new("hello") - io2 = IO::Memory.new("hella") - FileUtils.cmp(io1, io2).should be_false - end - - it "compares two ios, second way (false)" do - io1 = OneByOneIO.new("hello") - io2 = IO::Memory.new("hella") - FileUtils.cmp(io2, io1).should be_false - end end describe "touch" do @@ -170,6 +125,23 @@ describe "FileUtils" do File.exists?(File.join(dest_path, "b/c")).should be_true end end + + it "copies a directory recursively if destination exists leaving existing files" do + with_tempfile("cp_r-test", "cp_r-test-copied") do |src_path, dest_path| + Dir.mkdir_p(dest_path) + File.write(File.join(dest_path, "d"), "") + + Dir.mkdir_p(src_path) + File.write(File.join(src_path, "a"), "") + Dir.mkdir(File.join(src_path, "b")) + File.write(File.join(src_path, "b/c"), "") + + FileUtils.cp_r(src_path, dest_path) + File.exists?(File.join(dest_path, "a")).should be_true + File.exists?(File.join(dest_path, "b/c")).should be_true + File.exists?(File.join(dest_path, "d")).should be_true + end + end end describe "rm_r" do diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index f5cde6ed1f51..b49e6a0fd992 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -82,6 +82,27 @@ private class SimpleIOMemory < IO end end +private class OneByOneIO < IO + @bytes : Bytes + + def initialize(string) + @bytes = string.to_slice + @pos = 0 + end + + def read(slice : Bytes) + return 0 if slice.empty? + return 0 if @pos >= @bytes.size + + slice[0] = @bytes[@pos] + @pos += 1 + 1 + end + + def write(slice : Bytes) : Nil + end +end + describe IO do describe "partial read" do pending_win32 "doesn't block on first read. blocks on 2nd read" do @@ -398,6 +419,32 @@ describe IO do end end end + + describe ".same_content?" do + it "compares two ios, one way (true)" do + io1 = OneByOneIO.new("hello") + io2 = IO::Memory.new("hello") + IO.same_content?(io1, io2).should be_true + end + + it "compares two ios, second way (true)" do + io1 = OneByOneIO.new("hello") + io2 = IO::Memory.new("hello") + IO.same_content?(io2, io1).should be_true + end + + it "compares two ios, one way (false)" do + io1 = OneByOneIO.new("hello") + io2 = IO::Memory.new("hella") + IO.same_content?(io1, io2).should be_false + end + + it "compares two ios, second way (false)" do + io1 = OneByOneIO.new("hello") + io2 = IO::Memory.new("hella") + IO.same_content?(io2, io1).should be_false + end + end end describe "write operations" do diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index febd73636a7c..c1336f42e322 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -675,7 +675,7 @@ module Crystal if can_reuse_previous_compilation memory_io = IO::Memory.new(memory_buffer.to_slice) - changed = File.open(bc_name) { |bc_file| !FileUtils.cmp(bc_file, memory_io) } + changed = File.open(bc_name) { |bc_file| !IO.same_content?(bc_file, memory_io) } # If the user cancelled a previous compilation # it might be that the .o file is empty diff --git a/src/dir.cr b/src/dir.cr index 60934bf658b5..cac0382f88f9 100644 --- a/src/dir.cr +++ b/src/dir.cr @@ -249,8 +249,14 @@ class Dir end # Removes the directory at the given path. - def self.rmdir(path : Path | String) : Nil - Crystal::System::Dir.delete(path) + @[Deprecated("Use `Dir.delete` instead")] + def self.rmdir(path : Path | String) + delete(path) + end + + # Removes the directory at the given path. + def self.delete(path : Path | String) + Crystal::System::Dir.delete(path.to_s) end def to_s(io : IO) : Nil diff --git a/src/file.cr b/src/file.cr index 095da994e7f5..879cac96c5aa 100644 --- a/src/file.cr +++ b/src/file.cr @@ -168,6 +168,24 @@ class File < IO::FileDescriptor info(path1.to_s, follow_symlinks).same_file? info(path2.to_s, follow_symlinks) end + # Compares two files *filename1* to *filename2* to determine if they are identical. + # Returns `true` if content are the same, `false` otherwise. + # + # ``` + # File.write("file.cr", "1") + # File.write("bar.cr", "1") + # File.same_content?("file.cr", "bar.cr") # => true + # ``` + def self.same_content?(path1 : Path | String, path2 : Path | String) : Bool + open(path1, "rb") do |file1| + open(path2, "rb") do |file2| + return false unless file1.size == file2.size + + same_content?(file1, file2) + end + end + end + # Returns the size of the file at *filename* in bytes. # Raises `File::NotFoundError` if the file at *filename* does not exist. # @@ -701,6 +719,26 @@ class File < IO::FileDescriptor end end + # Copies the file *src* to the file *dst*. + # Permission bits are copied too. + # + # ``` + # File.chmod("afile", 0o600) + # File.copy("afile", "afile_copy") + # File.info("afile_copy").permissions.value # => 0o600 + # ``` + def self.copy(src : String | Path, dst : String | Path) + open(src) do |s| + open(dst, "wb") do |d| + # TODO use sendfile or copy_file_range syscall. See #8926, #8919 + IO.copy(s, d) + end + + # Set the permissions after the content is written in case src permissions is read-only + chmod(dst, s.info.permissions) + end + end + # Returns a new string formed by joining the strings using `File::SEPARATOR`. # # ``` diff --git a/src/file_utils.cr b/src/file_utils.cr index 5885ef0fd368..41fde0695edd 100644 --- a/src/file_utils.cr +++ b/src/file_utils.cr @@ -38,39 +38,10 @@ module FileUtils # File.write("bar.cr", "1") # FileUtils.cmp("file.cr", "bar.cr") # => true # ``` - def cmp(filename1 : String, filename2 : String) - return false unless File.size(filename1) == File.size(filename2) - - File.open(filename1, "rb") do |file1| - File.open(filename2, "rb") do |file2| - cmp(file1, file2) - end - end - end - - # Compares two streams *stream1* to *stream2* to determine if they are identical. - # Returns `true` if content are the same, `false` otherwise. # - # ``` - # require "file_utils" - # - # File.write("afile", "123") - # stream1 = File.open("afile") - # stream2 = IO::Memory.new("123") - # FileUtils.cmp(stream1, stream2) # => true - # ``` - def cmp(stream1 : IO, stream2 : IO) - buf1 = uninitialized UInt8[1024] - buf2 = uninitialized UInt8[1024] - - while true - read1 = stream1.read(buf1.to_slice) - read2 = stream2.read_fully?(buf2.to_slice[0, read1]) - return false unless read2 - - return false if buf1.to_unsafe.memcmp(buf2.to_unsafe, read1) != 0 - return true if read1 == 0 - end + # NOTE: Alias of `File.same_content?` + def cmp(filename1 : String, filename2 : String) + File.same_content?(filename1, filename2) end # Attempts to set the access and modification times of the file named @@ -117,12 +88,8 @@ module FileUtils # File.info("afile_copy").permissions.value # => 0o600 # ``` def cp(src_path : String, dest : String) - File.open(src_path) do |s| - dest += File::SEPARATOR + File.basename(src_path) if Dir.exists?(dest) - File.open(dest, "wb", s.info.permissions) do |d| - IO.copy(s, d) - end - end + dest += File::SEPARATOR + File.basename(src_path) if Dir.exists?(dest) + File.copy(src_path, dest) end # Copies a list of files *src* to *dest*. @@ -151,7 +118,7 @@ module FileUtils # ``` def cp_r(src_path : String, dest_path : String) if Dir.exists?(src_path) - Dir.mkdir(dest_path) + Dir.mkdir(dest_path) unless Dir.exists?(dest_path) Dir.each_child(src_path) do |entry| src = File.join(src_path, entry) dest = File.join(dest_path, entry) @@ -414,7 +381,7 @@ module FileUtils src = File.join(path, entry) rm_r(src) end - Dir.rmdir(path) + Dir.delete(path) else File.delete(path) end @@ -480,7 +447,7 @@ module FileUtils # # NOTE: Alias of `Dir.rmdir` def rmdir(path : String) : Nil - Dir.rmdir(path) + Dir.delete(path) end # Removes all directories at the given *paths*. @@ -492,7 +459,7 @@ module FileUtils # ``` def rmdir(paths : Enumerable(String)) : Nil paths.each do |path| - Dir.rmdir(path) + Dir.delete(path) end end end diff --git a/src/io.cr b/src/io.cr index 710282f8d6ca..a334bfc187da 100644 --- a/src/io.cr +++ b/src/io.cr @@ -1148,6 +1148,29 @@ abstract class IO limit - remaining end + # Compares two streams *stream1* to *stream2* to determine if they are identical. + # Returns `true` if content are the same, `false` otherwise. + # + # ``` + # File.write("afile", "123") + # stream1 = File.open("afile") + # stream2 = IO::Memory.new("123") + # IO.same_content?(stream1, stream2) # => true + # ``` + def self.same_content?(stream1 : IO, stream2 : IO) + buf1 = uninitialized UInt8[1024] + buf2 = uninitialized UInt8[1024] + + while true + read1 = stream1.read(buf1.to_slice) + read2 = stream2.read_fully?(buf2.to_slice[0, read1]) + return false unless read2 + + return false if buf1.to_unsafe.memcmp(buf2.to_unsafe, read1) != 0 + return true if read1 == 0 + end + end + private struct LineIterator(I, A, N) include Iterator(String) From 124bf678c6074b9681597b4755497af791f1e7b1 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 11 May 2020 10:32:22 -0300 Subject: [PATCH 015/263] Let `case when` be non-exhaustive, introduce `case in` as exhaustive (#9258) * Let `case when` be non-exhaustive, introduce `case in` as exhaustive * Replace `in` occurrences with a different because now it's a keyword * Flags enums can be checked exhaustively by matching with a type * Add missing argument to Case constructor * Preserve `When#to_s` working well * Let When also know whether it's exhaustive or not (when or in) --- spec/compiler/formatter/formatter_spec.cr | 2 + spec/compiler/macro/macro_methods_spec.cr | 38 ++- spec/compiler/normalize/case_spec.cr | 54 ++-- spec/compiler/parser/parser_spec.cr | 93 ++++--- spec/compiler/parser/to_s_spec.cr | 1 + spec/compiler/semantic/case_spec.cr | 243 +++++++++++------- .../crystal/semantic/cleanup_transformer.cr | 7 +- .../semantic/exhaustiveness_checker.cr | 130 +++++----- .../crystal/semantic/literal_expander.cr | 12 +- src/compiler/crystal/syntax/ast.cr | 17 +- src/compiler/crystal/syntax/parser.cr | 65 ++++- src/compiler/crystal/syntax/to_s.cr | 4 +- src/compiler/crystal/tools/formatter.cr | 2 +- src/digest/md5.cr | 28 +- src/time.cr | 2 +- 15 files changed, 422 insertions(+), 276 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index b37f01baac98..16f943dd3a14 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -542,6 +542,8 @@ describe Crystal::Formatter do assert_format "case\nend" assert_format "case\nelse\n 1\nend" + assert_format "case 1 \n in Int32 \n 3 \n end", "case 1\nin Int32\n 3\nend" + assert_format <<-CODE case 0 when 0 then 1; 2 diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index 7579a8596205..c7341f877b2a 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1990,26 +1990,36 @@ module Crystal end describe "case methods" do - case_node = Case.new(1.int32, [When.new([2.int32, 3.int32] of ASTNode, 4.int32)], 5.int32) + describe "when" do + case_node = Case.new(1.int32, [When.new([2.int32, 3.int32] of ASTNode, 4.int32)], 5.int32, exhaustive: false) - it "executes cond" do - assert_macro "x", %({{x.cond}}), [case_node] of ASTNode, "1" - end + it "executes cond" do + assert_macro "x", %({{x.cond}}), [case_node] of ASTNode, "1" + end - it "executes whens" do - assert_macro "x", %({{x.whens}}), [case_node] of ASTNode, "[when 2, 3\n 4\n]" - end + it "executes whens" do + assert_macro "x", %({{x.whens}}), [case_node] of ASTNode, "[when 2, 3\n 4\n]" + end - it "executes when conds" do - assert_macro "x", %({{x.whens[0].conds}}), [case_node] of ASTNode, "[2, 3]" - end + it "executes when conds" do + assert_macro "x", %({{x.whens[0].conds}}), [case_node] of ASTNode, "[2, 3]" + end - it "executes when body" do - assert_macro "x", %({{x.whens[0].body}}), [case_node] of ASTNode, "4" + it "executes when body" do + assert_macro "x", %({{x.whens[0].body}}), [case_node] of ASTNode, "4" + end + + it "executes else" do + assert_macro "x", %({{x.else}}), [case_node] of ASTNode, "5" + end end - it "executes else" do - assert_macro "x", %({{x.else}}), [case_node] of ASTNode, "5" + describe "in" do + case_node = Case.new(1.int32, [When.new([2.int32, 3.int32] of ASTNode, 4.int32)], 5.int32, exhaustive: true) + + it "executes whens" do + assert_macro "x", %({{x.whens}}), [case_node] of ASTNode, "[in 2, 3\n 4\n]" + end end end diff --git a/spec/compiler/normalize/case_spec.cr b/spec/compiler/normalize/case_spec.cr index 6d56aeefbdcd..5ad7da9a61b0 100644 --- a/spec/compiler/normalize/case_spec.cr +++ b/spec/compiler/normalize/case_spec.cr @@ -6,103 +6,103 @@ describe "Normalize: case" do end it "normalizes case with var in cond" do - assert_expand_second "x = 1; case x; when 1; 'b'; else; end", "if 1 === x\n 'b'\nend" + assert_expand_second "x = 1; case x; when 1; 'b'; end", "if 1 === x\n 'b'\nend" end it "normalizes case with Path to is_a?" do - assert_expand_second "x = 1; case x; when Foo; 'b'; else; end", "if x.is_a?(Foo)\n 'b'\nend" + assert_expand_second "x = 1; case x; when Foo; 'b'; end", "if x.is_a?(Foo)\n 'b'\nend" end it "normalizes case with generic to is_a?" do - assert_expand_second "x = 1; case x; when Foo(T); 'b'; else; end", "if x.is_a?(Foo(T))\n 'b'\nend" + assert_expand_second "x = 1; case x; when Foo(T); 'b'; end", "if x.is_a?(Foo(T))\n 'b'\nend" end it "normalizes case with Path.class to is_a?" do - assert_expand_second "x = 1; case x; when Foo.class; 'b'; else; end", "if x.is_a?(Foo.class)\n 'b'\nend" + assert_expand_second "x = 1; case x; when Foo.class; 'b'; end", "if x.is_a?(Foo.class)\n 'b'\nend" end it "normalizes case with Generic.class to is_a?" do - assert_expand_second "x = 1; case x; when Foo(T).class; 'b'; else; end", "if x.is_a?(Foo(T).class)\n 'b'\nend" + assert_expand_second "x = 1; case x; when Foo(T).class; 'b'; end", "if x.is_a?(Foo(T).class)\n 'b'\nend" end it "normalizes case with many expressions in when" do - assert_expand_second "x = 1; case x; when 1, 2; 'b'; else; end", "if (1 === x) || (2 === x)\n 'b'\nend" + assert_expand_second "x = 1; case x; when 1, 2; 'b'; end", "if (1 === x) || (2 === x)\n 'b'\nend" end it "normalizes case with implicit call" do - assert_expand "case x; when .foo(1); 2; else; end", "__temp_1 = x\nif __temp_1.foo(1)\n 2\nend" + assert_expand "case x; when .foo(1); 2; end", "__temp_1 = x\nif __temp_1.foo(1)\n 2\nend" end it "normalizes case with implicit responds_to? (#3040)" do - assert_expand "case x; when .responds_to?(:foo); 2; else; end", "__temp_1 = x\nif __temp_1.responds_to?(:foo)\n 2\nend" + assert_expand "case x; when .responds_to?(:foo); 2; end", "__temp_1 = x\nif __temp_1.responds_to?(:foo)\n 2\nend" end it "normalizes case with implicit is_a? (#3040)" do - assert_expand "case x; when .is_a?(T); 2; else; end", "__temp_1 = x\nif __temp_1.is_a?(T)\n 2\nend" + assert_expand "case x; when .is_a?(T); 2; end", "__temp_1 = x\nif __temp_1.is_a?(T)\n 2\nend" end it "normalizes case with implicit as (#3040)" do - assert_expand "case x; when .as(T); 2; else; end", "__temp_1 = x\nif __temp_1.as(T)\n 2\nend" + assert_expand "case x; when .as(T); 2; end", "__temp_1 = x\nif __temp_1.as(T)\n 2\nend" end it "normalizes case with implicit as? (#3040)" do - assert_expand "case x; when .as?(T); 2; else; end", "__temp_1 = x\nif __temp_1.as?(T)\n 2\nend" + assert_expand "case x; when .as?(T); 2; end", "__temp_1 = x\nif __temp_1.as?(T)\n 2\nend" end it "normalizes case with implicit !" do - assert_expand "case x; when .!; 2; else; end", "__temp_1 = x\nif !__temp_1\n 2\nend" + assert_expand "case x; when .!; 2; end", "__temp_1 = x\nif !__temp_1\n 2\nend" end it "normalizes case with assignment" do - assert_expand "case x = 1; when 2; 3; else; end", "x = 1\nif 2 === x\n 3\nend" + assert_expand "case x = 1; when 2; 3; end", "x = 1\nif 2 === x\n 3\nend" end it "normalizes case with assignment wrapped by paren" do - assert_expand "case (x = 1); when 2; 3; else; end", "x = 1\nif 2 === x\n 3\nend" + assert_expand "case (x = 1); when 2; 3; end", "x = 1\nif 2 === x\n 3\nend" end it "normalizes case without value" do - assert_expand "case when 2; 3; when 4; 5; else; end", "if 2\n 3\nelse\n if 4\n 5\n end\nend" + assert_expand "case when 2; 3; when 4; 5; end", "if 2\n 3\nelse\n if 4\n 5\n end\nend" end it "normalizes case without value with many expressions in when" do - assert_expand "case when 2, 9; 3; when 4; 5; else; end", "if 2 || 9\n 3\nelse\n if 4\n 5\n end\nend" + assert_expand "case when 2, 9; 3; when 4; 5; end", "if 2 || 9\n 3\nelse\n if 4\n 5\n end\nend" end it "normalizes case with nil to is_a?" do - assert_expand_second "x = 1; case x; when nil; 'b'; else; end", "if x.is_a?(::Nil)\n 'b'\nend" + assert_expand_second "x = 1; case x; when nil; 'b'; end", "if x.is_a?(::Nil)\n 'b'\nend" end it "normalizes case with multiple expressions" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {2, 3}; 4; else; end", "if (2 === x) && (3 === y)\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {2, 3}; 4; end", "if (2 === x) && (3 === y)\n 4\nend" end it "normalizes case with multiple expressions and types" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {Int32, Float64}; 4; else; end", "if x.is_a?(Int32) && y.is_a?(Float64)\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {Int32, Float64}; 4; end", "if x.is_a?(Int32) && y.is_a?(Float64)\n 4\nend" end it "normalizes case with multiple expressions and implicit obj" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {.foo, .bar}; 4; else; end", "if x.foo && y.bar\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {.foo, .bar}; 4; end", "if x.foo && y.bar\n 4\nend" end it "normalizes case with multiple expressions and comma" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {2, 3}, {4, 5}; 6; else; end", "if ((2 === x) && (3 === y)) || ((4 === x) && (5 === y))\n 6\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {2, 3}, {4, 5}; 6; end", "if ((2 === x) && (3 === y)) || ((4 === x) && (5 === y))\n 6\nend" end it "normalizes case with multiple expressions with underscore" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {2, _}; 4; else; end", "if 2 === x\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {2, _}; 4; end", "if 2 === x\n 4\nend" end it "normalizes case with multiple expressions with all underscores" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {_, _}; 4; else; end", "if true\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {_, _}; 4; end", "if true\n 4\nend" end it "normalizes case with multiple expressions with all underscores twice" do - assert_expand_second "x, y = 1, 2; case {x, y}; when {_, _}, {_, _}; 4; else; end", "if true\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when {_, _}, {_, _}; 4; end", "if true\n 4\nend" end it "normalizes case with multiple expressions and non-tuple" do - assert_expand_second "x, y = 1, 2; case {x, y}; when 1; 4; else; end", "if 1 === {x, y}\n 4\nend" + assert_expand_second "x, y = 1, 2; case {x, y}; when 1; 4; end", "if 1 === {x, y}\n 4\nend" end it "normalizes case without when and else" do @@ -120,4 +120,8 @@ describe "Normalize: case" do it "normalizes case without cond, when but else" do assert_expand "case; else; y; end", "y" end + + it "normalizes case with Path.class to is_a? (in)" do + assert_expand_second "x = 1; case x; in Foo.class; 'b'; end", "if x.is_a?(Foo.class)\n 'b'\nelse\n raise \"unreachable\"\nend" + end end diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index e3123f5e3947..73e0a648b763 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1068,41 +1068,64 @@ module Crystal it_parses "require \"foo\"", Require.new("foo") it_parses "require \"foo\"; [1]", [Require.new("foo"), ([1.int32] of ASTNode).array] - it_parses "case 1; when 1; 2; else; 3; end", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32) - it_parses "case 1; when 0, 1; 2; else; 3; end", Case.new(1.int32, [When.new([0.int32, 1.int32] of ASTNode, 2.int32)], 3.int32) - it_parses "case 1\nwhen 1\n2\nelse\n3\nend", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32) - it_parses "case 1\nwhen 1\n2\nend", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)]) - it_parses "case / /; when / /; / /; else; / /; end", Case.new(regex(" "), [When.new([regex(" ")] of ASTNode, regex(" "))], regex(" ")) - it_parses "case / /\nwhen / /\n/ /\nelse\n/ /\nend", Case.new(regex(" "), [When.new([regex(" ")] of ASTNode, regex(" "))], regex(" ")) - - it_parses "case 1; when 1 then 2; else; 3; end", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32) - it_parses "case 1; when x then 2; else; 3; end", Case.new(1.int32, [When.new(["x".call] of ASTNode, 2.int32)], 3.int32) - it_parses "case 1\nwhen 1\n2\nend\nif a\nend", [Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)]), If.new("a".call)] - it_parses "case\n1\nwhen 1\n2\nend\nif a\nend", [Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)]), If.new("a".call)] - - it_parses "case 1\nwhen .foo\n2\nend", Case.new(1.int32, [When.new([Call.new(ImplicitObj.new, "foo")] of ASTNode, 2.int32)]) - it_parses "case 1\nwhen .responds_to?(:foo)\n2\nend", Case.new(1.int32, [When.new([RespondsTo.new(ImplicitObj.new, "foo")] of ASTNode, 2.int32)]) - it_parses "case 1\nwhen .is_a?(T)\n2\nend", Case.new(1.int32, [When.new([IsA.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)]) - it_parses "case 1\nwhen .as(T)\n2\nend", Case.new(1.int32, [When.new([Cast.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)]) - it_parses "case 1\nwhen .as?(T)\n2\nend", Case.new(1.int32, [When.new([NilableCast.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)]) - it_parses "case 1\nwhen .!()\n2\nend", Case.new(1.int32, [When.new([Not.new(ImplicitObj.new)] of ASTNode, 2.int32)]) - it_parses "case when 1\n2\nend", Case.new(nil, [When.new([1.int32] of ASTNode, 2.int32)]) - it_parses "case \nwhen 1\n2\nend", Case.new(nil, [When.new([1.int32] of ASTNode, 2.int32)]) - it_parses "case {1, 2}\nwhen {3, 4}\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([3.int32, 4.int32] of ASTNode)] of ASTNode, 5.int32)]) - it_parses "case {1, 2}\nwhen {3, 4}, {5, 6}\n7\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([3.int32, 4.int32] of ASTNode), TupleLiteral.new([5.int32, 6.int32] of ASTNode)] of ASTNode, 7.int32)]) - it_parses "case {1, 2}\nwhen {.foo, .bar}\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([Call.new(ImplicitObj.new, "foo"), Call.new(ImplicitObj.new, "bar")] of ASTNode)] of ASTNode, 5.int32)]) - it_parses "case {1, 2}\nwhen foo\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new(["foo".call] of ASTNode, 5.int32)]) - it_parses "case a\nwhen b\n1 / 2\nelse\n1 / 2\nend", Case.new("a".call, [When.new(["b".call] of ASTNode, Call.new(1.int32, "/", 2.int32))], Call.new(1.int32, "/", 2.int32)) - it_parses "case a\nwhen b\n/ /\n\nelse\n/ /\nend", Case.new("a".call, [When.new(["b".call] of ASTNode, RegexLiteral.new(StringLiteral.new(" ")))], RegexLiteral.new(StringLiteral.new(" "))) + it_parses "case 1; when 1; 2; else; 3; end", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32, exhaustive: false) + it_parses "case 1; when 0, 1; 2; else; 3; end", Case.new(1.int32, [When.new([0.int32, 1.int32] of ASTNode, 2.int32)], 3.int32, exhaustive: false) + it_parses "case 1\nwhen 1\n2\nelse\n3\nend", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32, exhaustive: false) + it_parses "case 1\nwhen 1\n2\nend", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case / /; when / /; / /; else; / /; end", Case.new(regex(" "), [When.new([regex(" ")] of ASTNode, regex(" "))], regex(" "), exhaustive: false) + it_parses "case / /\nwhen / /\n/ /\nelse\n/ /\nend", Case.new(regex(" "), [When.new([regex(" ")] of ASTNode, regex(" "))], regex(" "), exhaustive: false) + + it_parses "case 1; when 1 then 2; else; 3; end", Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], 3.int32, exhaustive: false) + it_parses "case 1; when x then 2; else; 3; end", Case.new(1.int32, [When.new(["x".call] of ASTNode, 2.int32)], 3.int32, exhaustive: false) + it_parses "case 1\nwhen 1\n2\nend\nif a\nend", [Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], else: nil, exhaustive: false), If.new("a".call)] + it_parses "case\n1\nwhen 1\n2\nend\nif a\nend", [Case.new(1.int32, [When.new([1.int32] of ASTNode, 2.int32)], else: nil, exhaustive: false), If.new("a".call)] + + it_parses "case 1\nwhen .foo\n2\nend", Case.new(1.int32, [When.new([Call.new(ImplicitObj.new, "foo")] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case 1\nwhen .responds_to?(:foo)\n2\nend", Case.new(1.int32, [When.new([RespondsTo.new(ImplicitObj.new, "foo")] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case 1\nwhen .is_a?(T)\n2\nend", Case.new(1.int32, [When.new([IsA.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case 1\nwhen .as(T)\n2\nend", Case.new(1.int32, [When.new([Cast.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case 1\nwhen .as?(T)\n2\nend", Case.new(1.int32, [When.new([NilableCast.new(ImplicitObj.new, "T".path)] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case 1\nwhen .!()\n2\nend", Case.new(1.int32, [When.new([Not.new(ImplicitObj.new)] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case when 1\n2\nend", Case.new(nil, [When.new([1.int32] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case \nwhen 1\n2\nend", Case.new(nil, [When.new([1.int32] of ASTNode, 2.int32)], else: nil, exhaustive: false) + it_parses "case {1, 2}\nwhen {3, 4}\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([3.int32, 4.int32] of ASTNode)] of ASTNode, 5.int32)], else: nil, exhaustive: false) + it_parses "case {1, 2}\nwhen {3, 4}, {5, 6}\n7\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([3.int32, 4.int32] of ASTNode), TupleLiteral.new([5.int32, 6.int32] of ASTNode)] of ASTNode, 7.int32)], else: nil, exhaustive: false) + it_parses "case {1, 2}\nwhen {.foo, .bar}\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new([TupleLiteral.new([Call.new(ImplicitObj.new, "foo"), Call.new(ImplicitObj.new, "bar")] of ASTNode)] of ASTNode, 5.int32)], else: nil, exhaustive: false) + it_parses "case {1, 2}\nwhen foo\n5\nend", Case.new(TupleLiteral.new([1.int32, 2.int32] of ASTNode), [When.new(["foo".call] of ASTNode, 5.int32)], else: nil, exhaustive: false) + it_parses "case a\nwhen b\n1 / 2\nelse\n1 / 2\nend", Case.new("a".call, [When.new(["b".call] of ASTNode, Call.new(1.int32, "/", 2.int32))], Call.new(1.int32, "/", 2.int32), exhaustive: false) + it_parses "case a\nwhen b\n/ /\n\nelse\n/ /\nend", Case.new("a".call, [When.new(["b".call] of ASTNode, RegexLiteral.new(StringLiteral.new(" ")))], RegexLiteral.new(StringLiteral.new(" ")), exhaustive: false) assert_syntax_error "case {1, 2}; when {3}; 4; end", "wrong number of tuple elements (given 1, expected 2)", 1, 19 - it_parses "case 1; end", [Case.new(1.int32, [] of When)] - it_parses "case foo; end", [Case.new("foo".call, [] of When)] - it_parses "case\nend", [Case.new(nil, [] of When)] - it_parses "case;end", [Case.new(nil, [] of When)] - it_parses "case 1\nelse\n2\nend", [Case.new(1.int32, [] of When, 2.int32)] - it_parses "a = 1\ncase 1\nwhen a then 1\nend", [Assign.new("a".var, 1.int32), Case.new(1.int32, [When.new(["a".var] of ASTNode, 1.int32)])] of ASTNode - it_parses "case\nwhen true\n1\nend", [Case.new(nil, [When.new([true.bool] of ASTNode, 1.int32)] of When)] - it_parses "case;when true;1;end", [Case.new(nil, [When.new([true.bool] of ASTNode, 1.int32)] of When)] + it_parses "case 1; end", Case.new(1.int32, [] of When, else: nil, exhaustive: false) + it_parses "case foo; end", Case.new("foo".call, [] of When, else: nil, exhaustive: false) + it_parses "case\nend", Case.new(nil, [] of When, else: nil, exhaustive: false) + it_parses "case;end", Case.new(nil, [] of When, else: nil, exhaustive: false) + it_parses "case 1\nelse\n2\nend", Case.new(1.int32, [] of When, 2.int32, exhaustive: false) + it_parses "a = 1\ncase 1\nwhen a then 1\nend", [Assign.new("a".var, 1.int32), Case.new(1.int32, [When.new(["a".var] of ASTNode, 1.int32)], else: nil, exhaustive: false)] of ASTNode + it_parses "case\nwhen true\n1\nend", Case.new(nil, [When.new([true.bool] of ASTNode, 1.int32)] of When, else: nil, exhaustive: false) + it_parses "case;when true;1;end", Case.new(nil, [When.new([true.bool] of ASTNode, 1.int32)] of When, else: nil, exhaustive: false) + + it_parses "case 1\nin Int32; 2; end", Case.new(1.int32, [When.new(["Int32".path] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin Int32.class; 2; end", Case.new(1.int32, [When.new([Call.new("Int32".path, "class")] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin Foo(Int32); 2; end", Case.new(1.int32, [When.new([Generic.new("Foo".path, ["Int32".path] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin false; 2; end", Case.new(1.int32, [When.new([false.bool] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin true; 2; end", Case.new(1.int32, [When.new([true.bool] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin nil; 2; end", Case.new(1.int32, [When.new([NilLiteral.new] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case 1\nin .bar?; 2; end", Case.new(1.int32, [When.new([Call.new(ImplicitObj.new, "bar?")] of ASTNode, 2.int32)], else: nil, exhaustive: true) + + it_parses "case {1}\nin {Int32}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new(["Int32".path] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {Int32.class}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([Call.new("Int32".path, "class")] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {Foo(Int32)}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([Generic.new("Foo".path, ["Int32".path] of ASTNode)] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {false}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([false.bool] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {true}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([true.bool] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {nil}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([NilLiteral.new] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {.bar?}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([Call.new(ImplicitObj.new, "bar?")] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + it_parses "case {1}\nin {_}; 2; end", Case.new(TupleLiteral.new([1.int32] of ASTNode), [When.new([TupleLiteral.new([Underscore.new] of ASTNode)] of ASTNode, 2.int32)], else: nil, exhaustive: true) + + assert_syntax_error "case 1\nin Int32; 2; when 2", "expected 'in', not 'when'" + assert_syntax_error "case 1\nwhen Int32; 2; in 2", "expected 'when', not 'in'" + assert_syntax_error "case 1\nin Int32; 2; else", "exhaustive case (case ... in) doesn't allow an 'else'" + assert_syntax_error "case 1\nin 1; 2", "expression of exhaustive case (case ... in) must be a constant (like `IO::Memory`), a generic (like `Array(Int32)`) a bool literal (true or false), a nil literal (nil) or a question method (like `.red?`)" + assert_syntax_error "case 1\nin _;", "'when _' is not supported" it_parses "select\nwhen foo\n2\nend", Select.new([Select::When.new("foo".call, 2.int32)]) it_parses "select\nwhen foo\n2\nwhen bar\n4\nend", Select.new([Select::When.new("foo".call, 2.int32), Select::When.new("bar".call, 4.int32)]) @@ -1475,7 +1498,7 @@ module Crystal it_parses "call(foo : A, end : B)", Call.new(nil, "call", [TypeDeclaration.new("foo".var, "A".path), TypeDeclaration.new("end".var, "B".path)] of ASTNode) it_parses "call foo : A, end : B", Call.new(nil, "call", [TypeDeclaration.new("foo".var, "A".path), TypeDeclaration.new("end".var, "B".path)] of ASTNode) - it_parses "case :foo; when :bar; 2; end", Case.new("foo".symbol, [When.new(["bar".symbol] of ASTNode, 2.int32)]) + it_parses "case :foo; when :bar; 2; end", Case.new("foo".symbol, [When.new(["bar".symbol] of ASTNode, 2.int32)], else: nil, exhaustive: false) it_parses "Foo.foo(count: 3).bar { }", Call.new(Call.new("Foo".path, "foo", named_args: [NamedArgument.new("count", 3.int32)]), "bar", block: Block.new) diff --git a/spec/compiler/parser/to_s_spec.cr b/spec/compiler/parser/to_s_spec.cr index 9d01a867107d..8a46d5d69438 100644 --- a/spec/compiler/parser/to_s_spec.cr +++ b/spec/compiler/parser/to_s_spec.cr @@ -132,6 +132,7 @@ describe "ASTNode#to_s" do expect_to_s %((1 <= 2) <= 3) expect_to_s %(1 <= (2 <= 3)) expect_to_s %(case 1; when .foo?; 2; end), %(case 1\nwhen .foo?\n 2\nend) + expect_to_s %(case 1; in .foo?; 2; end), %(case 1\nin .foo?\n 2\nend) expect_to_s %({(1 + 2)}) expect_to_s %({foo: (1 + 2)}) expect_to_s %q("#{(1 + 2)}") diff --git a/spec/compiler/semantic/case_spec.cr b/spec/compiler/semantic/case_spec.cr index 548fb5551bf7..8a6300ddc4a6 100644 --- a/spec/compiler/semantic/case_spec.cr +++ b/spec/compiler/semantic/case_spec.cr @@ -1,48 +1,75 @@ require "../../spec_helper" describe "semantic: case" do - it "checks exhaustiveness of union type" do - assert_error %( + it "doesn't check ehxaustiveness when using 'when'" do + assert_no_errors %( a = 1 || nil case a when Int32 end - ), - "case is not exhaustive.\n\nMissing types:\n - Nil" + ) end it "checks exhaustiveness of single type" do assert_error %( case 1 - when Nil + in Nil end ), "case is not exhaustive.\n\nMissing types:\n - Int32" end - it "covers all types" do + it "checks exhaustiveness of single type (T.class)" do assert_no_errors %( - a = 1 || nil - case a - when Int32 - when Nil + case Int32 + in Int32.class end ) end - it "can't prove exhaustiveness" do - assert_error %( - struct Int32 - def ===(other) - true - end + it "checks exhaustiveness of single type (Foo(T).class)" do + assert_no_errors %( + class Foo(T) end - case 1 - when 2 + case Foo(Int32) + in Foo(Int32).class + end + ) + end + + it "checks exhaustiveness of single type (generic)" do + assert_no_errors %( + class Foo(T) + end + + case Foo(Int32).new + in Foo(Int32) + end + ) + end + + it "errors if casing against a constant" do + assert_error %( + #{bool_case_eq} + + FOO = false + + case true + in FOO end ), - "can't prove case is exhaustive.\n\nPlease add an `else` clause." + "can't use constant values in exhaustive case, only constant types" + end + + it "covers all types" do + assert_no_errors %( + a = 1 || nil + case a + in Int32 + in Nil + end + ) end it "checks exhaustiveness of bool type (missing true)" do @@ -50,7 +77,7 @@ describe "semantic: case" do #{bool_case_eq} case false - when false + in false end ), "case is not exhaustive.\n\nMissing cases:\n - true" @@ -61,7 +88,7 @@ describe "semantic: case" do #{bool_case_eq} case false - when true + in true end ), "case is not exhaustive.\n\nMissing cases:\n - false" @@ -79,7 +106,7 @@ describe "semantic: case" do e = Color::Red case e - when .red? + in .red? end ), "case is not exhaustive for enum Color.\n\nMissing members:\n - Green\n - Blue" @@ -97,7 +124,7 @@ describe "semantic: case" do e = Color::Red case e - when Color::Red + in Color::Red end ), "case is not exhaustive for enum Color.\n\nMissing members:\n - Green\n - Blue" @@ -115,9 +142,9 @@ describe "semantic: case" do e = Color::Red case e - when .red? - when .green? - when .blue? + in .red? + in .green? + in .blue? end ) end @@ -137,9 +164,9 @@ describe "semantic: case" do end case foo - when .red? - when .green? - when .blue? + in .red? + in .green? + in .blue? end ) end @@ -149,9 +176,9 @@ describe "semantic: case" do #{bool_case_eq} case 1 || true - when Int32 - when true - when false + in Int32 + in true + in false end ) end @@ -166,13 +193,13 @@ describe "semantic: case" do a = 1 || Foo.new || Bar.new case a - when Foo + in Foo end ), "case is not exhaustive.\n\nMissing types:\n - Int32" end - it "checks exhaustiveness, covers when base type covers" do + it "checks exhaustiveness, covers in base type covers" do assert_no_errors %( class Foo end @@ -182,19 +209,19 @@ describe "semantic: case" do a = Bar.new case a - when Foo + in Foo end ) end - it "checks exhaustiveness, covers when base type covers (generic type)" do + it "checks exhaustiveness, covers in base type covers (generic type)" do assert_no_errors %( class Foo(T) end a = Foo(Int32).new case a - when Foo + in Foo end ) end @@ -208,7 +235,7 @@ describe "semantic: case" do end case nil - when nil + in nil end ) end @@ -223,22 +250,16 @@ describe "semantic: case" do a = 1 || nil case a - when nil - when Int32 - end - ) - end - - it "never warns on condless case without else" do - assert_no_errors %( - case - when 1 == 2 + in nil + in Int32 end ) end - it "always requires an else for Flags enum (no coverage)" do + it "can't prove case is exhaustive for @[Flags] enum" do assert_error %( + #{enum_eq} + struct Enum def includes?(other : self) false @@ -254,14 +275,42 @@ describe "semantic: case" do e = Color::Red case e - when .red? + in .red? end ), - "can't prove case is exhaustive.\n\nPlease add an `else` clause." + <<-ERROR + case is not exhaustive. + + Missing cases: + - Color + + Note that @[Flags] enum can't be proved to be exhaustive by matching against enum members. + In particular, the enum Color can't be proved to be exhaustive like that. + ERROR + end + + it "can prove case is exhaustive for @[Flags] enum when matching type" do + assert_no_errors %( + require "prelude" + + @[Flags] + enum Color + Red + Green + Blue + end + + e = Color::Red + case e + in Color + end + ) end - it "always requires an else for Flags enum (all members covered but doesn't count)" do + it "can't prove case is exhaustive for @[Flags] enum, tuple case" do assert_error %( + #{enum_eq} + struct Enum def includes?(other : self) false @@ -276,13 +325,19 @@ describe "semantic: case" do end e = Color::Red - case e - when .red? - when .green? - when .blue? + case {e} + in {.red?} end ), - "can't prove case is exhaustive.\n\nPlease add an `else` clause." + <<-ERROR + case is not exhaustive. + + Missing cases: + - {Color} + + Note that @[Flags] enum can't be proved to be exhaustive by matching against enum members. + In particular, the enum Color can't be proved to be exhaustive like that. + ERROR end it "checks exhaustiveness of enum combined with another type" do @@ -297,8 +352,8 @@ describe "semantic: case" do e = Color::Red || 1 case e - when Int32 - when .red? + in Int32 + in .red? end ), "case is not exhaustive for enum Color.\n\nMissing members:\n - Green\n - Blue" @@ -310,7 +365,7 @@ describe "semantic: case" do e = 1 || true case e - when true + in true end ), "case is not exhaustive.\n\nMissing cases:\n - false\n - Int32" @@ -322,10 +377,10 @@ describe "semantic: case" do b = 1 || 'a' case {a, b} - when {Int32, Char} - when {Int32, Int32} - when {Char, Int32} - when {Char, Char} + in {Int32, Char} + in {Int32, Int32} + in {Char, Int32} + in {Char, Char} end ) end @@ -335,9 +390,9 @@ describe "semantic: case" do a = 1 || 'a' case {a, a} - when {Int32, Char} - when {Int32, Int32} - when {Char, Char} + in {Int32, Char} + in {Int32, Int32} + in {Char, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Int32}" @@ -348,12 +403,12 @@ describe "semantic: case" do a = 1 || 'a' case {a, a, a} - when {Int32, Int32, Int32} - when {Int32, Char, Int32} - when {Int32, Char, Char} - when {Char, Int32, Int32} - when {Char, Char, Int32} - when {Char, Char, Char} + in {Int32, Int32, Int32} + in {Int32, Char, Int32} + in {Int32, Char, Char} + in {Char, Int32, Int32} + in {Char, Char, Int32} + in {Char, Char, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Int32, Char}\n - {Int32, Int32, Char}" @@ -364,7 +419,7 @@ describe "semantic: case" do #{bool_case_eq} case {true, 'a'} - when {true, Char} + in {true, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {false, Char}" @@ -381,8 +436,8 @@ describe "semantic: case" do end case {Color::Red, 'a'} - when {.red?, Char} - when {.blue?, Char} + in {.red?, Char} + in {.blue?, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {Color::Green, Char}" @@ -393,7 +448,7 @@ describe "semantic: case" do a = 1 || 'a' case {a, a} - when {_, Int32} + in {_, Int32} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Char}\n - {Int32, Char}" @@ -404,7 +459,7 @@ describe "semantic: case" do a = 1 || 'a' case {a, a} - when {Int32, _} + in {Int32, _} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Char}\n - {Char, Int32}" @@ -415,7 +470,7 @@ describe "semantic: case" do #{bool_case_eq} case {true, 1 || 'a'} - when {_, Int32} + in {_, Int32} end ), "case is not exhaustive.\n\nMissing cases:\n - {Bool, Char}" @@ -426,8 +481,8 @@ describe "semantic: case" do #{bool_case_eq} case {true, 1 || 'a'} - when {_, Int32} - when {false, Char} + in {_, Int32} + in {false, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {true, Char}" @@ -438,7 +493,7 @@ describe "semantic: case" do #{bool_case_eq} case {1 || 'a', true} - when {Int32, _} + in {Int32, _} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Bool}" @@ -449,8 +504,8 @@ describe "semantic: case" do #{bool_case_eq} case {1 || 'a', true} - when {Int32, _} - when {Char, false} + in {Int32, _} + in {Char, false} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, true}" @@ -467,7 +522,7 @@ describe "semantic: case" do end case {Color::Red, 1 || 'a'} - when {_, Int32} + in {_, Int32} end ), "case is not exhaustive.\n\nMissing cases:\n - {Color, Char}" @@ -484,8 +539,8 @@ describe "semantic: case" do end case {Color::Red, 1 || 'a'} - when {_, Int32} - when {.blue?, Char} + in {_, Int32} + in {.blue?, Char} end ), "case is not exhaustive.\n\nMissing cases:\n - {Color::Red, Char}\n - {Color::Green, Char}" @@ -502,7 +557,7 @@ describe "semantic: case" do end case {1 || 'a', Color::Red} - when {Int32, _} + in {Int32, _} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Color}" @@ -519,8 +574,8 @@ describe "semantic: case" do end case {1 || 'a', Color::Red} - when {Int32, _} - when {Char, .blue?} + in {Int32, _} + in {Char, .blue?} end ), "case is not exhaustive.\n\nMissing cases:\n - {Char, Color::Red}\n - {Char, Color::Green}" @@ -537,10 +592,10 @@ describe "semantic: case" do foo = 1 case {foo.bar, foo.bar} - when {Int32, Char} - when {Int32, Int32} - when {Char, Int32} - when {Char, Char} + in {Int32, Char} + in {Int32, Int32} + in {Char, Int32} + in {Char, Char} end ) end diff --git a/src/compiler/crystal/semantic/cleanup_transformer.cr b/src/compiler/crystal/semantic/cleanup_transformer.cr index e6ed1543c3c0..a0d1ffb39364 100644 --- a/src/compiler/crystal/semantic/cleanup_transformer.cr +++ b/src/compiler/crystal/semantic/cleanup_transformer.cr @@ -172,10 +172,10 @@ module Crystal end def transform(node : Case) - @exhaustiveness_checker.check(node) + @exhaustiveness_checker.check(node) if node.exhaustive? if expanded = node.expanded - unless node.else + if node.exhaustive? replace_unreachable_if_needed(node, expanded) end @@ -188,8 +188,7 @@ module Crystal # If any of the types checked in `case` is an enum, it can happen that # the unreachable can be reached by doing `SomeEnum.new(some_value)`. # In that case we replace the Unreachable node with `raise "..."`. - # In the future we should disallow creating such values unless the - # enum is marked as "open". + # In the future we should disallow creating such values. def replace_unreachable_if_needed(node, expanded) cond = node.cond return unless cond diff --git a/src/compiler/crystal/semantic/exhaustiveness_checker.cr b/src/compiler/crystal/semantic/exhaustiveness_checker.cr index be8cb3362f6c..f481e1f202df 100644 --- a/src/compiler/crystal/semantic/exhaustiveness_checker.cr +++ b/src/compiler/crystal/semantic/exhaustiveness_checker.cr @@ -3,13 +3,7 @@ struct Crystal::ExhaustivenessChecker end def check(node : Case) - # If there's an else clause we don't need to check anything - return if node.else - - cond = node.cond - - # No condition means it's just like a series of if/else - return unless cond + cond = node.cond.not_nil! if cond.is_a?(TupleLiteral) check_tuple_exp(node, cond) @@ -33,17 +27,12 @@ struct Crystal::ExhaustivenessChecker # Compute all the targets that we must cover targets = cond_types.map { |cond_type| compute_target(cond_type) } + # Is any type a @[Flags] enum? + flags_enum = cond_types.find { |type| type.is_a?(EnumType) && type.flags? } + # Are all patterns Path types? all_patterns_are_types = true - # Are all patterns things that we can handle? - # For example an integer literal is something that we don't - # take into account for exhaustiveness. - all_provable_patterns = true - - # Is any type a @[Flags] enum? - has_flags_enum = cond_types.any? { |type| type.is_a?(EnumType) && type.flags? } - # Start checking each `when`... node.whens.each do |a_when| a_when.conds.each do |when_cond| @@ -53,11 +42,7 @@ struct Crystal::ExhaustivenessChecker all_patterns_are_types = false end - if pattern - targets.each &.cover(pattern) - else - all_provable_patterns = false - end + targets.each &.cover(pattern) end end @@ -87,30 +72,27 @@ struct Crystal::ExhaustivenessChecker MSG when EnumTarget node.raise <<-MSG - case is not exhaustive for enum #{single_target.type}. + case is not exhaustive for enum #{single_target.type}. - Missing members: - - #{single_target.members.map(&.name).join("\n - ")} - MSG + Missing members: + - #{single_target.members.map(&.name).join("\n - ")} + MSG else # No specific error messages for non-single types end - if all_provable_patterns && !has_flags_enum - node.raise <<-MSG - case is not exhaustive. + msg = <<-MSG + case is not exhaustive. - Missing cases: - - #{targets.flat_map(&.missing_cases).join("\n - ")} - MSG - end + Missing cases: + - #{targets.flat_map(&.missing_cases).join("\n - ")} + MSG - # Otherwise we can't prove exhaustiveness and an `else` clause is required - node.raise <<-MSG - can't prove case is exhaustive. + if flags_enum + msg += "\n\n" + flags_enum_message(flags_enum) + end - Please add an `else` clause. - MSG + node.raise msg end private def check_tuple_exp(node, cond) @@ -126,13 +108,19 @@ struct Crystal::ExhaustivenessChecker expand_types(element_type) end - # Compute all the targets that we must cover - targets = compute_targets(all_expanded_types) + # Is any type a @[Flags] enum? + flags_enum = nil + all_expanded_types.each do |types| + types.each do |type| + if type.is_a?(EnumType) && type.flags? + flags_enum = type + break + end + end + break if flags_enum + end - # Are all patterns things that we can handle? - # For example an integer literal is something that we don't - # take into account for exhaustiveness. - all_provable_patterns = true + targets = compute_targets(all_expanded_types) # Start checking each `when`... node.whens.each do |a_when| @@ -142,12 +130,7 @@ struct Crystal::ExhaustivenessChecker when_pattern(when_cond_exp) end - if patterns.all? - patterns = patterns.map &.not_nil! - targets.each &.cover(patterns, 0) - else - all_provable_patterns = false - end + targets.each &.cover(patterns, 0) else # Not a tuple literal so we don't care # TODO: one could put `Tuple` or `Object` here and that would make @@ -162,26 +145,29 @@ struct Crystal::ExhaustivenessChecker # If we covered all types, we are done. return if targets.empty? - # If all patterns are stuff we can handle, show the missing cases - if all_provable_patterns - missing_cases = targets - .flat_map(&.missing_cases) - .map { |cases| "{#{cases}}" } - .join("\n - ") + missing_cases = targets + .flat_map(&.missing_cases) + .map { |cases| "{#{cases}}" } + .join("\n - ") - node.raise <<-MSG - case is not exhaustive. + msg = <<-MSG + case is not exhaustive. - Missing cases: - - #{missing_cases} - MSG + Missing cases: + - #{missing_cases} + MSG + + if flags_enum + msg += "\n\n" + flags_enum_message(flags_enum) end - # Otherwise we can't prove exhaustiveness and an `else` clause is required - node.raise <<-MSG - can't prove case is exhaustive. + node.raise msg + end - Please add an `else` clause. + private def flags_enum_message(flags_enum) + <<-MSG + Note that @[Flags] enum can't be proved to be exhaustive by matching against enum members. + In particular, the enum #{flags_enum} can't be proved to be exhaustive like that. MSG end @@ -236,20 +222,26 @@ struct Crystal::ExhaustivenessChecker # if it's an enum member, try to remove it from the targets. EnumMemberPattern.new(target_const) else - nil + when_cond.raise "can't use constant values in exhaustive case, only constant types" end + when Generic + TypePattern.new(when_cond.type.devirtualize) when Call + obj = when_cond.obj + # Check if it's something like `.foo?` to remove that member from the ones # we must cover. # Note: a user could override the meaning of such methods. # In the future it would be wise to mark these as non-redefinable # so this checks are sounds. - if when_cond.obj.is_a?(ImplicitObj) && - when_cond.args.empty? && when_cond.named_args.nil? && - !when_cond.block && !when_cond.block_arg && when_cond.name.ends_with?('?') + if obj.is_a?(ImplicitObj) && when_cond.name.ends_with?('?') EnumMemberNamePattern.new(when_cond.name.rchop) + elsif obj.is_a?(Path) && when_cond.name == "class" + TypePattern.new(obj.type.metaclass.devirtualize) + elsif obj.is_a?(Generic) && when_cond.name == "class" + TypePattern.new(obj.type.metaclass.devirtualize) else - nil + raise "Bug: unknown pattern in exhaustive case" end when BoolLiteral BoolPattern.new(when_cond.value) @@ -258,7 +250,7 @@ struct Crystal::ExhaustivenessChecker when Underscore UnderscorePattern.new else - nil + raise "Bug: unknown pattern in exhaustive case" end end diff --git a/src/compiler/crystal/semantic/literal_expander.cr b/src/compiler/crystal/semantic/literal_expander.cr index f54dda1b8f4e..af3002efe2da 100644 --- a/src/compiler/crystal/semantic/literal_expander.cr +++ b/src/compiler/crystal/semantic/literal_expander.cr @@ -465,7 +465,11 @@ module Crystal a_if = wh_if end - a_if.not_nil!.else = node.else || Unreachable.new + if node.exhaustive? + a_if.not_nil!.else = node.else || Unreachable.new + elsif node_else = node.else + a_if.not_nil!.else = node_else + end final_if = final_if.not_nil! final_exp = if assigns && !assigns.empty? @@ -524,7 +528,7 @@ module Crystal call = Call.new(channel, call_name, call_args).at(node) multi = MultiAssign.new(targets, [call] of ASTNode) case_cond = Var.new(index_name).at(node) - a_case = Case.new(case_cond, case_whens, case_else).at(node) + a_case = Case.new(case_cond, case_whens, case_else, exhaustive: false).at(node) Expressions.from([multi, a_case] of ASTNode).at(node) end @@ -631,11 +635,11 @@ module Crystal case obj when Path if cond.name == "class" - return IsA.new(right_side, Metaclass.new(obj.clone).at(obj)) + return IsA.new(right_side, Metaclass.new(obj).at(obj)) end when Generic if cond.name == "class" - return IsA.new(right_side, Metaclass.new(obj.clone).at(obj)) + return IsA.new(right_side, Metaclass.new(obj).at(obj)) end else # no special treatment diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index d22d9bbb2954..45c9582c5301 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -1178,8 +1178,9 @@ module Crystal class When < ASTNode property conds : Array(ASTNode) property body : ASTNode + property? exhaustive : Bool - def initialize(@conds, body = nil) + def initialize(@conds : Array(ASTNode), body : ASTNode? = nil, @exhaustive = false) @body = Expressions.from body end @@ -1189,18 +1190,22 @@ module Crystal end def clone_without_location - When.new(@conds.clone, @body.clone) + When.new(@conds.clone, @body.clone, @exhaustive) end - def_equals_and_hash @conds, @body + def_equals_and_hash @conds, @body, @exhaustive end class Case < ASTNode property cond : ASTNode? property whens : Array(When) property else : ASTNode? + property? exhaustive : Bool - def initialize(@cond, @whens, @else = nil) + def initialize(@cond : ASTNode?, @whens : Array(When), @else : ASTNode?, @exhaustive : Bool) + @whens.each do |wh| + wh.exhaustive = self.exhaustive? + end end def accept_children(visitor) @@ -1210,10 +1215,10 @@ module Crystal end def clone_without_location - Case.new(@cond.clone, @whens.clone, @else.clone) + Case.new(@cond.clone, @whens.clone, @else.clone, @exhaustive) end - def_equals_and_hash @cond, @whens, @else + def_equals_and_hash @exhaustive, @cond, @whens, @else end class Select < ASTNode diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 412a7445ef01..2c4a6a3bf80c 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -2578,6 +2578,7 @@ module Crystal whens = [] of When a_else = nil + exhaustive = nil # All when expressions, so we can detect duplicates when_exps = Set(ASTNode).new @@ -2586,7 +2587,18 @@ module Crystal case @token.type when :IDENT case @token.value - when :when + when :when, :in + if exhaustive.nil? + exhaustive = @token.value == :in + if exhaustive && !cond + raise "exhaustive case (case ... in) requires a case expression (case exp; in ..)" + end + elsif exhaustive && @token.value == :when + raise "expected 'in', not 'when'" + elsif !exhaustive && @token.value == :in + raise "expected 'when', not 'in'" + end + location = @token.location slash_is_regex! next_token_skip_space_or_newline @@ -2602,7 +2614,11 @@ module Crystal tuple_elements = [] of ASTNode while true - tuple_elements << parse_when_expression(cond, single: false) + exp = parse_when_expression(cond, single: false, exhaustive: exhaustive) + check_valid_exhaustive_expression(exp) if exhaustive + + tuple_elements << exp + skip_space if @token.type == :"," next_token_skip_space_or_newline @@ -2622,7 +2638,7 @@ module Crystal check :"}" next_token_skip_space else - exp = parse_when_expression(cond, single: true) + exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) when_conds << exp add_when_exp(when_exps, exp) skip_space @@ -2632,7 +2648,9 @@ module Crystal end else while true - exp = parse_when_expression(cond, single: true) + exp = parse_when_expression(cond, single: true, exhaustive: exhaustive) + check_valid_exhaustive_expression(exp) if exhaustive + when_conds << exp add_when_exp(when_exps, exp) skip_space @@ -2644,6 +2662,10 @@ module Crystal skip_space_or_newline whens << When.new(when_conds, when_body).at(location) when :else + if exhaustive + raise "exhaustive case (case ... in) doesn't allow an 'else'" + end + next_token_skip_statement_end a_else = parse_expressions skip_statement_end @@ -2661,7 +2683,30 @@ module Crystal end end - Case.new(cond, whens, a_else) + Case.new(cond, whens, a_else, exhaustive.nil? ? false : exhaustive) + end + + def check_valid_exhaustive_expression(exp) + case exp + when NilLiteral, BoolLiteral, Path, Generic, Underscore + return + when Call + if exp.obj.is_a?(ImplicitObj) && exp.name.ends_with?('?') && + exp.args.empty? && !exp.named_args && + !exp.block + return + end + + if (exp.obj.is_a?(Path) || exp.obj.is_a?(Generic)) && exp.name == "class" && + exp.args.empty? && !exp.named_args && + !exp.block + return + end + else + # Go on + end + + raise "expression of exhaustive case (case ... in) must be a constant (like `IO::Memory`), a generic (like `Array(Int32)`) a bool literal (true or false), a nil literal (nil) or a question method (like `.red?`)", exp.location.not_nil! end # Adds an expression to all when expressions and error on duplicates @@ -2720,7 +2765,7 @@ module Crystal false end - def parse_when_expression(cond, single) + def parse_when_expression(cond, single, exhaustive) if cond && @token.type == :"." next_token call = parse_var_or_call(force_call: true) @@ -2736,7 +2781,11 @@ module Crystal end call elsif single && @token.type == :UNDERSCORE - raise "'when _' is not supported, use 'else' block instead" + if exhaustive + raise "'when _' is not supported" + else + raise "'when _' is not supported, use 'else' block instead" + end else parse_op_assign_no_control end @@ -5794,7 +5843,7 @@ module Crystal true when :IDENT case @token.value - when :do, :end, :else, :elsif, :when, :rescue, :ensure, :then + when :do, :end, :else, :elsif, :when, :in, :rescue, :ensure, :then !next_comes_colon_space? else false diff --git a/src/compiler/crystal/syntax/to_s.cr b/src/compiler/crystal/syntax/to_s.cr index 94d514a6a3b9..cfc398c5ac8b 100644 --- a/src/compiler/crystal/syntax/to_s.cr +++ b/src/compiler/crystal/syntax/to_s.cr @@ -1343,9 +1343,11 @@ module Crystal cond.accept self end newline + node.whens.each do |wh| wh.accept self end + if node_else = node.else append_indent @str << keyword("else") @@ -1359,7 +1361,7 @@ module Crystal def visit(node : When) append_indent - @str << keyword("when") + @str << keyword(node.exhaustive? ? "in" : "when") @str << ' ' node.conds.join(", ", @str, &.accept self) newline diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index cf2bf4618f1d..6422c2ca7bae 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -3613,7 +3613,7 @@ module Crystal slash_is_regex! write_indent - write_keyword :when, " " + write_keyword(node.exhaustive? ? :in : :when, " ") base_indent = @column when_start_line = @line when_start_column = @column diff --git a/src/digest/md5.cr b/src/digest/md5.cr index 23ef552ce8d9..110859cbe966 100644 --- a/src/digest/md5.cr +++ b/src/digest/md5.cr @@ -24,7 +24,7 @@ class Digest::MD5 < Digest::Base end def update(inBuf, inLen) - in = uninitialized UInt32[16] + tmp_in = uninitialized UInt32[16] # compute number of bytes mod 64 mdi = (@i[0] >> 3) & 0x3F @@ -44,13 +44,13 @@ class Digest::MD5 < Digest::Base if mdi == 0x40 ii = 0 16.times do |i| - in[i] = (@in[ii + 3].to_u32 << 24) | - (@in[ii + 2].to_u32 << 16) | - (@in[ii + 1].to_u32 << 8) | - (@in[ii]) + tmp_in[i] = (@in[ii + 3].to_u32 << 24) | + (@in[ii + 2].to_u32 << 16) | + (@in[ii + 1].to_u32 << 8) | + (@in[ii]) ii += 4 end - transform in + transform tmp_in mdi = 0 end end @@ -208,11 +208,11 @@ class Digest::MD5 < Digest::Base end def final - in = uninitialized UInt32[16] + tmp_in = uninitialized UInt32[16] # save number of bits - in[14] = @i[0] - in[15] = @i[1] + tmp_in[14] = @i[0] + tmp_in[15] = @i[1] # compute number of bytes mod 64 mdi = ((@i[0] >> 3) & 0x3F).to_i32 @@ -224,13 +224,13 @@ class Digest::MD5 < Digest::Base # append length in bits and transform ii = 0 14.times do |i| - in[i] = (@in[ii + 3].to_u32 << 24) | - (@in[ii + 2].to_u32 << 16) | - (@in[ii + 1].to_u32 << 8) | - (@in[ii]) + tmp_in[i] = (@in[ii + 3].to_u32 << 24) | + (@in[ii + 2].to_u32 << 16) | + (@in[ii + 1].to_u32 << 8) | + (@in[ii]) ii += 4 end - transform in + transform tmp_in # store buffer in digest ii = 0 diff --git a/src/time.cr b/src/time.cr index edd5a4bee31f..bbeac6258e94 100644 --- a/src/time.cr +++ b/src/time.cr @@ -1296,7 +1296,7 @@ struct Time if local? self else - in(Location.local) + self.in(Location.local) end end From 73b5d93d7d43f5887799e08c4df75143fde59f53 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 11 May 2020 11:02:49 -0300 Subject: [PATCH 016/263] Add Experimental annotation and doc label (#9244) The annotation can be used without args or with a single string argument (similar to Deprecated). There is no semantics given to the annotation. --- spec/compiler/codegen/experimental_spec.cr | 50 ++++++++++ .../crystal/tools/doc/generator_spec.cr | 94 ++++++++++--------- src/annotations.cr | 13 +++ src/compiler/crystal/codegen/experimental.cr | 34 +++++++ src/compiler/crystal/program.cr | 3 +- .../crystal/semantic/top_level_visitor.cr | 4 + src/compiler/crystal/tools/doc/generator.cr | 29 ++++-- .../crystal/tools/doc/html/css/style.css | 6 ++ 8 files changed, 180 insertions(+), 53 deletions(-) create mode 100644 spec/compiler/codegen/experimental_spec.cr create mode 100644 src/compiler/crystal/codegen/experimental.cr diff --git a/spec/compiler/codegen/experimental_spec.cr b/spec/compiler/codegen/experimental_spec.cr new file mode 100644 index 000000000000..ddf66bd65b8f --- /dev/null +++ b/spec/compiler/codegen/experimental_spec.cr @@ -0,0 +1,50 @@ +require "../spec_helper" + +describe "Code gen: experimental" do + it "compiles with no argument" do + run(%( + @[Experimental] + def foo + end + + 2 + )).to_i.should eq(2) + end + + it "compiles with single string argument" do + run(%( + @[Experimental("lorem ipsum")] + def foo + end + + 2 + )).to_i.should eq(2) + end + + it "errors if invalid argument type" do + assert_error %( + @[Experimental(42)] + def foo + end + ), + "Error: first argument must be a String" + end + + it "errors if too many arguments" do + assert_error %( + @[Experimental("lorem ipsum", "extra arg")] + def foo + end + ), + "Error: wrong number of experimental annotation arguments (given 2, expected 1)" + end + + it "errors if missing link arguments" do + assert_error %( + @[Experimental(invalid: "lorem ipsum")] + def foo + end + ), + "Error: too many named arguments (given 1, expected maximum 0)" + end +end diff --git a/spec/compiler/crystal/tools/doc/generator_spec.cr b/spec/compiler/crystal/tools/doc/generator_spec.cr index 53380bb50a75..f659e580e330 100644 --- a/spec/compiler/crystal/tools/doc/generator_spec.cr +++ b/spec/compiler/crystal/tools/doc/generator_spec.cr @@ -1,5 +1,7 @@ require "../../../spec_helper" +private ANNOTATION_COLORS = {"Deprecated" => "red", "Experimental" => "lime"} + describe Doc::Generator do describe "#must_include_toplevel?" do it "returns false if program has nothing" do @@ -99,30 +101,32 @@ describe Doc::Generator do end describe "#formatted_summary" do - describe "with a Deprecated annotation, and no docs" do - it "should generate just the Deprecated tag" do - program = Program.new - generator = Doc::Generator.new program, ["."] - doc_type = Doc::Type.new generator, program - - a_def = Def.new "foo" - a_def.add_annotation(program.deprecated_annotation, Annotation.new(Crystal::Path.new("Deprecated"), ["don't use me".string] of ASTNode)) - doc_method = Doc::Method.new generator, doc_type, a_def, false - doc_method.formatted_summary.should eq %(

DEPRECATED don't use me

\n\n) + ANNOTATION_COLORS.each do |ann, color| + describe "with a #{ann} annotation, and no docs" do + it "should generate just the #{ann} tag" do + program = Program.new + generator = Doc::Generator.new program, ["."] + doc_type = Doc::Type.new generator, program + + a_def = Def.new "foo" + a_def.add_annotation(program.types[ann].as(Crystal::AnnotationType), Annotation.new(Crystal::Path.new(ann), ["lorem ipsum".string] of ASTNode)) + doc_method = Doc::Method.new generator, doc_type, a_def, false + doc_method.formatted_summary.should eq %(

#{ann.upcase} lorem ipsum

\n\n) + end end - end - describe "with a Deprecated annotation, and docs" do - it "should generate both the docs and Deprecated tag" do - program = Program.new - generator = Doc::Generator.new program, ["."] - doc_type = Doc::Type.new generator, program - - a_def = Def.new "foo" - a_def.doc = "Some Method" - a_def.add_annotation(program.deprecated_annotation, Annotation.new(Crystal::Path.new("Deprecated"), ["don't use me".string] of ASTNode)) - doc_method = Doc::Method.new generator, doc_type, a_def, false - doc_method.formatted_summary.should eq %(

Some Method

\n\n

DEPRECATED don't use me

\n\n) + describe "with a #{ann} annotation, and docs" do + it "should generate both the docs and #{ann} tag" do + program = Program.new + generator = Doc::Generator.new program, ["."] + doc_type = Doc::Type.new generator, program + + a_def = Def.new "foo" + a_def.doc = "Some Method" + a_def.add_annotation(program.types[ann].as(Crystal::AnnotationType), Annotation.new(Crystal::Path.new(ann), ["lorem ipsum".string] of ASTNode)) + doc_method = Doc::Method.new generator, doc_type, a_def, false + doc_method.formatted_summary.should eq %(

Some Method

\n\n

#{ann.upcase} lorem ipsum

\n\n) + end end end @@ -162,30 +166,32 @@ describe Doc::Generator do end describe "#formatted_doc" do - describe "with a Deprecated annotation, and no docs" do - it "should generate just the Deprecated tag" do - program = Program.new - generator = Doc::Generator.new program, ["."] - doc_type = Doc::Type.new generator, program - - a_def = Def.new "foo" - a_def.add_annotation(program.deprecated_annotation, Annotation.new(Crystal::Path.new("Deprecated"), ["don't use me".string] of ASTNode)) - doc_method = Doc::Method.new generator, doc_type, a_def, false - doc_method.formatted_doc.should eq %(

DEPRECATED don't use me

\n\n) + ANNOTATION_COLORS.each do |ann, color| + describe "with a #{ann} annotation, and no docs" do + it "should generate just the #{ann} tag" do + program = Program.new + generator = Doc::Generator.new program, ["."] + doc_type = Doc::Type.new generator, program + + a_def = Def.new "foo" + a_def.add_annotation(program.types[ann].as(Crystal::AnnotationType), Annotation.new(Crystal::Path.new(ann), ["lorem ipsum".string] of ASTNode)) + doc_method = Doc::Method.new generator, doc_type, a_def, false + doc_method.formatted_doc.should eq %(

#{ann.upcase} lorem ipsum

\n\n) + end end - end - describe "with a Deprecated annotation, and docs" do - it "should generate both the docs and Deprecated tag" do - program = Program.new - generator = Doc::Generator.new program, ["."] - doc_type = Doc::Type.new generator, program - - a_def = Def.new "foo" - a_def.doc = "Some Method" - a_def.add_annotation(program.deprecated_annotation, Annotation.new(Crystal::Path.new("Deprecated"), ["don't use me".string] of ASTNode)) - doc_method = Doc::Method.new generator, doc_type, a_def, false - doc_method.formatted_doc.should eq %(

Some Method

\n\n

DEPRECATED don't use me

\n\n) + describe "with a #{ann} annotation, and docs" do + it "should generate both the docs and #{ann} tag" do + program = Program.new + generator = Doc::Generator.new program, ["."] + doc_type = Doc::Type.new generator, program + + a_def = Def.new "foo" + a_def.doc = "Some Method" + a_def.add_annotation(program.types[ann].as(Crystal::AnnotationType), Annotation.new(Crystal::Path.new(ann), ["lorem ipsum".string] of ASTNode)) + doc_method = Doc::Method.new generator, doc_type, a_def, false + doc_method.formatted_doc.should eq %(

Some Method

\n\n

#{ann.upcase} lorem ipsum

\n\n) + end end end diff --git a/src/annotations.cr b/src/annotations.cr index fcf2d7c26e7c..db393cb6b772 100644 --- a/src/annotations.cr +++ b/src/annotations.cr @@ -51,3 +51,16 @@ end # using `ldflags`: `@[Link(ldflags: "-Lvendor/bin")]`. annotation Link end + +# This annotation marks methods, classes, constants, and macros as experimental. +# +# Experimental features are subject to change or be removed despite the +# [https://semver.org/](https://semver.org/) guarantees. +# +# ``` +# @[Experimental("Join discussion about this topic at ...")] +# def foo +# end +# ``` +annotation Experimental +end diff --git a/src/compiler/crystal/codegen/experimental.cr b/src/compiler/crystal/codegen/experimental.cr new file mode 100644 index 000000000000..3af1a6281b18 --- /dev/null +++ b/src/compiler/crystal/codegen/experimental.cr @@ -0,0 +1,34 @@ +module Crystal + struct ExperimentalAnnotation + getter message : String? + + def initialize(@message = nil) + end + + def self.from(ann : Annotation) + args = ann.args + named_args = ann.named_args + + if named_args + ann.raise "too many named arguments (given #{named_args.size}, expected maximum 0)" + end + + message = nil + count = 0 + + args.each do |arg| + case count + when 0 + arg.raise "first argument must be a String" unless arg.is_a?(StringLiteral) + message = arg.value + else + ann.wrong_number_of "experimental annotation arguments", args.size, "1" + end + + count += 1 + end + + new(message) + end + end +end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 2186fa7f9bba..118e8ffd8a89 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -242,6 +242,7 @@ module Crystal types["ReturnsTwice"] = @returns_twice_annotation = AnnotationType.new self, self, "ReturnsTwice" types["ThreadLocal"] = @thread_local_annotation = AnnotationType.new self, self, "ThreadLocal" types["Deprecated"] = @deprecated_annotation = AnnotationType.new self, self, "Deprecated" + types["Experimental"] = @experimental_annotation = AnnotationType.new self, self, "Experimental" define_crystal_constants end @@ -451,7 +452,7 @@ module Crystal packed_annotation thread_local_annotation no_inline_annotation always_inline_annotation naked_annotation returns_twice_annotation raises_annotation primitive_annotation call_convention_annotation - flags_annotation link_annotation extern_annotation deprecated_annotation) %} + flags_annotation link_annotation extern_annotation deprecated_annotation experimental_annotation) %} def {{name.id}} @{{name.id}}.not_nil! end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index e712c5eb1dce..7fe825cce656 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -1098,6 +1098,10 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor # arguments makes sense here. DeprecatedAnnotation.from(ann) yield annotation_type, ann + when @program.experimental_annotation + # ditto DeprecatedAnnotation + ExperimentalAnnotation.from(ann) + yield annotation_type, ann else yield annotation_type, ann end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 0a6ec2050ee4..6ebfcc437b9f 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -8,12 +8,13 @@ class Crystal::Doc::Generator # Adding a flag and associated css class will add support in parser FLAG_COLORS = { - "BUG" => "red", - "DEPRECATED" => "red", - "FIXME" => "yellow", - "NOTE" => "purple", - "OPTIMIZE" => "green", - "TODO" => "orange", + "BUG" => "red", + "DEPRECATED" => "red", + "EXPERIMENTAL" => "lime", + "FIXME" => "yellow", + "NOTE" => "purple", + "OPTIMIZE" => "green", + "TODO" => "orange", } FLAGS = FLAG_COLORS.keys @@ -306,7 +307,7 @@ class Crystal::Doc::Generator def summary(obj : Type | Method | Macro | Constant) doc = obj.doc - return if !doc && !obj.annotations(@program.deprecated_annotation) + return if !doc && !has_doc_annotations?(obj) summary obj, doc || "" end @@ -325,11 +326,15 @@ class Crystal::Doc::Generator def doc(obj : Type | Method | Macro | Constant) doc = obj.doc - return if !doc && !obj.annotations(@program.deprecated_annotation) + return if !doc && !has_doc_annotations?(obj) doc obj, doc || "" end + def has_doc_annotations?(obj) + obj.annotations(@program.deprecated_annotation) || obj.annotations(@program.experimental_annotation) + end + def doc(context, string) string = isolate_flag_lines string string += build_flag_lines_from_annotations context @@ -380,6 +385,14 @@ class Crystal::Doc::Generator io << "DEPRECATED: #{DeprecatedAnnotation.from(ann).message}\n\n" end end + + if anns = context.annotations(@program.experimental_annotation) + anns.each do |ann| + io << "\n\n" if first + first = false + io << "EXPERIMENTAL: #{ExperimentalAnnotation.from(ann).message}\n\n" + end + end end end diff --git a/src/compiler/crystal/tools/doc/html/css/style.css b/src/compiler/crystal/tools/doc/html/css/style.css index eaa0059ccc27..20bb0840335e 100644 --- a/src/compiler/crystal/tools/doc/html/css/style.css +++ b/src/compiler/crystal/tools/doc/html/css/style.css @@ -460,6 +460,12 @@ span.flag.purple { border-color: #1F0B37; } +span.flag.lime { + background-color: #a3ff00; + color: #222222; + border-color: #00ff1e; +} + .tooltip>span { position: absolute; opacity: 0; From edc261c85bc198abedb33f90f04d9b091337bdfb Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 11 May 2020 08:26:41 -0700 Subject: [PATCH 017/263] Implement host_flag? macro method, not affected by cross-compilation (#9049) --- spec/compiler/config_spec.cr | 4 ++-- src/compiler/crystal/compiler.cr | 2 +- src/compiler/crystal/config.cr | 8 +++---- src/compiler/crystal/crystal_path.cr | 2 +- src/compiler/crystal/macros.cr | 10 ++++++++ src/compiler/crystal/macros/methods.cr | 16 +++++++++---- src/compiler/crystal/program.cr | 2 +- src/compiler/crystal/semantic/flags.cr | 33 +++++++++++++++----------- 8 files changed, 50 insertions(+), 27 deletions(-) diff --git a/spec/compiler/config_spec.cr b/spec/compiler/config_spec.cr index 79510b22b6e3..df1667a38028 100644 --- a/spec/compiler/config_spec.cr +++ b/spec/compiler/config_spec.cr @@ -2,8 +2,8 @@ require "../spec_helper" require "./spec_helper" describe Crystal::Config do - it ".default_target" do - Crystal::Config.default_target.should eq Crystal::Codegen::Target.new({{ `crystal --version`.lines[-1] }}.lstrip("Default target: ")) + it ".host_target" do + Crystal::Config.host_target.should eq Crystal::Codegen::Target.new({{ `crystal --version`.lines[-1] }}.lstrip("Default target: ")) end {% if flag?(:linux) %} diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index c1336f42e322..7c0833329cfc 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -95,7 +95,7 @@ module Crystal # Codegen target to use in the compilation. # If not set, asks LLVM the default one for the current machine. - property codegen_target = Config.default_target + property codegen_target = Config.host_target # If `true`, prints the link command line that is performed # to create the executable. diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index 2cfa54023836..6d88fcfa9c0f 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -20,7 +20,7 @@ module Crystal Crystal #{version} #{formatted_sha}(#{date}) LLVM: #{llvm_version} - Default target: #{self.default_target} + Default target: #{self.host_target} DOC end @@ -36,10 +36,10 @@ module Crystal Time.unix(time).to_s("%Y-%m-%d") end - @@default_target : Crystal::Codegen::Target? + @@host_target : Crystal::Codegen::Target? - def self.default_target : Crystal::Codegen::Target - @@default_target ||= begin + def self.host_target : Crystal::Codegen::Target + @@host_target ||= begin target = Crystal::Codegen::Target.new({{env("CRYSTAL_CONFIG_TARGET")}} || LLVM.default_target_triple) if target.linux? diff --git a/src/compiler/crystal/crystal_path.cr b/src/compiler/crystal/crystal_path.cr index 16c79c3a8559..619cf1a4d343 100644 --- a/src/compiler/crystal/crystal_path.cr +++ b/src/compiler/crystal/crystal_path.cr @@ -12,7 +12,7 @@ module Crystal @crystal_path : Array(String) - def initialize(path = CrystalPath.default_path, codegen_target = Config.default_target) + def initialize(path = CrystalPath.default_path, codegen_target = Config.host_target) @crystal_path = path.split(Process::PATH_DELIMITER).reject &.empty? add_target_path(codegen_target) end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index 174f43f9724f..02ea666951a5 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -36,6 +36,16 @@ module Crystal::Macros def flag?(name) : BoolLiteral end + # Returns whether a [compile-time flag](https://crystal-lang.org/docs/syntax_and_semantics/compile_time_flags.html) + # is set for the *host* platform, which can differ from the target platform + # (`flag?`) during cross-compilation. + # + # ``` + # {{ host_flag?(:win32) }} # true or false + # ``` + def host_flag?(name) : BoolLiteral + end + # Prints AST nodes at compile-time. Useful for debugging macros. def puts(*expressions) : Nop end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 45dc8936c382..6937d06116f9 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -51,7 +51,7 @@ module Crystal interpret_debug(node) when "env" interpret_env(node) - when "flag?" + when "flag?", "host_flag?" interpret_flag?(node) when "puts" interpret_puts(node) @@ -145,10 +145,18 @@ module Crystal def interpret_flag?(node) if node.args.size == 1 node.args[0].accept self - flag = @last.to_macro_id - @last = BoolLiteral.new(@program.has_flag?(flag)) + flag_name = @last.to_macro_id + flags = case node.name + when "flag?" + @program.flags + when "host_flag?" + @program.host_flags + else + raise "Bug: unexpected macro method #{node.name}" + end + @last = BoolLiteral.new(flags.includes?(flag_name)) else - node.wrong_number_of_arguments "macro call 'flag?'", node.args.size, 1 + node.wrong_number_of_arguments "macro call '#{node.name}'", node.args.size, 1 end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 118e8ffd8a89..8325202e8a58 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -116,7 +116,7 @@ module Crystal # A `ProgressTracker` object which tracks compilation progress. property progress_tracker = ProgressTracker.new - property codegen_target = Config.default_target + property codegen_target = Config.host_target # Which kind of warnings wants to be detected. property warnings : Warnings = Warnings::All diff --git a/src/compiler/crystal/semantic/flags.cr b/src/compiler/crystal/semantic/flags.cr index 975a83444c6e..76ecd26a5783 100644 --- a/src/compiler/crystal/semantic/flags.cr +++ b/src/compiler/crystal/semantic/flags.cr @@ -1,5 +1,6 @@ class Crystal::Program @flags : Set(String)? + @host_flags : Set(String)? # Returns the flags for this program. By default these # are computed from the target triple (for example x86_64, @@ -11,6 +12,10 @@ class Crystal::Program @flags ||= flags_for_target(codegen_target) end + def host_flags + @host_flags ||= flags_for_target(Config.host_target) + end + # Returns `true` if *name* is in the program's flags. def has_flag?(name : String) flags.includes?(name) @@ -20,29 +25,29 @@ class Crystal::Program codegen_target.pointer_bit_width == 64 end - private def flags_for_target(codegen_target) + private def flags_for_target(target) flags = Set(String).new - flags.add codegen_target.architecture - flags.add codegen_target.vendor - flags.concat codegen_target.environment_parts + flags.add target.architecture + flags.add target.vendor + flags.concat target.environment_parts - flags.add "bits#{codegen_target.pointer_bit_width}" + flags.add "bits#{target.pointer_bit_width}" - flags.add "armhf" if codegen_target.armhf? + flags.add "armhf" if target.armhf? - flags.add "unix" if codegen_target.unix? - flags.add "win32" if codegen_target.win32? + flags.add "unix" if target.unix? + flags.add "win32" if target.win32? - flags.add "darwin" if codegen_target.macos? - if codegen_target.freebsd? + flags.add "darwin" if target.macos? + if target.freebsd? flags.add "freebsd" - flags.add "freebsd#{codegen_target.freebsd_version}" + flags.add "freebsd#{target.freebsd_version}" end - flags.add "openbsd" if codegen_target.openbsd? - flags.add "dragonfly" if codegen_target.dragonfly? + flags.add "openbsd" if target.openbsd? + flags.add "dragonfly" if target.dragonfly? - flags.add "bsd" if codegen_target.bsd? + flags.add "bsd" if target.bsd? flags end From b8d4ca0de1ca39c88fc959fb2a5e1bcd2b1cc153 Mon Sep 17 00:00:00 2001 From: Ewan Slater Date: Mon, 11 May 2020 16:29:12 +0100 Subject: [PATCH 018/263] Prefer HTTP basic authentication in OAuth2 client (#9127) --- spec/std/oauth2/client_spec.cr | 173 +++++++++++++++++++++++---------- src/oauth2/auth_scheme.cr | 15 +++ src/oauth2/client.cr | 39 +++++--- 3 files changed, 162 insertions(+), 65 deletions(-) create mode 100644 src/oauth2/auth_scheme.cr diff --git a/spec/std/oauth2/client_spec.cr b/spec/std/oauth2/client_spec.cr index 4a30c2b8d2a0..c5506cbbede8 100644 --- a/spec/std/oauth2/client_spec.cr +++ b/spec/std/oauth2/client_spec.cr @@ -40,79 +40,150 @@ describe OAuth2::Client do end describe "get_access_token_using_*" do - it "#get_access_token_using_authorization_code" do - server = HTTP::Server.new do |context| - body = context.request.body.not_nil!.gets_to_end - response = {access_token: "access_token", body: body} - context.response.print response.to_json + describe "using HTTP Basic authentication to pass credentials" do + it "#get_access_token_using_authorization_code" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" + + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + + token = client.get_access_token_using_authorization_code(authorization_code: "SDFhw39fwfg23flSfpawbef") + token.extra.not_nil!["body"].should eq %("redirect_uri=&grant_type=authorization_code&code=SDFhw39fwfg23flSfpawbef") + token.access_token.should eq "access_token" + end end - expected = %("client_id=client_id&client_secret=client_secret&redirect_uri=&grant_type=authorization_code&code=SDFhw39fwfg23flSfpawbef") - address = server.bind_unused_port "::1" + it "#get_access_token_using_resource_owner_credentials" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end - run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + address = server.bind_unused_port "::1" - token = client.get_access_token_using_authorization_code(authorization_code: "SDFhw39fwfg23flSfpawbef") - token.extra.not_nil!["body"].should eq expected - token.access_token.should eq "access_token" + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + + token = client.get_access_token_using_resource_owner_credentials(username: "user123", password: "monkey", scope: "read_posts") + token.extra.not_nil!["body"].should eq %("grant_type=password&username=user123&password=monkey&scope=read_posts") + token.access_token.should eq "access_token" + end end - end - it "#get_access_token_using_resource_owner_credentials" do - server = HTTP::Server.new do |context| - body = context.request.body.not_nil!.gets_to_end - response = {access_token: "access_token", body: body} - context.response.print response.to_json + it "#get_access_token_using_client_credentials" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" + + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + + token = client.get_access_token_using_client_credentials(scope: "read_posts") + token.extra.not_nil!["body"].should eq %("grant_type=client_credentials&scope=read_posts") + token.access_token.should eq "access_token" + end end - expected = %("client_id=client_id&client_secret=client_secret&grant_type=password&username=user123&password=monkey&scope=read_posts") - address = server.bind_unused_port "::1" + it "#get_access_token_using_refresh_token" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" - run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" - token = client.get_access_token_using_resource_owner_credentials(username: "user123", password: "monkey", scope: "read_posts") - token.extra.not_nil!["body"].should eq expected - token.access_token.should eq "access_token" + token = client.get_access_token_using_refresh_token(scope: "read_posts", refresh_token: "some_refresh_token") + token.extra.not_nil!["body"].should eq %("grant_type=refresh_token&refresh_token=some_refresh_token&scope=read_posts") + token.access_token.should eq "access_token" + end end end - - it "#get_access_token_using_client_credentials" do - server = HTTP::Server.new do |context| - body = context.request.body.not_nil!.gets_to_end - response = {access_token: "access_token", body: body} - context.response.print response.to_json + describe "using Request Body to pass credentials" do + it "#get_access_token_using_authorization_code" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" + + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + + token = client.get_access_token_using_authorization_code(authorization_code: "SDFhw39fwfg23flSfpawbef") + token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&redirect_uri=&grant_type=authorization_code&code=SDFhw39fwfg23flSfpawbef") + token.access_token.should eq "access_token" + end end - expected = %("client_id=client_id&client_secret=client_secret&grant_type=client_credentials&scope=read_posts") - address = server.bind_unused_port "::1" + it "#get_access_token_using_resource_owner_credentials" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end - run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + address = server.bind_unused_port "::1" - token = client.get_access_token_using_client_credentials(scope: "read_posts") - token.extra.not_nil!["body"].should eq expected - token.access_token.should eq "access_token" + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + + token = client.get_access_token_using_resource_owner_credentials(username: "user123", password: "monkey", scope: "read_posts") + token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=password&username=user123&password=monkey&scope=read_posts") + token.access_token.should eq "access_token" + end end - end - it "#get_access_token_using_refresh_token" do - server = HTTP::Server.new do |context| - body = context.request.body.not_nil!.gets_to_end - response = {access_token: "access_token", body: body} - context.response.print response.to_json + it "#get_access_token_using_client_credentials" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" + + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + + token = client.get_access_token_using_client_credentials(scope: "read_posts") + token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=client_credentials&scope=read_posts") + token.access_token.should eq "access_token" + end end - expected = %("client_id=client_id&client_secret=client_secret&grant_type=refresh_token&refresh_token=some_refresh_token&scope=read_posts") - address = server.bind_unused_port "::1" + it "#get_access_token_using_refresh_token" do + server = HTTP::Server.new do |context| + body = context.request.body.not_nil!.gets_to_end + response = {access_token: "access_token", body: body} + context.response.print response.to_json + end + + address = server.bind_unused_port "::1" - run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + run_server(server) do + client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody - token = client.get_access_token_using_refresh_token(scope: "read_posts", refresh_token: "some_refresh_token") - token.extra.not_nil!["body"].should eq expected - token.access_token.should eq "access_token" + token = client.get_access_token_using_refresh_token(scope: "read_posts", refresh_token: "some_refresh_token") + token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=refresh_token&refresh_token=some_refresh_token&scope=read_posts") + token.access_token.should eq "access_token" + end end end end diff --git a/src/oauth2/auth_scheme.cr b/src/oauth2/auth_scheme.cr new file mode 100644 index 000000000000..1402239d4b15 --- /dev/null +++ b/src/oauth2/auth_scheme.cr @@ -0,0 +1,15 @@ +# Enum of supported mechanisms used to pass credentials to the server. +# +# According to https://tools.ietf.org/html/rfc6749#section-2.3.1: +# +# > "Including the client credentials in the request-body using the +# > two parameters is NOT RECOMMENDED and SHOULD be limited to +# > clients unable to directly utilize the HTTP Basic authentication +# > scheme (or other password-based HTTP authentication schemes)." +# +# Therefore, HTTP Basic is preferred, and Request Body should only +# be used if the server does not support HTTP Basic. +enum OAuth2::AuthScheme + HTTPBasic + RequestBody +end diff --git a/src/oauth2/client.cr b/src/oauth2/client.cr index b6b14e852d49..71e1ab2f498b 100644 --- a/src/oauth2/client.cr +++ b/src/oauth2/client.cr @@ -60,12 +60,18 @@ class OAuth2::Client # *token_uri* can be relative or absolute. # If they are relative, the given *host*, *port* and *scheme* will be used. # If they are absolute, the absolute URL will be used. + # + # As per https://tools.ietf.org/html/rfc6749#section-2.3.1, + # `AuthScheme::HTTPBasic` is the default *auth_scheme* (the mechanism used to + # transmit the client credentials to the server). `AuthScheme::RequestBody` should + # only be used if the server does not support HTTP Basic. def initialize(@host : String, @client_id : String, @client_secret : String, @port : Int32? = nil, @scheme = "https", @authorize_uri = "/oauth2/authorize", @token_uri = "/oauth2/token", - @redirect_uri : String? = nil) + @redirect_uri : String? = nil, + @auth_scheme : AuthScheme = :http_basic) end # Builds an authorize URI, as specified by @@ -145,18 +151,26 @@ class OAuth2::Client end private def get_access_token : AccessToken - body = HTTP::Params.build do |form| - form.add("client_id", @client_id) - form.add("client_secret", @client_secret) - yield form - end - headers = HTTP::Headers{ "Accept" => "application/json", "Content-Type" => "application/x-www-form-urlencoded", } - response = HTTP::Client.post(token_uri, form: body, headers: headers) + body = HTTP::Params.build do |form| + case @auth_scheme + when .request_body? + form.add("client_id", @client_id) + form.add("client_secret", @client_secret) + when .http_basic? + headers.add( + "Authorization", + "Basic #{Base64.strict_encode("#{@client_id}:#{@client_secret}")}" + ) + end + yield form + end + + response = HTTP::Client.post token_uri, form: body, headers: headers case response.status when .ok?, .created? OAuth2::AccessToken.from_json(response.body) @@ -165,15 +179,12 @@ class OAuth2::Client end end - private def token_uri + private def token_uri : URI uri = URI.parse(@token_uri) - if uri.host - # If it's an absolute URI, use that one - @token_uri + uri else - # Otherwise use the default one - URI.new(@scheme, @host, @port, @token_uri).to_s + URI.new(@scheme, @host, @port, @token_uri) end end end From 90d89d022b80378b7c3316c0c549433c519746d1 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 11 May 2020 12:29:58 -0300 Subject: [PATCH 019/263] Change `HTTP::Request#remote_address` type to `Socket::Address?` (#9210) --- .../http/server/handlers/log_handler_spec.cr | 6 +++++- spec/std/http/server/server_spec.cr | 4 ++-- src/http/request.cr | 20 +++++++++++++++---- src/http/server/handlers/log_handler.cr | 17 +++++++++++++++- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/spec/std/http/server/handlers/log_handler_spec.cr b/spec/std/http/server/handlers/log_handler_spec.cr index e7d7e2de046a..4f78b8dcd100 100644 --- a/spec/std/http/server/handlers/log_handler_spec.cr +++ b/spec/std/http/server/handlers/log_handler_spec.cr @@ -7,7 +7,11 @@ describe HTTP::LogHandler do it "logs" do io = IO::Memory.new request = HTTP::Request.new("GET", "/") - request.remote_address = "192.168.0.1" + {% if flag?(:win32) %} + request.remote_address = "192.168.0.1" + {% else %} + request.remote_address = Socket::IPAddress.new("192.168.0.1", 1234) + {% end %} response = HTTP::Server::Response.new(io) context = HTTP::Server::Context.new(request, response) diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index 84decd23afb4..cc932a7f7b4d 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -493,7 +493,7 @@ describe "#remote_address" do HTTP::Client.new(URI.parse("http://#{address1}/")) do |client| client.get("/") - remote_address.should eq(client.@socket.as(IPSocket).local_address.to_s) + remote_address.should eq(client.@socket.as(IPSocket).local_address) end end end @@ -516,7 +516,7 @@ describe "#remote_address" do uri: URI.parse("https://#{ip_address1}"), tls: client_context) do |client| client.get("/") - remote_address.should eq(client.@socket.as(OpenSSL::SSL::Socket).local_address.to_s) + remote_address.should eq(client.@socket.as(OpenSSL::SSL::Socket).local_address) end end end diff --git a/src/http/request.cr b/src/http/request.cr index ade24aeabc56..ab95dc6c1c2f 100644 --- a/src/http/request.cr +++ b/src/http/request.cr @@ -2,6 +2,16 @@ require "./common" require "uri" require "http/params" +# TODO: Remove this once `Socket` is working on Windows +{% begin %} +private alias RemoteAddressType = + {% if flag?(:win32) %} + String? + {% else %} + Socket::Address? + {% end %} +{% end %} + # An HTTP request. # # It serves both to perform requests by an `HTTP::Client` and to @@ -27,7 +37,7 @@ class HTTP::Request # Middlewares can overwrite this value. # # This property is not used by `HTTP::Client`. - property remote_address : String? + property remote_address : RemoteAddressType def self.new(method : String, resource : String, headers : Headers? = nil, body : String | Bytes | IO | Nil = nil, version = "HTTP/1.1") # Duplicate headers to prevent the request from modifying data that the user might hold. @@ -109,9 +119,11 @@ class HTTP::Request # No need to dup headers since nobody else holds them request = new line.method, line.resource, headers, body, line.http_version, internal: nil - if io.responds_to?(:remote_address) - request.remote_address = io.remote_address.try &.to_s - end + {% unless flag?(:win32) %} + if io.responds_to?(:remote_address) + request.remote_address = io.remote_address + end + {% end %} return request end diff --git a/src/http/server/handlers/log_handler.cr b/src/http/server/handlers/log_handler.cr index c3fb0af31e8b..43c9af150c3e 100644 --- a/src/http/server/handlers/log_handler.cr +++ b/src/http/server/handlers/log_handler.cr @@ -24,7 +24,22 @@ class HTTP::LogHandler req = context.request res = context.response - @log.info { "#{req.remote_address || "-"} - #{req.method} #{req.resource} #{req.version} - #{res.status_code} (#{elapsed_text})" } + + addr = + {% begin %} + case remote_address = req.remote_address + when nil + "-" + {% unless flag?(:win32) %} + when Socket::IPAddress + remote_address.address + {% end %} + else + remote_address + end + {% end %} + + @log.info { "#{addr} - #{req.method} #{req.resource} #{req.version} - #{res.status_code} (#{elapsed_text})" } end end From 8e382d78f45829396f87f45724c22a9bbcddbcbf Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 11 May 2020 12:42:42 -0300 Subject: [PATCH 020/263] HTTP::Server: defer request upgrade (aka: WebSockets) (#9243) --- src/http/server.cr | 2 ++ src/http/server/handlers/websocket_handler.cr | 2 -- src/http/server/request_processor.cr | 17 ++++++----------- src/http/server/response.cr | 18 +++++------------- 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/http/server.cr b/src/http/server.cr index 8d4d948a41b3..ee19e61666e8 100644 --- a/src/http/server.cr +++ b/src/http/server.cr @@ -511,6 +511,8 @@ class HTTP::Server {% end %} @processor.process(io, io) + ensure + io.close rescue IO::Error end # This method handles exceptions raised at `Socket#accept?`. diff --git a/src/http/server/handlers/websocket_handler.cr b/src/http/server/handlers/websocket_handler.cr index df10143a6381..52813d8c8734 100644 --- a/src/http/server/handlers/websocket_handler.cr +++ b/src/http/server/handlers/websocket_handler.cr @@ -57,8 +57,6 @@ class HTTP::WebSocketHandler ws_session = WebSocket.new(io, sync_close: false) @proc.call(ws_session, context) ws_session.run - ensure - io.close end end diff --git a/src/http/server/request_processor.cr b/src/http/server/request_processor.cr index 05e43e1495b7..632d46eb0a38 100644 --- a/src/http/server/request_processor.cr +++ b/src/http/server/request_processor.cr @@ -23,7 +23,6 @@ class HTTP::Server::RequestProcessor end def process(input, output) - must_close = true response = Response.new(output) begin @@ -63,13 +62,15 @@ class HTTP::Server::RequestProcessor response.output.close end - if response.upgraded? - must_close = false + output.flush + + # If there is an upgrade handler, hand over + # the connection to it and return + if upgrade_handler = response.upgrade_handler + upgrade_handler.call(output) return end - output.flush - break unless request.keep_alive? # Don't continue if the handler set `Connection` header to `close` @@ -93,12 +94,6 @@ class HTTP::Server::RequestProcessor end rescue IO::Error # IO-related error, nothing to do - ensure - begin - input.close if must_close - rescue IO::Error - # IO-related error, nothing to do - end end end end diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 07ff9d24f841..2d1c171bb022 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -14,7 +14,6 @@ class HTTP::Server # # A response can be upgraded with the `upgrade` method. Once invoked, headers # are written and the connection `IO` (a socket) is yielded to the given block. - # The block must invoke `close` afterwards, the server won't do it in this case. # This is useful to implement protocol upgrades, such as websockets. class Response < IO # The response headers (`HTTP::Headers`). These must be set before writing to the response. @@ -34,6 +33,9 @@ class HTTP::Server # body. If not set, the default value is 200 (OK). property status : HTTP::Status + # :nodoc: + property upgrade_handler : (IO ->)? + @cookies : HTTP::Cookies? # :nodoc: @@ -41,7 +43,6 @@ class HTTP::Server @headers = Headers.new @status = :ok @wrote_headers = false - @upgraded = false @output = output = @original_output = Output.new(@io) output.response = self end @@ -53,7 +54,6 @@ class HTTP::Server @cookies = nil @status = :ok @wrote_headers = false - @upgraded = false @output = @original_output @original_output.reset end @@ -97,18 +97,10 @@ class HTTP::Server end # Upgrades this response, writing headers and yieling the connection `IO` (a socket) to the given block. - # The block must invoke `close` afterwards, the server won't do it in this case. # This is useful to implement protocol upgrades, such as websockets. - def upgrade - @upgraded = true + def upgrade(&block : IO ->) write_headers - flush - yield @io - end - - # :nodoc: - def upgraded? - @upgraded + @upgrade_handler = block end # Flushes the output. This method must be implemented if wrapping the response output. From 0a3824b07d75b7f5039fc371ed07f32185a20fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 11 May 2020 17:49:08 +0200 Subject: [PATCH 021/263] Skip no closure check for non-Crystal procs (#9248) * Validate raise has the right return type before using it in the no closure check Otherwise this returns a hard to understand LLVM IR verification error * Skip no closure check for non-Crystal procs They can't have a closure anyways and more importantly are represented as a single pointer. So the check could not even deconstruct them into two pointers. This previously generated invalid code trying to pass the function pointer type to the check function expecting the Crystal proc type --- spec/compiler/codegen/closure_spec.cr | 47 +++++++++++++++++++++++++ src/compiler/crystal/codegen/codegen.cr | 11 +++--- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/closure_spec.cr b/spec/compiler/codegen/closure_spec.cr index 15f2d1cf92d4..90e71c36317a 100644 --- a/spec/compiler/codegen/closure_spec.cr +++ b/spec/compiler/codegen/closure_spec.cr @@ -681,4 +681,51 @@ describe "Code gen: closure" do f1.call &+ f2.call )) end + + it "ensures it can raise from the closure check" do + expect_raises(Exception, "::raise must be of NoReturn return type!") do + codegen(%( + def raise(m : String) + end + + fun a(a : -> Int32) + end + + value = 1 + p = ->{ value } + a(p) + )) + end + end + + it "allows passing an external function along" do + codegen(%( + lib LibC + fun exit(c : Int32) : NoReturn + end + + def raise(a) : NoReturn + LibC.exit(1) + end + + lib LibA + fun a(a : Void* -> Void*) + end + + fun b(a : Void* -> Void*) + LibA.a(a) + end + )) + + codegen(%( + lib LibFoo + struct S + callback : -> + end + end + + s = LibFoo::S.new + s.callback = nil + )) + end end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index 41617ee584d5..8ee47d118040 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -1324,6 +1324,7 @@ module Crystal location = Location.new(@program.filename, 1, 1) call = Call.global("raise", StringLiteral.new("passing a closure to C is not allowed")).at(location) @program.visit_main call + call.raise "::raise must be of NoReturn return type!" unless call.type.is_a?(NoReturnType) call end end @@ -1530,10 +1531,12 @@ module Crystal end def check_proc_is_not_closure(value, type) - check_fun_name = "~check_proc_is_not_closure" - func = @main_mod.functions[check_fun_name]? || create_check_proc_is_not_closure_fun(check_fun_name) - func = check_main_fun check_fun_name, func - value = call func, [value] of LLVM::Value + if value.type == llvm_typer.proc_type + check_fun_name = "~check_proc_is_not_closure" + func = @main_mod.functions[check_fun_name]? || create_check_proc_is_not_closure_fun(check_fun_name) + func = check_main_fun check_fun_name, func + value = call func, [value] of LLVM::Value + end bit_cast value, llvm_proc_type(type) end From a2fa135b1bc45eb7d664fc3c10233309800bb8ec Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 11 May 2020 12:49:30 -0300 Subject: [PATCH 022/263] Make `NamedTuple#sorted_keys` public (#9263) --- spec/std/named_tuple_spec.cr | 5 +++++ src/named_tuple.cr | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/std/named_tuple_spec.cr b/spec/std/named_tuple_spec.cr index 0b49297077ce..7f575b757940 100644 --- a/spec/std/named_tuple_spec.cr +++ b/spec/std/named_tuple_spec.cr @@ -352,6 +352,11 @@ describe "NamedTuple" do tup.keys.should eq({:a, :b}) end + it "does sorted_keys" do + tup = {foo: 1, bar: 2, baz: 3} + tup.sorted_keys.should eq({:bar, :baz, :foo}) + end + it "does values" do tup = {a: 1, b: 'a'} tup.values.should eq({1, 'a'}) diff --git a/src/named_tuple.cr b/src/named_tuple.cr index 09b56f44af84..fc5b76d8a732 100644 --- a/src/named_tuple.cr +++ b/src/named_tuple.cr @@ -266,7 +266,13 @@ struct NamedTuple {% end %} end - protected def sorted_keys + # Returns a `Tuple` of symbols with the keys in this named tuple, sorted by name. + # + # ``` + # tuple = {foo: 1, bar: 2, baz: 3} + # tuple.sorted_keys # => {:bar, :baz, :foo} + # ``` + def sorted_keys {% begin %} Tuple.new( {% for key in T.keys.sort %} From 70959cf121f35130c465228cb4a68449b942f26b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 11 May 2020 14:21:43 -0300 Subject: [PATCH 023/263] Add Log Metadata (#9227) * Split Context from Metadata Keep Log::Context for DSL over the current fiber context. Make it a wrapper over Log::Metadata * Revert guarantee of restoring fiber context automatically With entry local metadata this is no longer need. * Rename methods * Add emitter dsl * Enforce hash like metadata in entry. And display it in IO::Backend#default_format * Fix docs * Fix immutability of datum in nested containers --- spec/std/log/broadcast_backend_spec.cr | 6 +- spec/std/log/context_spec.cr | 184 +++++++++---------------- spec/std/log/log_spec.cr | 151 +++++++++++++++++++- spec/std/log/metadata_spec.cr | 69 ++++++++++ src/crystal/datum.cr | 6 +- src/log.cr | 9 +- src/log/context.cr | 70 ---------- src/log/entry.cr | 6 +- src/log/io_backend.cr | 3 + src/log/json.cr | 4 +- src/log/log.cr | 15 +- src/log/main.cr | 76 +++++++--- src/log/metadata.cr | 96 +++++++++++++ 13 files changed, 465 insertions(+), 230 deletions(-) create mode 100644 spec/std/log/metadata_spec.cr delete mode 100644 src/log/context.cr create mode 100644 src/log/metadata.cr diff --git a/spec/std/log/broadcast_backend_spec.cr b/spec/std/log/broadcast_backend_spec.cr index eff16aaf1b62..fa18ff5fa72f 100644 --- a/spec/std/log/broadcast_backend_spec.cr +++ b/spec/std/log/broadcast_backend_spec.cr @@ -14,9 +14,9 @@ describe Log::BroadcastBackend do main.append(backend_a, s(:info)) main.append(backend_b, s(:error)) - debug_entry = Log::Entry.new("", s(:debug), "", nil) - info_entry = Log::Entry.new("", s(:info), "", nil) - error_entry = Log::Entry.new("", s(:error), "", nil) + debug_entry = Log::Entry.new("", s(:debug), "", Log::Metadata.empty, nil) + info_entry = Log::Entry.new("", s(:info), "", Log::Metadata.empty, nil) + error_entry = Log::Entry.new("", s(:error), "", Log::Metadata.empty, nil) main.write debug_entry main.write info_entry diff --git a/spec/std/log/context_spec.cr b/spec/std/log/context_spec.cr index 17e2921ae63a..ccc66b48503d 100644 --- a/spec/std/log/context_spec.cr +++ b/spec/std/log/context_spec.cr @@ -1,12 +1,11 @@ require "spec" require "log" -require "log/json" -private def c(value) - Log::Context.new(value) +private def m(value) + Log::Metadata.new(value) end -describe Log::Context do +describe "Log.context" do before_each do Log.context.clear end @@ -15,152 +14,93 @@ describe Log::Context do Log.context.clear end - it "initialize" do - c({a: 1}).should eq(c({"a" => c(1)})) - c({a: 1, b: ["str", true], num: 1i64}).should eq(c({"a" => c(1), "b" => c([c("str"), c(true)]), "num" => c(1i64)})) - c({a: 1f32, b: 1f64}).should eq(c({"a" => c(1f32), "b" => c(1f64)})) - t = Time.local - c({time: t}).should eq(c({"time" => c(t)})) - Log::Context.new.should eq(c(NamedTuple.new)) - end - - it "empty" do - Log::Context.empty.should eq(Log::Context.new) - Log::Context.empty.object_id.should_not eq(Log::Context.new.object_id) - Log::Context.empty.object_id.should eq(Log::Context.empty.object_id) - end - it "validates hash" do expect_raises(ArgumentError, "Expected hash context, not Int32") do - Log.context = c(1) + Log.context = m(1) end end - it "immutability" do - context = c({a: 1}) - other = context.as_h - other["a"] = c(2) + it "can be set and cleared" do + Log.context.metadata.should eq(Log::Metadata.new) - other.should eq({"a" => c(2)}) - context.should eq(c({a: 1})) - end + Log.context.set a: 1 + Log.context.metadata.should eq(m({a: 1})) - it "merge" do - c({a: 1}).merge(c({b: 2})).should eq(c({a: 1, b: 2})) - c({a: 1, b: 3}).merge(c({b: 2})).should eq(c({a: 1, b: 2})) - c({a: 1, b: 3}).merge(c({b: nil})).should eq(c({a: 1, b: nil})) + Log.context.clear + Log.context.metadata.should eq(Log::Metadata.new) end - it "merge against Log::Context.empty without creating a new instance" do - c1 = c({a: 1, b: 3}) - c1.merge(Log::Context.empty).should be(c1) - Log::Context.empty.merge(c1).should be(c1) + it "is extended by set" do + Log.context.set a: 1 + Log.context.set b: 2 + Log.context.metadata.should eq(m({a: 1, b: 2})) end - it "accessors" do - c(nil).as_nil.should be_nil - - c(1).as_i.should eq(1) - - c("a").as_s.should eq("a") - c(1).as_s?.should be_nil - - c(true).as_bool.should eq(true) - c(false).as_bool.should eq(false) - c(true).as_bool?.should eq(true) - c(false).as_bool?.should eq(false) - c(nil).as_bool?.should be_nil + it "existing keys are overwritten by set" do + Log.context.set a: 1, b: 1 + Log.context.set b: 2, c: 3 + Log.context.metadata.should eq(m({a: 1, b: 2, c: 3})) end - describe "implicit context" do - it "can be set and cleared" do - Log.context.should eq(Log::Context.new) + it "is restored after with_context" do + Log.context.set a: 1 - Log.context.set a: 1 - Log.context.should eq(c({a: 1})) - - Log.context.clear - Log.context.should eq(Log::Context.new) - end - - it "is extended by set" do - Log.context.set a: 1 + Log.with_context do Log.context.set b: 2 - Log.context.should eq(c({a: 1, b: 2})) - end - - it "existing keys are overwritten by set" do - Log.context.set a: 1, b: 1 - Log.context.set b: 2, c: 3 - Log.context.should eq(c({a: 1, b: 2, c: 3})) + Log.context.metadata.should eq(m({a: 1, b: 2})) end - it "is restored after with_context" do - Log.context.set a: 1 - - Log.with_context do - Log.context.set b: 2 - Log.context.should eq(c({a: 1, b: 2})) - end - - Log.context.should eq(c({a: 1})) - end - - it "is restored after with_context of Log instance" do - Log.context.set a: 1 - log = Log.for("temp") + Log.context.metadata.should eq(m({a: 1})) + end - log.with_context do - log.context.set b: 2 - log.context.should eq(c({a: 1, b: 2})) - end + it "is restored after with_context of Log instance" do + Log.context.set a: 1 + log = Log.for("temp") - log.context.should eq(c({a: 1})) + log.with_context do + log.context.set b: 2 + log.context.metadata.should eq(m({a: 1, b: 2})) end - it "is per fiber" do - Log.context.set a: 1 - done = Channel(Nil).new + log.context.metadata.should eq(m({a: 1})) + end - f = spawn do - Log.context.should eq(Log::Context.new) - Log.context.set b: 2 - Log.context.should eq(c({b: 2})) + it "is per fiber" do + Log.context.set a: 1 + done = Channel(Nil).new - done.receive - done.receive - end + f = spawn do + Log.context.metadata.should eq(Log::Metadata.new) + Log.context.set b: 2 + Log.context.metadata.should eq(m({b: 2})) - done.send nil - Log.context.should eq(c({a: 1})) - done.send nil + done.receive + done.receive end - it "is assignable from a hash with symbol keys" do - Log.context.set a: 1 - extra = {:b => 2} - Log.context.set extra - Log.context.should eq(c({a: 1, b: 2})) - end + done.send nil + Log.context.metadata.should eq(m({a: 1})) + done.send nil + end - it "is assignable from a hash with string keys" do - Log.context.set a: 1 - extra = {"b" => 2} - Log.context.set extra - Log.context.should eq(c({a: 1, b: 2})) - end + it "is assignable from a hash with symbol keys" do + Log.context.set a: 1 + extra = {:b => 2} + Log.context.set extra + Log.context.metadata.should eq(m({a: 1, b: 2})) + end - it "is assignable from a named tuple" do - Log.context.set a: 1 - extra = {b: 2} - Log.context.set extra - Log.context.should eq(c({a: 1, b: 2})) - end + it "is assignable from a hash with string keys" do + Log.context.set a: 1 + extra = {"b" => 2} + Log.context.set extra + Log.context.metadata.should eq(m({a: 1, b: 2})) + end - it "can be output as JSON" do - raw_context_data = {name: "James"} - Log.context.set raw_context_data - Log.context.to_json.should eq(raw_context_data.to_json) - end + it "is assignable from a named tuple" do + Log.context.set a: 1 + extra = {b: 2} + Log.context.set extra + Log.context.metadata.should eq(m({a: 1, b: 2})) end end diff --git a/spec/std/log/log_spec.cr b/spec/std/log/log_spec.cr index feffc050d73c..fe0d98637ab8 100644 --- a/spec/std/log/log_spec.cr +++ b/spec/std/log/log_spec.cr @@ -5,6 +5,10 @@ private def s(value : Log::Severity) value end +private def m(value) + Log::Metadata.new(value) +end + describe Log do before_each do Log.context.clear @@ -110,10 +114,10 @@ describe Log do log.info { "info message" } - backend.entries.first.context.should eq(Log::Context.new({a: 1})) + backend.entries.first.context.should eq(Log::Metadata.new({a: 1})) end - it "context can be changed within the block and is restored" do + it "context can be changed within the block, yet it's not restored" do Log.context.set a: 1 backend = Log::MemoryBackend.new @@ -121,7 +125,146 @@ describe Log do log.info { Log.context.set(b: 2); "info message" } - backend.entries.first.context.should eq(Log::Context.new({a: 1, b: 2})) - Log.context.should eq(Log::Context.new({a: 1})) + backend.entries.first.context.should eq(Log::Metadata.new({a: 1, b: 2})) + Log.context.metadata.should eq(Log::Metadata.new({a: 1, b: 2})) + end + + describe "emitter dsl" do + it "can be used with message" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :debug) + + log.info &.emit("info message") + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:info)) + entry.message.should eq("info message") + entry.data.should eq(Log::Metadata.empty) + entry.exception.should be_nil + end + + it "can be used with message and exception" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :debug) + ex = Exception.new "the attached exception" + + log.debug exception: ex, &.emit("debug message") + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:debug)) + entry.message.should eq("debug message") + entry.data.should eq(Log::Metadata.empty) + entry.exception.should eq(ex) + end + + it "can be used with message and metada explicitly" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + + log.notice &.emit("notice message", m({a: 1})) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:notice)) + entry.message.should eq("notice message") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "can be used with message and data via named arguments" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :fatal) + + log.fatal &.emit("fatal message", a: 1) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:fatal)) + entry.message.should eq("fatal message") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "can be used with message and data via named tuple" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :fatal) + + log.fatal &.emit("fatal message", {a: 1}) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:fatal)) + entry.message.should eq("fatal message") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "can be used with exception" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :fatal) + ex = Exception.new "the attached exception" + + log.fatal exception: ex, &.emit("fatal message", a: 1) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:fatal)) + entry.message.should eq("fatal message") + entry.data.should eq(m({a: 1})) + entry.exception.should eq(ex) + end + + it "can be used with data only explicitly" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + + log.notice &.emit(m({a: 1})) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:notice)) + entry.message.should eq("") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "can be used with data only via named arguments" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + + log.notice &.emit(a: 1) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:notice)) + entry.message.should eq("") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "can be used with data only via named tuple" do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + + log.notice &.emit(a: 1) + + entry = backend.entries.first + entry.source.should eq("a") + entry.severity.should eq(s(:notice)) + entry.message.should eq("") + entry.data.should eq(m({a: 1})) + entry.exception.should be_nil + end + + it "validates hash" do + expect_raises(ArgumentError, "Expected hash data, not Int32") do + backend = Log::MemoryBackend.new + log = Log.new("a", backend, :notice) + + log.notice &.emit(m(1)) + end + end end end diff --git a/spec/std/log/metadata_spec.cr b/spec/std/log/metadata_spec.cr new file mode 100644 index 000000000000..4674c9ecef99 --- /dev/null +++ b/spec/std/log/metadata_spec.cr @@ -0,0 +1,69 @@ +require "spec" +require "log" +require "log/json" + +private def m(value) + Log::Metadata.new(value) +end + +describe Log::Metadata do + it "initialize" do + m({a: 1}).should eq(m({"a" => m(1)})) + m({a: 1, b: ["str", true], num: 1i64}).should eq(m({"a" => m(1), "b" => m([m("str"), m(true)]), "num" => m(1i64)})) + m({a: 1f32, b: 1f64}).should eq(m({"a" => m(1f32), "b" => m(1f64)})) + t = Time.local + m({time: t}).should eq(m({"time" => m(t)})) + Log::Metadata.new.should eq(m(NamedTuple.new)) + end + + it "empty" do + Log::Metadata.empty.should eq(Log::Metadata.new) + Log::Metadata.empty.object_id.should_not eq(Log::Metadata.new.object_id) + Log::Metadata.empty.object_id.should eq(Log::Metadata.empty.object_id) + end + + it "immutability" do + context = m({a: 1}) + other = context.as_h + other["a"] = m(2) + + other.should eq({"a" => m(2)}) + context.should eq(m({a: 1})) + end + + it "nested immutability" do + context = m({a: {b: 1}}) + other = context.as_h + other["a"].raw.as(Hash)["b"] = m(2) + + other.should eq({"a" => m({"b" => 2})}) + context.should eq({"a" => m({"b" => 1})}) + end + + it "merge" do + m({a: 1}).merge(m({b: 2})).should eq(m({a: 1, b: 2})) + m({a: 1, b: 3}).merge(m({b: 2})).should eq(m({a: 1, b: 2})) + m({a: 1, b: 3}).merge(m({b: nil})).should eq(m({a: 1, b: nil})) + end + + it "merge against Log::Metadata.empty without creating a new instance" do + c1 = m({a: 1, b: 3}) + c1.merge(Log::Metadata.empty).should be(c1) + Log::Metadata.empty.merge(c1).should be(c1) + end + + it "accessors" do + m(nil).as_nil.should be_nil + + m(1).as_i.should eq(1) + + m("a").as_s.should eq("a") + m(1).as_s?.should be_nil + + m(true).as_bool.should eq(true) + m(false).as_bool.should eq(false) + m(true).as_bool?.should eq(true) + m(false).as_bool?.should eq(false) + m(nil).as_bool?.should be_nil + end +end diff --git a/src/crystal/datum.cr b/src/crystal/datum.cr index e9844fcd08ff..6e95c6bb6de3 100644 --- a/src/crystal/datum.cr +++ b/src/crystal/datum.cr @@ -5,7 +5,7 @@ module Crystal # Raises otherwise. def as_{{short.id}} : {{type}} {% if immutable == true %} - @raw.as({{type}}).dup + @raw.as({{type}}).clone {% else %} @raw.as({{type}}) {% end %} @@ -16,7 +16,7 @@ module Crystal # Returns `nil` otherwise. def as_{{short.id}}? : {{type}}? {% if immutable == true %} - @raw.as?({{type}}).dup + @raw.as?({{type}}).clone {% else %} @raw.as?({{type}}) {% end %} @@ -29,7 +29,7 @@ module Crystal # # * **types**: contains a named tuple of prefixes and datatypes of each leaf # * **hash_key_type** specifies the type used as the key of `Hash` - # * **immutable**: will generate honor immutability of the values via `.dup` + # * **immutable**: will generate honor immutability of the values via `.clone` # * **target_type**: is the type where the macro is invoked (it's a workaround for #9099). # :nodoc: diff --git a/src/log.cr b/src/log.cr index bbbc9f8dc493..7c680a4c32b7 100644 --- a/src/log.cr +++ b/src/log.cr @@ -12,10 +12,17 @@ # Log.info { "Program started" } # ``` # +# Data can be associated with a log entry via the `Log::Emitter` yielded in the logging methods. +# +# ``` +# Log.info &.emit("User logged in", user_id: 42) +# ``` +# # If you want to log an exception, you can indicate it in the `exception:` named argument. # # ``` # Log.warn(exception: e) { "Oh no!" } +# Log.warn exception: e, &.emit("Oh no!", user_id: 42) # ``` # # The block is only evaluated if the current message is to be emitted to some `Log::Backend`. @@ -138,7 +145,7 @@ end require "./log/backend" require "./log/broadcast_backend" require "./log/builder" -require "./log/context" +require "./log/metadata" require "./log/entry" require "./log/main" require "./log/setup" diff --git a/src/log/context.cr b/src/log/context.cr deleted file mode 100644 index 0361f87d594b..000000000000 --- a/src/log/context.cr +++ /dev/null @@ -1,70 +0,0 @@ -# Immutable structured context information for logging. -# -# See `Log.context`, `Log.context=`, `Log::Context#clear`, `Log::Context#set`, `Log.with_context`. -# -# NOTE: If you'd like to format the context as JSON, remember to `require "log/json"`. -class Log::Context - Crystal.datum types: {nil: Nil, bool: Bool, i: Int32, i64: Int64, f: Float32, f64: Float64, s: String, time: Time}, hash_key_type: String, immutable: true, target_type: Log::Context - - # Returns an empty `Log::Context`. - # - # NOTE: Since `Log::Context` is immutable, it's safe to share this instance. - class_getter empty : Log::Context = Log::Context.new - - # Creates an empty `Log::Context`. - def initialize - @raw = Hash(String, Context).new - end - - # Creates `Log::Context` from the given *tuple*. - def initialize(tuple : NamedTuple) - @raw = raw = Hash(String, Context).new - tuple.each do |key, value| - raw[key.to_s] = to_context(value) - end - end - - # Creates `Log::Context` from the given *hash*. - def initialize(hash : Hash(String, V)) forall V - @raw = raw = Hash(String, Context).new - hash.each do |key, value| - raw[key] = to_context(value) - end - end - - # Creates `Log::Context` from the given *hash*. - def initialize(hash : Hash(Symbol, V)) forall V - @raw = raw = Hash(String, Context).new - hash.each do |key, value| - raw[key.to_s] = to_context(value) - end - end - - # :nodoc: - def initialize(ary : Array) - @raw = ary.map { |e| to_context(e) } - end - - # Returns a new `Log::Context` with the keys and values of this context and *other* combined. - # A value in *other* takes precedence over the one in this context. - def merge(other : Context) - return other if self.object_id == @@empty.object_id - return self if other.object_id == @@empty.object_id - Context.new(self.as_h.merge(other.as_h).clone) - end - - private def to_context(value) - value.is_a?(Context) ? value : Context.new(value) - end -end - -class Fiber - # :nodoc: - getter logging_context : Log::Context { Log::Context.empty } - - # :nodoc: - def logging_context=(value : Log::Context) - raise ArgumentError.new "Expected hash context, not #{value.raw.class}" unless value.as_h? - @logging_context = value - end -end diff --git a/src/log/entry.cr b/src/log/entry.cr index 7ded5220fcf5..4603c764baed 100644 --- a/src/log/entry.cr +++ b/src/log/entry.cr @@ -38,9 +38,11 @@ struct Log::Entry getter severity : Severity getter message : String getter timestamp = Time.local - getter context = Log.context + getter context : Metadata = Log.context.metadata + getter data : Metadata getter exception : Exception? - def initialize(@source : String, @severity : Severity, @message : String, @exception : Exception?) + def initialize(@source : String, @severity : Severity, @message : String, @data : Log::Metadata, @exception : Exception?) + raise ArgumentError.new "Expected hash data, not #{data.raw.class}" unless data.as_h? end end diff --git a/src/log/io_backend.cr b/src/log/io_backend.cr index 1e5020353a34..a4b98f2be9a1 100644 --- a/src/log/io_backend.cr +++ b/src/log/io_backend.cr @@ -35,6 +35,9 @@ class Log::IOBackend < Log::Backend io << " #" << Process.pid << "] " label.rjust(6, io) io << " -- " << @progname << ":" << entry.source << ": " << entry.message + if entry.data.size > 0 + io << " -- " << entry.data + end if entry.context.size > 0 io << " -- " << entry.context end diff --git a/src/log/json.cr b/src/log/json.cr index 3d2045ea9306..2e6f43710767 100644 --- a/src/log/json.cr +++ b/src/log/json.cr @@ -1,7 +1,7 @@ require "json" -class Log::Context - # Returns `Log::Context` as JSON value. +class Log::Metadata + # Returns `Log::Metadata` as JSON value. # # NOTE: `require "log/json"` is required to opt-in to this feature. # diff --git a/src/log/log.cr b/src/log/log.cr index 6681d3fe093d..e2573f56ff3a 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -48,10 +48,17 @@ class Log return unless backend = @backend severity = Severity.new({{severity}}) return unless level <= severity - entry = Log.with_context do - message = yield.to_s - Entry.new @source, severity, message, exception - end + + dsl = Emitter.new(@source, severity, exception) + result = yield dsl + entry = + case result + when Entry + result + else + dsl.emit(result.to_s) + end + backend.write entry end {% end %} diff --git a/src/log/main.cr b/src/log/main.cr index 14937c17a1b8..fa5ae4883cb0 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -38,8 +38,8 @@ class Log {% for method in %i(trace debug info notice warn error fatal) %} # See `Log#{{method.id}}`. def self.{{method.id}}(*, exception : Exception? = nil) - Top.{{method.id}}(exception: exception) do - yield + Top.{{method.id}}(exception: exception) do |dsl| + yield dsl end end {% end %} @@ -53,21 +53,29 @@ class Log # Returns the current fiber logging context. def self.context : Log::Context - Fiber.current.logging_context + Log::Context.new(Fiber.current.logging_context) end # Sets the current fiber logging context. - def self.context=(value : Log::Context) + def self.context=(value : Log::Metadata) Fiber.current.logging_context = value end + # :ditto: + def self.context=(value : Log::Context) + # NOTE: There is a need for `Metadata` and `Context` setters in + # becuase `Log.context` returns a `Log::Context` for allowing DSL like `Log.context.set(a: 1)` + # but if the metadata is built manually the construct `Log.context = metadata` will be used. + Log.context = value.metadata + end + # Returns the current fiber logging context. def context : Log::Context Log.context end # Sets the current fiber logging context. - def context=(value : Log::Context) + def context=(value : Log::Metadata | Log::Context) Log.context = value end @@ -98,7 +106,12 @@ class Log end end - class Context + struct Context + getter metadata : Metadata + + def initialize(@metadata : Metadata) + end + # Clears the current `Fiber` logging context. # # ``` @@ -106,7 +119,7 @@ class Log # Log.info { "message with empty context" } # ``` def clear - Fiber.current.logging_context = Log::Context.empty + Fiber.current.logging_context = @metadata = Log::Metadata.empty end # Extends the current `Fiber` logging context. @@ -123,27 +136,52 @@ class Log # Log.info { %q(message with {"a" => 1, "b" => 2, "c" => 3 } context) } # ``` def set(**kwargs) - extend_fiber_context(Fiber.current, Log::Context.new(kwargs)) + extend_fiber_context(Fiber.current, Log::Metadata.build(kwargs)) end # :ditto: - def set(values : Hash(String, V)) forall V - extend_fiber_context(Fiber.current, Log::Context.new(values)) + def set(values) + extend_fiber_context(Fiber.current, Log::Metadata.build(values)) end - # :ditto: - def set(values : Hash(Symbol, V)) forall V - extend_fiber_context(Fiber.current, Log::Context.new(values)) + private def extend_fiber_context(fiber : Fiber, values : Metadata) + context = fiber.logging_context + fiber.logging_context = @metadata = context.merge(values) end + end - # :ditto: - def set(values : NamedTuple) - extend_fiber_context(Fiber.current, Log::Context.new(values)) + # Helper DSL module for emitting log entries with data. + struct Emitter + # :nodoc: + def initialize(@source : String, @severity : Severity, @exception : Exception?) end - private def extend_fiber_context(fiber : Fiber, values : Context) - context = fiber.logging_context - fiber.logging_context = context.merge(values) + # Emits a logs entry with a message, and data attached to + # + # ``` + # Log.info &.emit("Program started") # No data, same as Log.info { "Program started" } + # Log.info &.emit("User logged in", user_id: 42) # With entry data + # Log.info &.emit(action: "Logged in", user_id: 42) # Empty string message, only data + # Log.error exception: ex, &.emit("Oopps", account: {id: 42}) # With data and exception + # ``` + def emit(message : String) : Entry + emit(message, Metadata.empty) + end + + def emit(message : String, **kwargs) : Entry + emit(message, kwargs) + end + + def emit(message : String, data : Metadata | Hash | NamedTuple) : Entry + Entry.new(@source, @severity, message, Metadata.build(data), @exception) + end + + def emit(**kwargs) : Entry + emit(kwargs) + end + + def emit(data : Metadata | Hash | NamedTuple) : Entry + emit("", Metadata.build(data)) end end end diff --git a/src/log/metadata.cr b/src/log/metadata.cr new file mode 100644 index 000000000000..8d7c26f645f2 --- /dev/null +++ b/src/log/metadata.cr @@ -0,0 +1,96 @@ +# Immutable structured metadata information for logging. +# +# See `Log.context`, `Log.context=`, `Log::Context#clear`, `Log::Context#set`, `Log.with_context`, and `Log::Emitter`. +# +# NOTE: If you'd like to format the context as JSON, remember to `require "log/json"`. +class Log::Metadata + Crystal.datum types: {nil: Nil, bool: Bool, i: Int32, i64: Int64, f: Float32, f64: Float64, s: String, time: Time}, hash_key_type: String, immutable: true, target_type: Log::Metadata + + # Returns an empty `Log::Metadata`. + # + # NOTE: Since `Log::Metadata` is immutable, it's safe to share this instance. + class_getter empty : Log::Metadata = Log::Metadata.new + + # Creates an empty `Log::Metadata`. + def initialize + @raw = Hash(String, Metadata).new + end + + # Creates `Log::Metadata` from the given *tuple*. + def initialize(tuple : NamedTuple) + @raw = raw = Hash(String, Metadata).new + tuple.each do |key, value| + raw[key.to_s] = to_metadata(value) + end + end + + # Creates `Log::Metadata` from the given *hash*. + def initialize(hash : Hash(String, V)) forall V + @raw = raw = Hash(String, Metadata).new + hash.each do |key, value| + raw[key] = to_metadata(value) + end + end + + # Creates `Log::Metadata` from the given *hash*. + def initialize(hash : Hash(Symbol, V)) forall V + @raw = raw = Hash(String, Metadata).new + hash.each do |key, value| + raw[key.to_s] = to_metadata(value) + end + end + + # :nodoc: + def initialize(ary : Array) + @raw = ary.map { |e| to_metadata(e) } + end + + # Returns a new `Log::Metadata` with the keys and values of this context and *other* combined. + # A value in *other* takes precedence over the one in this context. + def merge(other : Metadata) + return other if self.object_id == @@empty.object_id + return self if other.object_id == @@empty.object_id + Metadata.new(self.as_h.merge(other.as_h).clone) + end + + private def to_metadata(value) + value.is_a?(Metadata) ? value : Metadata.new(value) + end + + # Returns a `Metadata` with the information of the argument. + # Used to handle `Log::Context#set` and `Log#Emitter.emit` overloads. + def self.build(value : Nil) + Metadata.empty + end + + # :ditto: + def self.build(value : NamedTuple) + Metadata.new(value) + end + + # :ditto: + def self.build(value : Hash(String, V)) forall V + Metadata.new(value) + end + + # :ditto: + def self.build(value : Hash(Symbol, V)) forall V + Metadata.new(value) + end + + # :ditto: + def self.build(value : Metadata) + value + end +end + +class Fiber + # :nodoc: + getter logging_context : Log::Metadata { Log::Metadata.empty } + + # :nodoc: + def logging_context=(value : Log::Metadata) + raise ArgumentError.new "Expected hash context, not #{value.raw.class}" unless value.as_h? + @logging_context = value + end +end From 35c1f41f05144d9645c7bba83cd1b1deaea68fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 11 May 2020 19:30:26 +0200 Subject: [PATCH 024/263] Add link to GitHub repo to README.md (#9163) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4c073d5b15bd..b93265724db2 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ Questions or suggestions? Ask on the [Crystal Forum](https://forum.crystal-lang. Contributing ------------ +The Crystal repository is hosted at [crystal-lang/crystal](https://github.com/crystal-lang/crystal) on GitHub. + Read the general [Contributing guide](https://github.com/crystal-lang/crystal/blob/master/CONTRIBUTING.md), and then: 1. Fork it () From 0f3f29c834efe61e9032e2caaee7fc1a8bbaeaad Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 13 May 2020 09:38:41 -0300 Subject: [PATCH 025/263] Refactor Log::Formatter (#9211) * Refactor Log::Formatter * Add docs to log/format.cr * Some renames in log/format.cr * Add Log::StaticFormatter#pid * Apply log formatter renames on spec --- spec/std/log/format_spec.cr | 88 ++++++++++++++ spec/std/log/io_backend_spec.cr | 13 +-- src/log.cr | 1 + src/log/backend.cr | 2 - src/log/format.cr | 198 ++++++++++++++++++++++++++++++++ src/log/io_backend.cr | 33 +----- 6 files changed, 297 insertions(+), 38 deletions(-) create mode 100644 spec/std/log/format_spec.cr create mode 100644 src/log/format.cr diff --git a/spec/std/log/format_spec.cr b/spec/std/log/format_spec.cr new file mode 100644 index 000000000000..af9045f35195 --- /dev/null +++ b/spec/std/log/format_spec.cr @@ -0,0 +1,88 @@ +require "spec" +require "log" + +class Log + describe ShortFormat do + it "formats an entry" do + entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + io = IO::Memory.new + ShortFormat.format(entry, io) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message$/) + end + + it "hides the source if empty" do + entry = Entry.new("", :info, "message", Log::Metadata.empty, nil) + io = IO::Memory.new + ShortFormat.format(entry, io) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - message$/) + end + + it "shows the context data" do + entry = Log.with_context do + Log.context.set a: 1, b: 2 + Entry.new("source", :info, "message", Log::Metadata.empty, nil) + end + io = IO::Memory.new + ShortFormat.format(entry, io) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- {"a" => 1, "b" => 2}$/) + end + + it "shows context and entry data" do + entry = Log.with_context do + Log.context.set a: 1, b: 2 + Entry.new("source", :info, "message", Log::Metadata.new({c: 3, d: 4}), nil) + end + io = IO::Memory.new + ShortFormat.format(entry, io) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- {"c" => 3, "d" => 4} -- {"a" => 1, "b" => 2}$/) + end + + it "appends the exception" do + exception = expect_raises(Exception) { raise "foo" } + entry = Entry.new("source", :error, "message", Log::Metadata.empty, exception) + io = IO::Memory.new + ShortFormat.format(entry, io) + io.rewind + io.gets.should match(/^[\d\-.:TZ]+\s* ERROR - source: message$/) + io.gets_to_end.should eq(exception.inspect_with_backtrace) + end + end + + describe ProcFormatter do + it "formats" do + entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + io = IO::Memory.new + formatter = Formatter.new do |entry, io| + io << "[" << entry.message << "]" + end + formatter.format(entry, io) + io.to_s.should eq("[message]") + end + end + + define_formatter TestFormatter, "#{severity} #{source(before: '[', after: "] ")}#{progname} #{message}" \ + "#{context(before: " (", after: ')')}#{exception}" + Log.progname = "test" + + describe TestFormatter do + it "formats" do + exception = expect_raises(Exception) { raise "foo" } + entry = Log.with_context do + Log.context.set a: 1, b: 2 + Entry.new("source", :info, "message", Log::Metadata.empty, nil) + end + io = IO::Memory.new + TestFormatter.format(entry, io) + io.puts + TestFormatter.format(Entry.new("", :info, "message", Log::Metadata.empty, nil), io) + io.puts + TestFormatter.format(Entry.new("source", :error, "Oh, no", Log::Metadata.empty, exception), io) + io.rewind + + io.gets.should eq(" INFO [source] test message ({\"a\" => 1, \"b\" => 2})") + io.gets.should eq(" INFO test message") + io.gets.should eq(" ERROR [source] test Oh, no") + io.gets_to_end.should eq(exception.inspect_with_backtrace) + end + end +end diff --git a/spec/std/log/io_backend_spec.cr b/spec/std/log/io_backend_spec.cr index 902d4cedee5a..051df3cc7d5f 100644 --- a/spec/std/log/io_backend_spec.cr +++ b/spec/std/log/io_backend_spec.cr @@ -5,10 +5,9 @@ private def s(value : Log::Severity) value end -private def io_logger(*, stdout : IO, config = nil, source : String = "", progname : String? = nil) +private def io_logger(*, stdout : IO, config = nil, source : String = "") builder = Log::Builder.new backend = Log::IOBackend.new - backend.progname = progname if progname backend.io = stdout builder.bind("*", s(:info), backend) builder.for(source) @@ -59,10 +58,10 @@ describe Log::IOBackend do it "formats message" do IO.pipe do |r, w| - logger = io_logger(progname: "the-program", stdout: w, source: "db.pool") + logger = io_logger(stdout: w, source: "db.pool") logger.warn { "message" } - r.gets(chomp: false).should match(/W, \[.+? #\d+\] WARNING -- the-program:db.pool: message\n/) + r.gets(chomp: false).should match(/.+? WARNING - db.pool: message\n/) end end @@ -89,12 +88,12 @@ describe Log::IOBackend do it "yields message" do IO.pipe do |r, w| - logger = io_logger(stdout: w, progname: "prog", source: "db") + logger = io_logger(stdout: w, source: "db") logger.error { "message" } logger.fatal { "another message" } - r.gets(chomp: false).should match(/ERROR -- prog:db: message\n/) - r.gets(chomp: false).should match(/FATAL -- prog:db: another message\n/) + r.gets(chomp: false).should match(/ERROR - db: message\n/) + r.gets(chomp: false).should match(/FATAL - db: another message\n/) end end end diff --git a/src/log.cr b/src/log.cr index 7c680a4c32b7..c4b7a6feaf64 100644 --- a/src/log.cr +++ b/src/log.cr @@ -147,6 +147,7 @@ require "./log/broadcast_backend" require "./log/builder" require "./log/metadata" require "./log/entry" +require "./log/format" require "./log/main" require "./log/setup" require "./log/log" diff --git a/src/log/backend.cr b/src/log/backend.cr index 5fd394596f4a..cdecb1749429 100644 --- a/src/log/backend.cr +++ b/src/log/backend.cr @@ -9,5 +9,3 @@ abstract class Log::Backend def close end end - -alias Log::Formatter = Entry, IO -> diff --git a/src/log/format.cr b/src/log/format.cr new file mode 100644 index 000000000000..37facfb47e2e --- /dev/null +++ b/src/log/format.cr @@ -0,0 +1,198 @@ +class Log + # The program name used for log entries + # + # Defaults to the executable name + class_property progname = File.basename(PROGRAM_NAME) + + # The current process PID + protected class_getter pid : String = Process.pid.to_s + + # Base interface to convert log entries and write them to an `IO` + module Formatter + # Writes a `Log::Entry` through an `IO` + abstract def format(entry : Log::Entry, io : IO) + + # Creates an instance of a `Log::Formatter` that calls + # the specified `Proc` for every entry + def self.new(&proc : (Log::Entry, IO) ->) + ProcFormatter.new proc + end + end + + # :nodoc: + private struct ProcFormatter + include Formatter + + def initialize(@proc : (Log::Entry, IO) ->) + end + + def format(entry : Log::Entry, io : IO) + @proc.call(entry, io) + end + end + + # Base implementation of `Log::Formatter` to convert + # log entries into text representation + # + # This can be used to create efficient formatters: + # ``` + # struct MyFormat < Log::StaticFormat + # def run + # string "- " + # severity + # string ": " + # message + # end + # end + # + # Log.setup(:info, Log::IOBackend.new(formatter: MyFormat)) + # Log.info { "Hello" } # => - INFO: Hello + # Log.error { "Oh, no!" } # => - ERROR: Oh, no! + # ``` + # + # There is also a helper macro to generate these formatters. Here's + # an example that generates the same result: + # ``` + # Log.define_formatter MyFormat, "- #{severity}: #{message}" + # ``` + abstract struct StaticFormatter + extend Formatter + + def initialize(@entry : Log::Entry, @io : IO) + end + + # Write the entry timestamp in RFC3339 format + def timestamp + @entry.timestamp.to_rfc3339(@io) + end + + # Write a fixed string + def string(str) + @io << str + end + + # Write the message of the entry + def message + @io << @entry.message + end + + # Write the severity + # + # This writes the severity in uppercase and left padded + # with enough space so all the severities fit + def severity + @entry.severity.label.rjust(7, @io) + end + + # Write the source for non-root entries + # + # It doesn't write any output for entries generated from the root logger. + # Parameters `before` and `after` can be provided to be written around + # the value. + # ``` + # source(before: '[', after: ']') # => [http.server] + # ``` + def source(*, before = nil, after = nil) + if @entry.source.size > 0 + @io << before << @entry.source << after + end + end + + # Write all the values from the entry data + # + # It doesn't write any output if the entry data is empty. + # Parameters `before` and `after` can be provided to be written around + # the value. + def data(*, before = nil, after = nil) + if @entry.data.size > 0 + @io << before << @entry.data << after + end + end + + # Write all the values from the context + # + # It doesn't write any output if the context is empty. + # Parameters `before` and `after` can be provided to be written around + # the value. + def context(*, before = nil, after = nil) + if @entry.context.size > 0 + @io << before << @entry.context << after + end + end + + # Write the exception, including backtrace + # + # It doesn't write any output unless there is an exception in the entry. + # Parameters `before` and `after` can be provided to be written around + # the value. `before` defaults to `'\n'` so the exception is written + # on a separate line + def exception(*, before = '\n', after = nil) + if ex = @entry.exception + @io << before + ex.inspect_with_backtrace(@io) + @io << after + end + end + + # Write the program name. See `Log.progname`. + def progname + @io << Log.progname + end + + # Write the current process identifier + def pid(*, before = '#', after = nil) + @io << before << Log.pid << after + end + + # Write the `Log::Entry` to the `IO` using this pattern + def self.format(entry, io) + new(entry, io).run + end + + # Subclasses must implement this method to define the output pattern + abstract def run + end + + # Generate subclasses of `Log::StaticFormatter` from a string with interpolations + # + # Example: + # ``` + # Log.define_formatter MyFormat, "- #{severity}: #{message}" + # ``` + # See `Log::StaticFormatter` for the available methods that can + # be called within the interpolations. + macro define_formatter(name, pattern) + struct {{name}} < ::Log::StaticFormatter + def run + {% for part in pattern.expressions %} + {% if part.is_a?(StringLiteral) %} + string {{ part }} + {% else %} + {{ part }} + {% end %} + {% end %} + end + end + end +end + +# Default short format +# +# It writes log entries with the following format: +# ``` +# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything +# ``` +# +# When the entries have context data it's also written to the output: +# ``` +# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything -- {"data" => 123} +# ``` +# +# Exceptions are written in a separate line: +# ``` +# 2020-05-07T17:40:07.994508000Z ERROR - my.source: Something failed +# Oh, no (Exception) +# from ... +# ``` +Log.define_formatter Log::ShortFormat, "#{timestamp} #{severity} - #{source(after: ": ")}#{message}" \ + "#{data(before: " -- ")}#{context(before: " -- ")}#{exception}" diff --git a/src/log/io_backend.cr b/src/log/io_backend.cr index a4b98f2be9a1..ac9189db1b44 100644 --- a/src/log/io_backend.cr +++ b/src/log/io_backend.cr @@ -1,12 +1,10 @@ # A `Log::Backend` that emits to an `IO` (defaults to STDOUT). class Log::IOBackend < Log::Backend property io : IO - property progname : String - property formatter : Formatter? + property formatter : Formatter - def initialize(@io = STDOUT, @formatter = nil) + def initialize(@io = STDOUT, @formatter : Formatter = ShortFormat) @mutex = Mutex.new(:unchecked) - @progname = File.basename(PROGRAM_NAME) end def write(entry : Entry) @@ -18,31 +16,8 @@ class Log::IOBackend < Log::Backend end # Emits the *entry* to the given *io*. - # It will use the `#formatter` if defined, otherwise will call `#default_format`. + # It uses the `#formatter` to convert. def format(entry : Entry) - if formatter = @formatter - formatter.call(entry, io) - else - default_format(entry) - end - end - - # Emits the *entry* to the given *io*. - def default_format(entry : Entry) - label = entry.severity.label - io << label[0] << ", [" - entry.timestamp.to_rfc3339(io) - io << " #" << Process.pid << "] " - label.rjust(6, io) - io << " -- " << @progname << ":" << entry.source << ": " << entry.message - if entry.data.size > 0 - io << " -- " << entry.data - end - if entry.context.size > 0 - io << " -- " << entry.context - end - if ex = entry.exception - io << " -- " << ex.class << ": " << ex - end + @formatter.format(entry, io) end end From 433ca8ed9a4d404fab8f175eea24ff979b5c99a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 13 May 2020 14:40:06 +0200 Subject: [PATCH 026/263] Refactor spec hooks (#9090) * Refactor spec hooks * Refactor all spec hooks to work in the root context * Deprecate hook methods in Spec namespace * Improve documentation of spec hooks * Add spec for spec hooks * Reset DSL methods to raise on top level * crystal tool format --- spec/std/spec/hooks_spec.cr | 145 +++++++++++++++++++++ src/spec/context.cr | 253 +++++++++++++++--------------------- src/spec/dsl.cr | 58 ++++----- src/spec/example.cr | 4 - src/spec/methods.cr | 189 ++++++++++++++++++++++----- 5 files changed, 437 insertions(+), 212 deletions(-) create mode 100644 spec/std/spec/hooks_spec.cr diff --git a/spec/std/spec/hooks_spec.cr b/spec/std/spec/hooks_spec.cr new file mode 100644 index 000000000000..4dac431b7e9b --- /dev/null +++ b/spec/std/spec/hooks_spec.cr @@ -0,0 +1,145 @@ +require "../../spec_helper" + +describe Spec do + describe "hooks" do + it "runs in correct order" do + run(<<-CR).to_string.lines[..-5].should eq <<-OUT.lines + require "prelude" + require "spec" + + begin + before_all {} + rescue exc + puts exc.message + end + begin + before_each {} + rescue exc + puts exc.message + end + begin + after_all {} + rescue exc + puts exc.message + end + begin + after_each {} + rescue exc + puts exc.message + end + begin + around_all {} + rescue exc + puts exc.message + end + begin + around_each {} + rescue exc + puts exc.message + end + + Spec.before_suite { puts "Spec:before_suite" } + Spec.before_each { puts "Spec:before_each" } + Spec.after_suite { puts "Spec:after_all" } + Spec.after_each { puts "Spec:after_each" } + Spec.around_each do |example| + puts "Spec:around_each:before" + example.run + puts "Spec:around_each:after" + end + + describe "foo" do + before_all { puts "foo:before_all" } + before_each { puts "foo:before_each" } + after_all { puts "foo:after_all" } + after_each { puts "foo:after_each" } + around_all do |example| + puts "foo:around_all:before" + example.run + puts "foo:around_all:after" + end + around_each do |example| + puts "foo:around_each:before" + example.run + puts "foo:around_each:after" + end + + it {} + it {} + + describe "foofoo" do + it {} + end + end + + describe "bar" do + before_all { puts "bar:before_all" } + before_each { puts "bar:before_each" } + after_all { puts "bar:after_all" } + after_each { puts "bar:after_each" } + around_all do |example| + puts "bar:around_all:before" + example.run + puts "bar:around_all:after" + end + around_each do |example| + puts "bar:around_each:before" + example.run + puts "bar:around_each:after" + end + + it {} + end + CR + Can't call `before_all` outside of a describe/context + Can't call `before_each` outside of a describe/context + Can't call `after_all` outside of a describe/context + Can't call `after_each` outside of a describe/context + Can't call `around_all` outside of a describe/context + Can't call `around_each` outside of a describe/context + Spec:before_suite + foo:around_all:before + foo:before_all + Spec:around_each:before + foo:around_each:before + Spec:before_each + foo:before_each + .foo:after_each + Spec:after_each + foo:around_each:after + Spec:around_each:after + Spec:around_each:before + foo:around_each:before + Spec:before_each + foo:before_each + .foo:after_each + Spec:after_each + foo:around_each:after + Spec:around_each:after + Spec:around_each:before + foo:around_each:before + Spec:before_each + foo:before_each + .foo:after_each + Spec:after_each + foo:around_each:after + Spec:around_each:after + foo:after_all + foo:around_all:after + bar:around_all:before + bar:before_all + Spec:around_each:before + bar:around_each:before + Spec:before_each + bar:before_each + .bar:after_each + Spec:after_each + bar:around_each:after + Spec:around_each:after + bar:after_all + bar:around_all:after + Spec:after_all + OUT + end + end +end diff --git a/src/spec/context.cr b/src/spec/context.cr index 18ab10b20d8e..8ba4635d5349 100644 --- a/src/spec/context.cr +++ b/src/spec/context.cr @@ -12,6 +12,100 @@ module Spec end children.shuffle!(randomizer) end + + protected def internal_run + run_before_all_hooks + children.each &.run + run_after_all_hooks + end + + protected def before_each(&block) + (@before_each ||= [] of ->) << block + end + + protected def run_before_each_hooks + @before_each.try &.each &.call + end + + protected def after_each(&block) + (@after_each ||= [] of ->) << block + end + + protected def run_after_each_hooks + @after_each.try &.reverse_each &.call + end + + protected def before_all(&block) + (@before_all ||= [] of ->) << block + end + + protected def run_before_all_hooks + @before_all.try &.each &.call + end + + protected def after_all(&block) + (@after_all ||= [] of ->) << block + end + + protected def run_after_all_hooks + @after_all.try &.reverse_each &.call + end + + protected def around_each(&block : Example::Procsy ->) + (@around_each ||= [] of Example::Procsy ->) << block + end + + protected def run_around_each_hooks(procsy : Example::Procsy) : Bool + internal_run_around_each_hooks(procsy) + end + + protected def internal_run_around_each_hooks(procsy : Example::Procsy) : Bool + around_each = @around_each + return false unless around_each + + run_around_each_hook(around_each, procsy, 0) + true + end + + protected def run_around_each_hook(around_each, procsy, index) : Nil + around_each[index].call( + if index == around_each.size - 1 + # If we don't have any more hooks after this one, call the procsy + procsy + else + # Otherwise, create a procsy that will invoke the next hook + Example::Procsy.new(procsy.example) do + run_around_each_hook(around_each, procsy, index + 1) + end + end + ) + end + + protected def around_all(&block : ExampleGroup::Procsy ->) + (@around_all ||= [] of ExampleGroup::Procsy ->) << block + end + + protected def run_around_all_hooks(procsy : ExampleGroup::Procsy) : Bool + around_all = @around_all + return false unless around_all + + run_around_all_hook(around_all, procsy, 0) + true + end + + protected def run_around_all_hook(around_all, procsy, index) : Nil + around_all[index].call( + if index == around_all.size - 1 + # If we don't have any more hooks after this one, call the procsy + procsy + else + # Otherwise, create a procsy that will invoke the next hook + ExampleGroup::Procsy.new(procsy.example_group) do + run_around_all_hook(around_all, procsy, index + 1) + end + end + ) + end end # :nodoc: @@ -28,12 +122,17 @@ module Spec RootContext.instance end + # :nodoc: + def self.current_context : Context + RootContext.current_context + end + # :nodoc: # # The root context is the main interface that the spec DSL interacts with. class RootContext < Context class_getter instance = RootContext.new - @@current_context : Context = @@instance + class_getter current_context : Context = @@instance def initialize @results = { @@ -45,7 +144,7 @@ module Spec end def run - children.each &.run + internal_run end def report(kind, full_description, file, line, elapsed = nil, ex = nil) @@ -206,68 +305,8 @@ module Spec end end - def before_each(&block) - if @@current_context == self - raise "Can't call `before_each` outside of a describe/context" - end - - @@current_context.before_each(&block) - end - - def run_before_each_hooks - # Nothing - end - - def after_each(&block) - if @@current_context == self - raise "Can't call `after_each` outside of a describe/context" - end - - @@current_context.after_each(&block) - end - - def run_after_each_hooks - # Nothing - end - - def before_all(&block) - if @@current_context == self - raise "Can't call `before_all` outside of a describe/context" - end - - @@current_context.before_all(&block) - end - - def after_all(&block) - if @@current_context == self - raise "Can't call `after_all` outside of a describe/context" - end - - @@current_context.after_all(&block) - end - - def around_each(&block : Example::Procsy ->) - if @@current_context == self - raise "Can't call `around_each` outside of a describe/context" - end - - @@current_context.around_each(&block) - end - - def run_around_each_hooks(procsy : Example::Procsy) : Bool - false - end - - def around_all(&block : ExampleGroup::Procsy ->) - if @@current_context == self - raise "Can't call `around_all` outside of a describe/context" - end - - @@current_context.around_all(&block) - end - - def run_around_all_hooks(procsy : ExampleGroup::Procsy) : Bool - false + protected def around_all(&block : ExampleGroup::Procsy ->) + raise "Can't call `around_all` outside of a describe/context" end end @@ -291,54 +330,20 @@ module Spec Spec.formatters.each(&.pop) end - protected def internal_run - run_before_all_hooks - children.each &.run - run_after_all_hooks - end - protected def report(kind, description, file, line, elapsed = nil, ex = nil) parent.report kind, "#{@description} #{description}", file, line, elapsed, ex end - protected def before_each(&block) - (@before_each ||= [] of ->) << block - end - protected def run_before_each_hooks @parent.run_before_each_hooks - @before_each.try &.each &.call - end - - protected def after_each(&block) - (@after_each ||= [] of ->) << block + super end protected def run_after_each_hooks - @after_each.try &.reverse_each &.call + super @parent.run_after_each_hooks end - protected def before_all(&block) - (@before_all ||= [] of ->) << block - end - - protected def run_before_all_hooks - @before_all.try &.each &.call - end - - protected def after_all(&block) - (@after_all ||= [] of ->) << block - end - - protected def run_after_all_hooks - @after_all.try &.reverse_each &.call - end - - protected def around_each(&block : Example::Procsy ->) - (@around_each ||= [] of Example::Procsy ->) << block - end - protected def run_around_each_hooks(procsy : Example::Procsy) : Bool ran = @parent.run_around_each_hooks(Example::Procsy.new(procsy.example) do if @around_each @@ -353,54 +358,6 @@ module Spec end) ran || internal_run_around_each_hooks(procsy) end - - protected def internal_run_around_each_hooks(procsy : Example::Procsy) : Bool - around_each = @around_each - return false unless around_each - - run_around_each_hook(around_each, procsy, 0) - true - end - - protected def run_around_each_hook(around_each, procsy, index) : Nil - around_each[index].call( - if index == around_each.size - 1 - # If we don't have any more hooks after this one, call the procsy - procsy - else - # Otherwise, create a procsy that will invoke the next hook - Example::Procsy.new(procsy.example) do - run_around_each_hook(around_each, procsy, index + 1) - end - end - ) - end - - protected def around_all(&block : ExampleGroup::Procsy ->) - (@around_all ||= [] of ExampleGroup::Procsy ->) << block - end - - protected def run_around_all_hooks(procsy : ExampleGroup::Procsy) : Bool - around_all = @around_all - return false unless around_all - - run_around_all_hook(around_all, procsy, 0) - true - end - - protected def run_around_all_hook(around_all, procsy, index) : Nil - around_all[index].call( - if index == around_all.size - 1 - # If we don't have any more hooks after this one, call the procsy - procsy - else - # Otherwise, create a procsy that will invoke the next hook - ExampleGroup::Procsy.new(procsy.example_group) do - run_around_all_hook(around_all, procsy, index + 1) - end - end - ) - end end end diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index bbdf8666bccc..c6524e4edf93 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -162,7 +162,7 @@ module Spec class_property? focus = false # Instructs the spec runner to execute the given block - # before each spec, regardless of where this method is invoked. + # before each spec in the spec suite. # # If multiple blocks are registered they run in the order # that they are given. @@ -176,12 +176,11 @@ module Spec # # will print, just before each spec, 1 and then 2. def self.before_each(&block) - before_each = @@before_each ||= [] of -> - before_each << block + root_context.before_each(&block) end # Instructs the spec runner to execute the given block - # after each spec, regardless of where this method is invoked. + # after each spec spec in the spec suite. # # If multiple blocks are registered they run in the reversed # order that they are given. @@ -195,8 +194,7 @@ module Spec # # will print, just after each spec, 2 and then 1. def self.after_each(&block) - after_each = @@after_each ||= [] of -> - after_each << block + root_context.after_each(&block) end # Instructs the spec runner to execute the given block @@ -214,8 +212,7 @@ module Spec # # will print, just before the spec suite starts, 1 and then 2. def self.before_suite(&block) - before_suite = @@before_suite ||= [] of -> - before_suite << block + root_context.before_all(&block) end # Instructs the spec runner to execute the given block @@ -233,28 +230,31 @@ module Spec # # will print, just after the spec suite ends, 2 and then 1. def self.after_suite(&block) - after_suite = @@after_suite ||= [] of -> - after_suite << block + root_context.after_all(&block) end - # :nodoc: - def self.run_before_each_hooks - @@before_each.try &.each &.call - end - - # :nodoc: - def self.run_after_each_hooks - @@after_each.try &.reverse_each &.call - end - - # :nodoc: - def self.run_before_suite_hooks - @@before_suite.try &.each &.call - end - - # :nodoc: - def self.run_after_suite_hooks - @@after_suite.try &.reverse_each &.call + # Instructs the spec runner to execute the given block when each spec in the + # spec suite runs. + # + # The block must call `run` on the given `Example::Procsy` object. + # + # If multiple blocks are registered they run in the reversed + # order that they are given. + # + # ``` + # require "spec" + # + # Spec.around_each do |example| + # puts "runs before each sample" + # example.run + # puts "runs after each sample" + # end + # + # it { } + # it { } + # ``` + def self.around_each(&block : Example::Procsy ->) + root_context.around_each(&block) end @@start_time : Time::Span? = nil @@ -267,9 +267,7 @@ module Spec log_setup maybe_randomize run_filters - run_before_suite_hooks root_context.run - run_after_suite_hooks ensure finish_run end diff --git a/src/spec/example.cr b/src/spec/example.cr index 199d20be2671..efd639604679 100644 --- a/src/spec/example.cr +++ b/src/spec/example.cr @@ -29,13 +29,9 @@ module Spec non_nil_block = block start = Time.monotonic - Spec.run_before_each_hooks - ran = @parent.run_around_each_hooks(Example::Procsy.new(self) { internal_run(start, non_nil_block) }) ran || internal_run(start, non_nil_block) - Spec.run_after_each_hooks - # We do this to give a chance for signals (like CTRL+C) to be handled, # which currently are only handled when there's a fiber switch # (IO stuff, sleep, etc.). Without it the user might wait more than needed diff --git a/src/spec/methods.cr b/src/spec/methods.cr index c86f0f257fae..019c90f34c41 100644 --- a/src/spec/methods.cr +++ b/src/spec/methods.cr @@ -76,78 +76,207 @@ module Spec::Methods raise Spec::AssertionFailed.new(msg, file, line) end - # Executes the given block before each spec runs. + # Executes the given block before each spec in the current context runs. + # + # A context is defined by `describe` or `context` blocks, or outside of them + # it's the root context. Nested contexts inherit the `*_each` blocks of + # their ancestors. + # + # If multiple blocks are registered for the same spec, the blocks defined in + # the outermost context go first. Blocks on the same context are executed in + # order of definition. + # + # ``` + # require "spec + # + # it "sample_a" {} + # + # describe "nested_context" do + # before_each do + # puts "runs before sample_b" + # end + # + # it "sample_b" {} + # end + # ``` def before_each(&block) - Spec.root_context.before_each(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `before_each` outside of a describe/context" + end + Spec.current_context.before_each(&block) end - # Executes the given block after each spec runs. + # Executes the given block after each spec in the current context runs. + # + # A context is defined by `describe` or `context` blocks, or outside of them + # it's the root context. Nested contexts inherit the `*_each` blocks of + # their ancestors. + # + # If multiple blocks are registered for the same spec, the blocks defined in + # the outermost context go first. Blocks on the same context are executed in + # order of definition. + # + # ``` + # require "spec + # + # it "sample_a" {} + # + # describe "nested_context" do + # after_each do + # puts "runs after sample_b" + # end + # + # it "sample_b" {} + # end + # ``` def after_each(&block) - Spec.root_context.after_each(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `after_each` outside of a describe/context" + end + Spec.current_context.after_each(&block) end - # Executes the given block before all specs in a given - # `description` or `context` run. + # Executes the given block before the first spec in the current context runs. + # + # A context is defined by `describe` or `context` blocks, or outside of them + # it's the root context. + # This is independent of the source location the specs and this hook are + # defined. + # + # If multiple blocks are registered on the same context, they are executed in + # order of definition. + # + # ``` + # require "spec + # + # it "sample_a" {} + # + # describe "nested_context" do + # before_all do + # puts "runs at start of nested_context" + # end + # + # it "sample_b" {} + # end + # ``` def before_all(&block) - Spec.root_context.before_all(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `before_all` outside of a describe/context" + end + Spec.current_context.before_all(&block) end - # Executes the given block after all specs in a given - # `description` or `context` run. + # Executes the given block after the last spec in the current context runs. + # + # A context is defined by `describe` or `context` blocks, or outside of them + # it's the root context. + # This is independent of the source location the specs and this hook are + # defined. + # + # If multiple blocks are registered on the same context, they are executed in + # order of definition. + # + # ``` + # require "spec + # + # it "sample_a" {} + # + # describe "nested_context" do + # after_all do + # puts "runs at end of nested_context" + # end + # + # it "sample_b" {} + # end + # ``` def after_all(&block) - Spec.root_context.after_all(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `after_all` outside of a describe/context" + end + Spec.current_context.after_all(&block) end - # Executes the given block when each spec runs. + # Executes the given block when each spec in the current context runs. # # The block must call `run` on the given `Example::Procsy` # object. # - # For example: + # This is essentially a `before_each` and `after_each` hook combined into one. + # It is useful for example when setup and teardown steps need shared state. + # + # A context is defined by `describe` or `context` blocks, or outside of them + # it's the root context. Nested contexts inherit the `*_each` blocks of + # their ancestors. + # + # If multiple blocks are registered for the same spec, the blocks defined in + # the outermost context go first. Blocks on the same context are executed in + # order of definition. # # ``` - # require "spec" + # require "spec + # + # it "sample_a" {} # - # describe "something" do + # describe "nested_context" do # around_each do |example| - # puts "before example runs" + # puts "runs before sample_b" # example.run - # puts "after example runs" + # puts "runs after sample_b" # end # - # it "tests something" do - # # ... - # end + # it "sample_b" {} # end # ``` def around_each(&block : Example::Procsy ->) - Spec.root_context.around_each(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `around_each` outside of a describe/context" + end + Spec.current_context.around_each(&block) end - # Executes the given block when each `describe` or `context` runs. + # Executes the given block when the current context runs. # # The block must call `run` on the given `Context::Procsy` # object. # - # For example: + # This is essentially a `before_all` and `after_all` hook combined into one. + # It is useful for example when setup and teardown steps need shared state. + # + # A context is defined by `describe` or `context` blocks. This hook does not + # work outside such a block (i.e. in the root context). + # + # If multiple blocks are registered for the same spec, the blocks defined in + # the outermost context go first. Blocks on the same context are executed in + # order of definition. # # ``` - # require "spec" + # require "spec # - # describe "something" do - # around_all do |context| - # puts "before describe runs" + # describe "main_context" do + # around_each do |example| + # puts "runs at beginning of main_context" # example.run - # puts "after describe runs" + # puts "runs at end of main_context" # end # - # it "tests something" do - # # ... + # it "sample_a" {} + # + # describe "nested_context" do + # around_each do |example| + # puts "runs at beginning of nested_context" + # example.run + # puts "runs at end of nested_context" + # end + # + # it "sample_b" {} # end # end # ``` def around_all(&block : ExampleGroup::Procsy ->) - Spec.root_context.around_all(&block) + if Spec.current_context.is_a?(RootContext) + raise "Can't call `around_all` outside of a describe/context" + end + Spec.current_context.around_all(&block) end end From c7948d1912229e7c49158a16bfa1ddc4097cd96b Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 13 May 2020 09:47:31 -0300 Subject: [PATCH 027/263] Rework DWARF loading (#9267) * Extract call stack code to different files acording to platform * On ELF platforms, use `dl_iterate_phdr` to load DWARF data already offsetted * Use SizeT for the `base_address` in DWARF::LineNumbers * Move definition of `dl_iterate_phdr` and ELF types to `lib_c` * Remove HACK from comment --- src/crystal/dwarf/line_numbers.cr | 4 +- src/exception/call_stack.cr | 234 +----------------------- src/exception/call_stack/dwarf.cr | 61 ++++++ src/exception/call_stack/elf.cr | 51 ++++++ src/exception/call_stack/mach_o.cr | 111 +++++++++++ src/exception/call_stack/null.cr | 13 ++ src/lib_c/aarch64-linux-gnu/c/elf.cr | 24 +++ src/lib_c/aarch64-linux-gnu/c/link.cr | 13 ++ src/lib_c/aarch64-linux-musl/c/elf.cr | 24 +++ src/lib_c/aarch64-linux-musl/c/link.cr | 13 ++ src/lib_c/arm-linux-gnueabihf/c/elf.cr | 24 +++ src/lib_c/arm-linux-gnueabihf/c/link.cr | 13 ++ src/lib_c/i386-linux-gnu/c/elf.cr | 24 +++ src/lib_c/i386-linux-gnu/c/link.cr | 13 ++ src/lib_c/i386-linux-musl/c/elf.cr | 24 +++ src/lib_c/i386-linux-musl/c/link.cr | 13 ++ src/lib_c/x86_64-dragonfly/c/elf.cr | 24 +++ src/lib_c/x86_64-dragonfly/c/link.cr | 13 ++ src/lib_c/x86_64-freebsd/c/elf.cr | 24 +++ src/lib_c/x86_64-freebsd/c/link.cr | 13 ++ src/lib_c/x86_64-linux-gnu/c/elf.cr | 24 +++ src/lib_c/x86_64-linux-gnu/c/link.cr | 13 ++ src/lib_c/x86_64-linux-musl/c/elf.cr | 24 +++ src/lib_c/x86_64-linux-musl/c/link.cr | 13 ++ src/lib_c/x86_64-openbsd/c/elf.cr | 24 +++ src/lib_c/x86_64-openbsd/c/link.cr | 13 ++ 26 files changed, 612 insertions(+), 232 deletions(-) create mode 100644 src/exception/call_stack/dwarf.cr create mode 100644 src/exception/call_stack/elf.cr create mode 100644 src/exception/call_stack/mach_o.cr create mode 100644 src/exception/call_stack/null.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/elf.cr create mode 100644 src/lib_c/aarch64-linux-gnu/c/link.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/elf.cr create mode 100644 src/lib_c/aarch64-linux-musl/c/link.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/elf.cr create mode 100644 src/lib_c/arm-linux-gnueabihf/c/link.cr create mode 100644 src/lib_c/i386-linux-gnu/c/elf.cr create mode 100644 src/lib_c/i386-linux-gnu/c/link.cr create mode 100644 src/lib_c/i386-linux-musl/c/elf.cr create mode 100644 src/lib_c/i386-linux-musl/c/link.cr create mode 100644 src/lib_c/x86_64-dragonfly/c/elf.cr create mode 100644 src/lib_c/x86_64-dragonfly/c/link.cr create mode 100644 src/lib_c/x86_64-freebsd/c/elf.cr create mode 100644 src/lib_c/x86_64-freebsd/c/link.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/elf.cr create mode 100644 src/lib_c/x86_64-linux-gnu/c/link.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/elf.cr create mode 100644 src/lib_c/x86_64-linux-musl/c/link.cr create mode 100644 src/lib_c/x86_64-openbsd/c/elf.cr create mode 100644 src/lib_c/x86_64-openbsd/c/link.cr diff --git a/src/crystal/dwarf/line_numbers.cr b/src/crystal/dwarf/line_numbers.cr index b44ab51e897f..ff6c836072f3 100644 --- a/src/crystal/dwarf/line_numbers.cr +++ b/src/crystal/dwarf/line_numbers.cr @@ -168,7 +168,7 @@ module Crystal @offset : LibC::OffT - def initialize(@io : IO::FileDescriptor, size) + def initialize(@io : IO::FileDescriptor, size, @base_address : LibC::SizeT = 0) @offset = @io.tell @matrix = Array(Array(Row)).new decode_sequences(size) @@ -366,7 +366,7 @@ module Crystal path = sequence.include_directories[file[1]] row = Row.new( - registers.address, + registers.address + @base_address, registers.op_index, path, file[0], diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 327e79eb4a42..2cd1ec042a63 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -5,18 +5,10 @@ require "c/stdio" require "c/string" require "./lib_unwind" -{% if flag?(:darwin) %} - require "crystal/mach_o" - require "crystal/dwarf" - - lib LibC - fun _dyld_image_count : UInt32 - fun _dyld_get_image_name(image_index : UInt32) : Char* - fun _dyld_get_image_vmaddr_slide(image_index : UInt32) : Long - end -{% elsif flag?(:bsd) || flag?(:linux) %} - require "crystal/elf" - require "crystal/dwarf" +{% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) %} + require "./call_stack/dwarf" +{% else %} + require "./call_stack/null" {% end %} # Returns the current execution stack as an array containing strings @@ -197,224 +189,6 @@ struct Exception::CallStack end end - {% if flag?(:darwin) || flag?(:bsd) || flag?(:linux) %} - @@dwarf_line_numbers : Crystal::DWARF::LineNumbers? - @@dwarf_function_names : Array(Tuple(LibC::SizeT, LibC::SizeT, String))? - - protected def self.decode_line_number(pc) - read_dwarf_sections unless @@dwarf_line_numbers - if ln = @@dwarf_line_numbers - if row = ln.find(pc) - path = "#{row.directory}/#{row.file}" - return {path, row.line, row.column} - end - end - {"??", 0, 0} - end - - protected def self.decode_function_name(pc) - read_dwarf_sections unless @@dwarf_function_names - if fn = @@dwarf_function_names - fn.each do |(low_pc, high_pc, function_name)| - return function_name if low_pc <= pc <= high_pc - end - end - end - - protected def self.parse_function_names_from_dwarf(info, strings) - info.each do |code, abbrev, attributes| - next unless abbrev && abbrev.tag.subprogram? - name = low_pc = high_pc = nil - - attributes.each do |(at, form, value)| - case at - when Crystal::DWARF::AT::DW_AT_name - value = strings.try(&.decode(value.as(UInt32 | UInt64))) if form.strp? - name = value.as(String) - when Crystal::DWARF::AT::DW_AT_low_pc - low_pc = value.as(LibC::SizeT) - when Crystal::DWARF::AT::DW_AT_high_pc - if form.addr? - high_pc = value.as(LibC::SizeT) - elsif value.responds_to?(:to_i) - high_pc = low_pc.as(LibC::SizeT) + value.to_i - end - else - # Not an attribute we care - end - end - - if low_pc && high_pc && name - yield low_pc, high_pc, name - end - end - end - - {% if flag?(:darwin) %} - @@image_slide : LibC::Long? - - protected def self.read_dwarf_sections - locate_dsym_bundle do |mach_o| - mach_o.read_section?("__debug_line") do |sh, io| - @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size) - end - - strings = mach_o.read_section?("__debug_str") do |sh, io| - Crystal::DWARF::Strings.new(io, sh.offset, sh.size) - end - - mach_o.read_section?("__debug_info") do |sh, io| - names = [] of {LibC::SizeT, LibC::SizeT, String} - - while (offset = io.pos - sh.offset) < sh.size - info = Crystal::DWARF::Info.new(io, offset) - - mach_o.read_section?("__debug_abbrev") do |sh, io| - info.read_abbreviations(io) - end - - parse_function_names_from_dwarf(info, strings) do |low_pc, high_pc, name| - names << {low_pc, high_pc, name} - end - end - - @@dwarf_function_names = names - end - end - end - - # DWARF uses fixed addresses but Darwin loads exectutables at a random - # address, so we must remove the load offset from the IP to match the - # addresses in DWARF sections. - # - # See https://en.wikipedia.org/wiki/Address_space_layout_randomization - protected def self.decode_address(ip) - ip.address - image_slide - end - - # Searches the companion dSYM bundle with the DWARF sections for the - # current program as generated by `dsymutil`. It may be a `foo.dwarf` file - # or within a `foo.dSYM` bundle for a program named `foo`. - # - # See for details. - private def self.locate_dsym_bundle - program = Process.executable_path - return unless program - - files = { - "#{program}.dSYM/Contents/Resources/DWARF/#{File.basename(program)}", - "#{program}.dwarf" - } - - files.each do |dwarf| - next unless File.exists?(dwarf) - - Crystal::MachO.open(program) do |mach_o| - Crystal::MachO.open(dwarf) do |dsym| - if dsym.uuid == mach_o.uuid - return yield dsym - end - end - end - end - - nil - end - - # The address offset at which the program was loaded at. - private def self.image_slide - @@image_slide ||= search_image_slide - end - - private def self.search_image_slide - buffer = GC.malloc_atomic(LibC::PATH_MAX).as(UInt8*) - size = LibC::PATH_MAX.to_u32 - - if LibC._NSGetExecutablePath(buffer, pointerof(size)) == -1 - buffer = GC.malloc_atomic(size).as(UInt8*) - if LibC._NSGetExecutablePath(buffer, pointerof(size)) == -1 - return LibC::Long.new(0) - end - end - - program = String.new(buffer) - - LibC._dyld_image_count.times do |i| - if program == String.new(LibC._dyld_get_image_name(i)) - return LibC._dyld_get_image_vmaddr_slide(i) - end - end - - LibC::Long.new(0) - end - {% else %} - @@base_address : UInt64|UInt32|Nil - - protected def self.read_dwarf_sections - program = Process.executable_path - return unless program && File.readable? program - Crystal::ELF.open(program) do |elf| - elf.read_section?(".text") do |sh, _| - @@base_address = sh.addr - sh.offset - end - - elf.read_section?(".debug_line") do |sh, io| - @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size) - end - - strings = elf.read_section?(".debug_str") do |sh, io| - Crystal::DWARF::Strings.new(io, sh.offset, sh.size) - end - - elf.read_section?(".debug_info") do |sh, io| - names = [] of {LibC::SizeT, LibC::SizeT, String} - - while (offset = io.pos - sh.offset) < sh.size - info = Crystal::DWARF::Info.new(io, offset) - - elf.read_section?(".debug_abbrev") do |sh, io| - info.read_abbreviations(io) - end - - parse_function_names_from_dwarf(info, strings) do |name, low_pc, high_pc| - names << {name, low_pc, high_pc} - end - end - - @@dwarf_function_names = names - end - end - end - - # DWARF uses fixed addresses but some platforms (e.g., OpenBSD or Linux - # with the [PaX patch](https://en.wikipedia.org/wiki/PaX)) load - # executables at a random address, so we must remove the load offset from - # the IP to match the addresses in DWARF sections. - # - # See https://en.wikipedia.org/wiki/Address_space_layout_randomization - protected def self.decode_address(ip) - if LibC.dladdr(ip, out info) != 0 - unless info.dli_fbase.address == @@base_address - return ip.address - info.dli_fbase.address - end - end - ip.address - end - {% end %} - {% else %} - def self.decode_address(ip) - ip - end - - def self.decode_line_number(pc) - {"??", 0, 0} - end - - def self.decode_function_name(pc) - nil - end - {% end %} - protected def self.decode_frame(ip, original_ip = ip) if LibC.dladdr(ip, out info) != 0 offset = original_ip - info.dli_saddr diff --git a/src/exception/call_stack/dwarf.cr b/src/exception/call_stack/dwarf.cr new file mode 100644 index 000000000000..8e2a95c7a2cb --- /dev/null +++ b/src/exception/call_stack/dwarf.cr @@ -0,0 +1,61 @@ +require "crystal/dwarf" +{% if flag?(:darwin) %} + require "./mach_o" +{% else %} + require "./elf" +{% end %} + +struct Exception::CallStack + @@dwarf_loaded = false + @@dwarf_line_numbers : Crystal::DWARF::LineNumbers? + @@dwarf_function_names : Array(Tuple(LibC::SizeT, LibC::SizeT, String))? + + protected def self.decode_line_number(pc) + load_dwarf unless @@dwarf_loaded + if ln = @@dwarf_line_numbers + if row = ln.find(pc) + path = "#{row.directory}/#{row.file}" + return {path, row.line, row.column} + end + end + {"??", 0, 0} + end + + protected def self.decode_function_name(pc) + load_dwarf unless @@dwarf_loaded + if fn = @@dwarf_function_names + fn.each do |(low_pc, high_pc, function_name)| + return function_name if low_pc <= pc <= high_pc + end + end + end + + protected def self.parse_function_names_from_dwarf(info, strings) + info.each do |code, abbrev, attributes| + next unless abbrev && abbrev.tag.subprogram? + name = low_pc = high_pc = nil + + attributes.each do |(at, form, value)| + case at + when Crystal::DWARF::AT::DW_AT_name + value = strings.try(&.decode(value.as(UInt32 | UInt64))) if form.strp? + name = value.as(String) + when Crystal::DWARF::AT::DW_AT_low_pc + low_pc = value.as(LibC::SizeT) + when Crystal::DWARF::AT::DW_AT_high_pc + if form.addr? + high_pc = value.as(LibC::SizeT) + elsif value.responds_to?(:to_i) + high_pc = low_pc.as(LibC::SizeT) + value.to_i + end + else + # Not an attribute we care + end + end + + if low_pc && high_pc && name + yield low_pc, high_pc, name + end + end + end +end diff --git a/src/exception/call_stack/elf.cr b/src/exception/call_stack/elf.cr new file mode 100644 index 000000000000..865b70029540 --- /dev/null +++ b/src/exception/call_stack/elf.cr @@ -0,0 +1,51 @@ +require "crystal/elf" +require "c/link" + +struct Exception::CallStack + protected def self.load_dwarf + phdr_callback = LibC::DlPhdrCallback.new do |info, size, data| + # The first entry is the header for the current program + read_dwarf_sections(info.value.addr) + 1 + end + + LibC.dl_iterate_phdr(phdr_callback, nil) + @@dwarf_loaded = true + end + + protected def self.read_dwarf_sections(base_address = 0) + program = Process.executable_path + return unless program && File.readable? program + Crystal::ELF.open(program) do |elf| + elf.read_section?(".debug_line") do |sh, io| + @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size, base_address) + end + + strings = elf.read_section?(".debug_str") do |sh, io| + Crystal::DWARF::Strings.new(io, sh.offset, sh.size) + end + + elf.read_section?(".debug_info") do |sh, io| + names = [] of {LibC::SizeT, LibC::SizeT, String} + + while (offset = io.pos - sh.offset) < sh.size + info = Crystal::DWARF::Info.new(io, offset) + + elf.read_section?(".debug_abbrev") do |sh, io| + info.read_abbreviations(io) + end + + parse_function_names_from_dwarf(info, strings) do |low_pc, high_pc, name| + names << {low_pc + base_address, high_pc + base_address, name} + end + end + + @@dwarf_function_names = names + end + end + end + + protected def self.decode_address(ip) + ip.address + end +end diff --git a/src/exception/call_stack/mach_o.cr b/src/exception/call_stack/mach_o.cr new file mode 100644 index 000000000000..883237dadf95 --- /dev/null +++ b/src/exception/call_stack/mach_o.cr @@ -0,0 +1,111 @@ +require "crystal/mach_o" + +lib LibC + fun _dyld_image_count : UInt32 + fun _dyld_get_image_name(image_index : UInt32) : Char* + fun _dyld_get_image_vmaddr_slide(image_index : UInt32) : Long +end + +struct Exception::CallStack + @@image_slide : LibC::Long? + + protected def self.load_dwarf + read_dwarf_sections + @@dwarf_loaded = true + end + + protected def self.read_dwarf_sections + locate_dsym_bundle do |mach_o| + mach_o.read_section?("__debug_line") do |sh, io| + @@dwarf_line_numbers = Crystal::DWARF::LineNumbers.new(io, sh.size) + end + + strings = mach_o.read_section?("__debug_str") do |sh, io| + Crystal::DWARF::Strings.new(io, sh.offset, sh.size) + end + + mach_o.read_section?("__debug_info") do |sh, io| + names = [] of {LibC::SizeT, LibC::SizeT, String} + + while (offset = io.pos - sh.offset) < sh.size + info = Crystal::DWARF::Info.new(io, offset) + + mach_o.read_section?("__debug_abbrev") do |sh, io| + info.read_abbreviations(io) + end + + parse_function_names_from_dwarf(info, strings) do |low_pc, high_pc, name| + names << {low_pc, high_pc, name} + end + end + + @@dwarf_function_names = names + end + end + end + + # DWARF uses fixed addresses but Darwin loads exectutables at a random + # address, so we must remove the load offset from the IP to match the + # addresses in DWARF sections. + # + # See https://en.wikipedia.org/wiki/Address_space_layout_randomization + protected def self.decode_address(ip) + ip.address - image_slide + end + + # Searches the companion dSYM bundle with the DWARF sections for the + # current program as generated by `dsymutil`. It may be a `foo.dwarf` file + # or within a `foo.dSYM` bundle for a program named `foo`. + # + # See for details. + private def self.locate_dsym_bundle + program = Process.executable_path + return unless program + + files = { + "#{program}.dSYM/Contents/Resources/DWARF/#{File.basename(program)}", + "#{program}.dwarf", + } + + files.each do |dwarf| + next unless File.exists?(dwarf) + + Crystal::MachO.open(program) do |mach_o| + Crystal::MachO.open(dwarf) do |dsym| + if dsym.uuid == mach_o.uuid + return yield dsym + end + end + end + end + + nil + end + + # The address offset at which the program was loaded at. + private def self.image_slide + @@image_slide ||= search_image_slide + end + + private def self.search_image_slide + buffer = GC.malloc_atomic(LibC::PATH_MAX).as(UInt8*) + size = LibC::PATH_MAX.to_u32 + + if LibC._NSGetExecutablePath(buffer, pointerof(size)) == -1 + buffer = GC.malloc_atomic(size).as(UInt8*) + if LibC._NSGetExecutablePath(buffer, pointerof(size)) == -1 + return LibC::Long.new(0) + end + end + + program = String.new(buffer) + + LibC._dyld_image_count.times do |i| + if program == String.new(LibC._dyld_get_image_name(i)) + return LibC._dyld_get_image_vmaddr_slide(i) + end + end + + LibC::Long.new(0) + end +end diff --git a/src/exception/call_stack/null.cr b/src/exception/call_stack/null.cr new file mode 100644 index 000000000000..6b9d2ab6820f --- /dev/null +++ b/src/exception/call_stack/null.cr @@ -0,0 +1,13 @@ +struct Exception::CallStack + def self.decode_address(ip) + ip + end + + def self.decode_line_number(pc) + {"??", 0, 0} + end + + def self.decode_function_name(pc) + nil + end +end diff --git a/src/lib_c/aarch64-linux-gnu/c/elf.cr b/src/lib_c/aarch64-linux-gnu/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/aarch64-linux-gnu/c/link.cr b/src/lib_c/aarch64-linux-gnu/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/aarch64-linux-gnu/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/aarch64-linux-musl/c/elf.cr b/src/lib_c/aarch64-linux-musl/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/aarch64-linux-musl/c/link.cr b/src/lib_c/aarch64-linux-musl/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/aarch64-linux-musl/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/elf.cr b/src/lib_c/arm-linux-gnueabihf/c/elf.cr new file mode 100644 index 000000000000..32a0dc3152d1 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt32T + alias Elf_Off = UInt32T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Word # Segment size in file + memsz : Elf_Word # Segment size in memory + flags : Elf_Word # Segment flags + align : Elf_Word # Segment alignment + end +end diff --git a/src/lib_c/arm-linux-gnueabihf/c/link.cr b/src/lib_c/arm-linux-gnueabihf/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/arm-linux-gnueabihf/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/i386-linux-gnu/c/elf.cr b/src/lib_c/i386-linux-gnu/c/elf.cr new file mode 100644 index 000000000000..32a0dc3152d1 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt32T + alias Elf_Off = UInt32T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Word # Segment size in file + memsz : Elf_Word # Segment size in memory + flags : Elf_Word # Segment flags + align : Elf_Word # Segment alignment + end +end diff --git a/src/lib_c/i386-linux-gnu/c/link.cr b/src/lib_c/i386-linux-gnu/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/i386-linux-gnu/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/i386-linux-musl/c/elf.cr b/src/lib_c/i386-linux-musl/c/elf.cr new file mode 100644 index 000000000000..32a0dc3152d1 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt32T + alias Elf_Off = UInt32T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Word # Segment size in file + memsz : Elf_Word # Segment size in memory + flags : Elf_Word # Segment flags + align : Elf_Word # Segment alignment + end +end diff --git a/src/lib_c/i386-linux-musl/c/link.cr b/src/lib_c/i386-linux-musl/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/i386-linux-musl/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-dragonfly/c/elf.cr b/src/lib_c/x86_64-dragonfly/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-dragonfly/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-dragonfly/c/link.cr b/src/lib_c/x86_64-dragonfly/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/x86_64-dragonfly/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-freebsd/c/elf.cr b/src/lib_c/x86_64-freebsd/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-freebsd/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-freebsd/c/link.cr b/src/lib_c/x86_64-freebsd/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/x86_64-freebsd/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-linux-gnu/c/elf.cr b/src/lib_c/x86_64-linux-gnu/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-linux-gnu/c/link.cr b/src/lib_c/x86_64-linux-gnu/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/x86_64-linux-gnu/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-linux-musl/c/elf.cr b/src/lib_c/x86_64-linux-musl/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-linux-musl/c/link.cr b/src/lib_c/x86_64-linux-musl/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/x86_64-linux-musl/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-openbsd/c/elf.cr b/src/lib_c/x86_64-openbsd/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-openbsd/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-openbsd/c/link.cr b/src/lib_c/x86_64-openbsd/c/link.cr new file mode 100644 index 000000000000..a29bd3030a42 --- /dev/null +++ b/src/lib_c/x86_64-openbsd/c/link.cr @@ -0,0 +1,13 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end From adf8112e1eeae79423986e11aa5ca8858af33a39 Mon Sep 17 00:00:00 2001 From: didactic-drunk <1479616+didactic-drunk@users.noreply.github.com> Date: Wed, 13 May 2020 06:03:53 -0700 Subject: [PATCH 028/263] Cleanup Digest and OpenSSL::Digest (#8426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Mark Digest::* methods private. * Add abstract methods to Digest::Base. Allow resetting digests. Move some OpenSSL::Digest methods to Digest::Base. * OpenSSL::Digest inherit from Digest::Base and use it's interface. * Suggestions from @straight-shoota * Remove @message_block reset. * [WIP] Deprecate #digest and #hexdigest. Detect #final misuse. TODO: Use #final in specs after API approval. * Update src/digest/base.cr Co-Authored-By: Sijawusz Pur Rahnama * Deprecate #digest and #hexdigest. Detect #final misuse. * Suggestions from @RX14. * Update src/digest/base.cr Co-Authored-By: Sijawusz Pur Rahnama * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Use private abstract methods instead of overrides. * Suggestions from @straight-shoota * Suggstions from @sija * Abstract reset and documentation. * Update src/digest/md5.cr Co-Authored-By: Johannes Müller * Update src/digest/sha1.cr Co-Authored-By: Sijawusz Pur Rahnama * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Update src/digest/base.cr Co-Authored-By: Johannes Müller * Suggestion from @straight-shoota * Update src/digest/base.cr Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Johannes Müller --- spec/std/digest/md5_spec.cr | 22 ++++ spec/std/digest/sha1_spec.cr | 30 ++++- spec/std/openssl/digest_spec.cr | 14 +- src/digest/base.cr | 210 ++++++++++++++++++++---------- src/digest/md5.cr | 57 ++++---- src/digest/sha1.cr | 54 ++++---- src/openssl/digest/digest.cr | 36 ++--- src/openssl/digest/digest_base.cr | 17 +-- src/openssl/digest/digest_io.cr | 2 +- 9 files changed, 283 insertions(+), 159 deletions(-) diff --git a/spec/std/digest/md5_spec.cr b/spec/std/digest/md5_spec.cr index 2f6afb8f4e3f..ac4cba08dc1c 100644 --- a/spec/std/digest/md5_spec.cr +++ b/spec/std/digest/md5_spec.cr @@ -34,4 +34,26 @@ describe Digest::MD5 do it "calculates base64'd hash from string" do Digest::MD5.base64digest("foo").should eq("rL0Y20zC+Fzt72VPzMSk2A==") end + + it "resets" do + digest = Digest::MD5.new + digest.update "foo" + digest.final.hexstring.should eq("acbd18db4cc2f85cedef654fccc4a4d8") + + digest.reset + digest.update "foo" + digest.final.hexstring.should eq("acbd18db4cc2f85cedef654fccc4a4d8") + end + + it "can't call final twice" do + digest = Digest::MD5.new + digest.final + expect_raises(Digest::FinalizedError) do + digest.final + end + end + + it "return the digest size" do + Digest::MD5.new.digest_size.should eq 16 + end end diff --git a/spec/std/digest/sha1_spec.cr b/spec/std/digest/sha1_spec.cr index 99a13e1548fe..484af50e2ec3 100644 --- a/spec/std/digest/sha1_spec.cr +++ b/spec/std/digest/sha1_spec.cr @@ -10,10 +10,28 @@ describe Digest::SHA1 do {"a", "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "hvfkN/qlp/zhXR3cuerq6jd2Z7g="}, {"0123456701234567012345670123456701234567012345670123456701234567", "e0c094e867ef46c350ef54a7f59dd60bed92ae83", "4MCU6GfvRsNQ71Sn9Z3WC+2SroM="}, {"fooø", "dcf4a1e3542b1a40a4ac2a3f7c92ffdb2d19812f", "3PSh41QrGkCkrCo/fJL/2y0ZgS8="}, - ].each do |(string, hexdigest, base64digest)| + ].each do |(string, hexstring, base64digest)| it "does digest for #{string.inspect}" do bytes = Digest::SHA1.digest(string) - bytes.to_slice.hexstring.should eq(hexdigest) + bytes.hexstring.should eq(hexstring) + end + + it "resets" do + digest = Digest::SHA1.new + digest.update string + digest.final.hexstring.should eq(hexstring) + + digest.reset + digest.update string + digest.final.hexstring.should eq(hexstring) + end + + it "can't call #final more than once" do + digest = Digest::SHA1.new + digest.final + expect_raises(Digest::FinalizedError) do + digest.final + end end it "does digest for #{string.inspect} in a block" do @@ -23,15 +41,19 @@ describe Digest::SHA1 do end end - bytes.to_slice.hexstring.should eq(hexdigest) + bytes.hexstring.should eq(hexstring) end it "does hexdigest for #{string.inspect}" do - Digest::SHA1.hexdigest(string).should eq(hexdigest) + Digest::SHA1.hexdigest(string).should eq(hexstring) end it "does base64digest for #{string.inspect}" do Digest::SHA1.base64digest(string).should eq(base64digest) end end + + it "returns the digest_size" do + Digest::SHA1.new.digest_size.should eq(20) + end end diff --git a/spec/std/openssl/digest_spec.cr b/spec/std/openssl/digest_spec.cr index b29fd59af52f..a93f40d829b0 100644 --- a/spec/std/openssl/digest_spec.cr +++ b/spec/std/openssl/digest_spec.cr @@ -10,10 +10,22 @@ describe OpenSSL::Digest do it "should be able to calculate #{algorithm}" do digest = OpenSSL::Digest.new(algorithm) digest << "fooø" + digest.final.hexstring.should eq(expected) + + digest.reset + digest << "fooø" digest.hexdigest.should eq(expected) end end + it "can't call #final more than once" do + digest = OpenSSL::Digest.new("SHA1") + digest.final + expect_raises(Digest::FinalizedError) do + digest.final + end + end + it "raises a UnsupportedError if digest is unsupported" do expect_raises OpenSSL::Digest::UnsupportedError do OpenSSL::Digest.new("unsupported") @@ -44,6 +56,6 @@ describe OpenSSL::Digest do digest << r r.close - digest.hexdigest.should eq("2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + digest.final.hexstring.should eq("2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") end end diff --git a/src/digest/base.cr b/src/digest/base.cr index b2e8a8c83cb0..53cfce8c8a8f 100644 --- a/src/digest/base.cr +++ b/src/digest/base.cr @@ -1,92 +1,158 @@ require "base64" +class Digest::FinalizedError < Exception +end + abstract class Digest::Base - # Returns the hash of *data*. *data* must respond to `#to_slice`. - def self.digest(data) - digest do |ctx| - ctx.update(data.to_slice) + macro inherited + # Returns the hash of *data*. *data* must respond to `#to_slice`. + def self.digest(data) + digest do |ctx| + ctx.update(data.to_slice) + end + end + + # Yields an instance of `self` which can receive calls to `#update(data : String | Bytes)` + # and returns the finalized digest afterwards. + # + # ``` + # require "digest/md5" + # + # digest = Digest::MD5.digest do |ctx| + # ctx.update "f" + # ctx.update "oo" + # end + # digest.to_slice.hexstring # => "acbd18db4cc2f85cedef654fccc4a4d8" + # ``` + def self.digest(& : Digest::Base -> _) : Bytes + context = new + yield context + context.final + end + + # Returns the hexadecimal representation of the hash of *data*. + # + # ``` + # require "digest/md5" + # + # Digest::MD5.hexdigest("foo") # => "acbd18db4cc2f85cedef654fccc4a4d8" + # ``` + def self.hexdigest(data) : String + hexdigest &.update(data) + end + + # Yields a context object with an `#update(data : String | Bytes)` + # method available. Returns the resulting digest in hexadecimal representation + # afterwards. + # + # ``` + # require "digest/md5" + # + # Digest::MD5.hexdigest("foo") # => "acbd18db4cc2f85cedef654fccc4a4d8" + # Digest::MD5.hexdigest do |ctx| + # ctx.update "f" + # ctx.update "oo" + # end + # # => "acbd18db4cc2f85cedef654fccc4a4d8" + # ``` + def self.hexdigest(& : Digest::Base -> _) : String + hashsum = digest do |ctx| + yield ctx + end + + hashsum.to_slice.hexstring + end + + # Returns the base64-encoded hash of *data*. + # + # ``` + # require "digest/sha1" + # + # Digest::SHA1.base64digest("foo") # => "C+7Hteo/D9vJXQ3UfzxbwnXaijM=" + # ``` + def self.base64digest(data) : String + base64digest &.update(data) + end + + # Yields a context object with an `#update(data : String | Bytes)` + # method available. Returns the resulting digest in base64 representation + # afterwards. + # + # ``` + # require "digest/sha1" + # + # Digest::SHA1.base64digest do |ctx| + # ctx.update "f" + # ctx.update "oo" + # end + # # => "C+7Hteo/D9vJXQ3UfzxbwnXaijM=" + # ``` + def self.base64digest(& : Digest::Base -> _) : String + hashsum = digest do |ctx| + yield ctx + end + + Base64.strict_encode(hashsum) end end - # Yields a context object with an `#update(data : String | Bytes)` - # method available. Returns the resulting digest afterwards. - # - # ``` - # require "digest/md5" - # - # digest = Digest::MD5.digest do |ctx| - # ctx.update "f" - # ctx.update "oo" - # end - # digest.to_slice.hexstring # => "acbd18db4cc2f85cedef654fccc4a4d8" - # ``` - def self.digest - context = new - yield context - context.final - context.result + @finished = false + + def update(data) : self + update data.to_slice end - # Returns the hexadecimal representation of the hash of *data*. - # - # ``` - # require "digest/md5" - # - # Digest::MD5.hexdigest("foo") # => "acbd18db4cc2f85cedef654fccc4a4d8" - # ``` - def self.hexdigest(data) : String - hexdigest &.update(data) + def update(data : Bytes) : self + check_finished + update_impl data + self end - # Yields a context object with an `#update(data : String | Bytes)` - # method available. Returns the resulting digest in hexadecimal representation - # afterwards. + # Returns the final digest output. # - # ``` - # require "digest/md5" + # This method can only be called once and raises `FinalizedError` on subsequent calls. # - # Digest::MD5.hexdigest("foo") # => "acbd18db4cc2f85cedef654fccc4a4d8" - # Digest::MD5.hexdigest do |ctx| - # ctx.update "f" - # ctx.update "oo" - # end - # # => "acbd18db4cc2f85cedef654fccc4a4d8" - # ``` - def self.hexdigest : String - hashsum = digest do |ctx| - yield ctx - end + # NOTE: `.dup.final` call may be used to get an intermediate hash value. + def final : Bytes + dst = Bytes.new digest_size + final dst + end - hashsum.to_slice.hexstring + def final(dst : Bytes) : Bytes + check_finished + @finished = true + final_impl dst + dst end - # Returns the base64-encoded hash of *data*. - # - # ``` - # require "digest/sha1" - # - # Digest::SHA1.base64digest("foo") # => "C+7Hteo/D9vJXQ3UfzxbwnXaijM=" - # ``` - def self.base64digest(data) : String - base64digest &.update(data) + def reset : self + reset_impl + @finished = false + self end - # Returns the base64-encoded hash of *data*. - # - # ``` - # require "digest/sha1" - # - # Digest::SHA1.base64digest do |ctx| - # ctx.update "f" - # ctx.update "oo" - # end - # # => "C+7Hteo/D9vJXQ3UfzxbwnXaijM=" - # ``` - def self.base64digest : String - hashsum = digest do |ctx| - yield ctx - end + # Dups and finishes the digest. + @[Deprecated("Use `final` instead.")] + def digest : Bytes + dup.final + end - Base64.strict_encode(hashsum) + # Returns a hexadecimal-encoded digest. + @[Deprecated("Use `final.hexstring` instead.")] + def hexdigest : String + digest.hexstring end + + private def check_finished : Nil + raise FinalizedError.new("finish already called") if @finished + end + + # Hashes data incrementally. + abstract def update_impl(data : Bytes) : Nil + # Stores the output digest of #digest_size bytes in dst. + abstract def final_impl(dst : Bytes) : Nil + # Resets the object to it's initial state. + abstract def reset_impl : Nil + # Returns the digest output size in bytes. + abstract def digest_size : Int32 end diff --git a/src/digest/md5.cr b/src/digest/md5.cr index 110859cbe966..69b7ddefa3f0 100644 --- a/src/digest/md5.cr +++ b/src/digest/md5.cr @@ -7,23 +7,28 @@ require "./base" # `Crypto::Bcrypt::Password`. For a generic cryptographic hash, use SHA-256 via # `OpenSSL::Digest.new("SHA256")`. class Digest::MD5 < Digest::Base + @i = StaticArray(UInt32, 2).new(0_u32) + @buf = StaticArray(UInt32, 4).new(0_u32) + @in = StaticArray(UInt8, 64).new(0_u8) + def initialize - @i = StaticArray(UInt32, 2).new(0_u32) - @buf = StaticArray(UInt32, 4).new(0_u32) + reset + end + + private def reset_impl : Nil + @i[0] = 0_u32 + @i[1] = 0_u32 @buf[0] = 0x67452301_u32 @buf[1] = 0xEFCDAB89_u32 @buf[2] = 0x98BADCFE_u32 @buf[3] = 0x10325476_u32 - @in = StaticArray(UInt8, 64).new(0_u8) - @digest = uninitialized UInt8[16] end - def update(data) - slice = data.to_slice - update(slice.to_unsafe, slice.bytesize.to_u32) + private def update_impl(data : Bytes) : Nil + update(data.to_unsafe, data.bytesize.to_u32) end - def update(inBuf, inLen) + private def update(inBuf, inLen) tmp_in = uninitialized UInt32[16] # compute number of bytes mod 64 @@ -76,57 +81,57 @@ class Digest::MD5 < Digest::Base S43 = 15 S44 = 21 - PADDING = begin + private PADDING = begin padding = StaticArray(UInt8, 64).new(0_u8) padding[0] = 0x80_u8 padding end - def f(x, y, z) + private def f(x, y, z) (x & y) | ((~x) & z) end - def g(x, y, z) + private def g(x, y, z) (x & z) | (y & (~z)) end - def h(x, y, z) + private def h(x, y, z) x ^ y ^ z end - def i(x, y, z) + private def i(x, y, z) y ^ (x | (~z)) end - def rotate_left(x, n) + private def rotate_left(x, n) (x << n) | (x >> (32 - n)) end - def ff(a, b, c, d, x, s, ac) + private def ff(a, b, c, d, x, s, ac) a &+= f(b, c, d) &+ x &+ ac.to_u32 a = rotate_left a, s a &+= b end - def gg(a, b, c, d, x, s, ac) + private def gg(a, b, c, d, x, s, ac) a &+= g(b, c, d) &+ x &+ ac.to_u32 a = rotate_left a, s a &+= b end - def hh(a, b, c, d, x, s, ac) + private def hh(a, b, c, d, x, s, ac) a &+= h(b, c, d) &+ x &+ ac.to_u32 a = rotate_left a, s a &+= b end - def ii(a, b, c, d, x, s, ac) + private def ii(a, b, c, d, x, s, ac) a &+= i(b, c, d) &+ x &+ ac.to_u32 a = rotate_left a, s a &+= b end - def transform(in) + private def transform(in) a, b, c, d = @buf # Round 1 @@ -207,7 +212,7 @@ class Digest::MD5 < Digest::Base @buf[3] &+= d end - def final + private def final_impl(dst : Bytes) : Nil tmp_in = uninitialized UInt32[16] # save number of bits @@ -235,15 +240,15 @@ class Digest::MD5 < Digest::Base # store buffer in digest ii = 0 4.times do |i| - @digest[ii] = (@buf[i] & 0xff).to_u8 - @digest[ii + 1] = ((@buf[i] >> 8) & 0xFF).to_u8 - @digest[ii + 2] = ((@buf[i] >> 16) & 0xFF).to_u8 - @digest[ii + 3] = ((@buf[i] >> 24) & 0xFF).to_u8 + dst[ii] = (@buf[i] & 0xff).to_u8 + dst[ii + 1] = ((@buf[i] >> 8) & 0xFF).to_u8 + dst[ii + 2] = ((@buf[i] >> 16) & 0xFF).to_u8 + dst[ii + 3] = ((@buf[i] >> 24) & 0xFF).to_u8 ii += 4 end end - def result - @digest + def digest_size : Int32 + 16 end end diff --git a/src/digest/sha1.cr b/src/digest/sha1.cr index bc6c7fa06d91..c75de4c07738 100644 --- a/src/digest/sha1.cr +++ b/src/digest/sha1.cr @@ -10,13 +10,20 @@ class Digest::SHA1 < Digest::Base # This is a direct translation of https://tools.ietf.org/html/rfc3174#section-7 # but we use loop unrolling for faster execution (about 1.07x slower than OpenSSL::SHA1). + @intermediate_hash = uninitialized UInt32[5] + @length_low = 0_u32 + @length_high = 0_u32 + @message_block_index = 0 + @message_block = StaticArray(UInt8, 64).new(0_u8) # uninitialized UInt8[64] + def initialize - @intermediate_hash = uninitialized UInt32[5] + reset + end + + private def reset_impl : Nil @length_low = 0_u32 @length_high = 0_u32 - @message_block = StaticArray(UInt8, 64).new(0_u8) # uninitialized UInt8[64] @message_block_index = 0 - @intermediate_hash[0] = 0x67452301_u32 @intermediate_hash[1] = 0xEFCDAB89_u32 @intermediate_hash[2] = 0x98BADCFE_u32 @@ -24,9 +31,8 @@ class Digest::SHA1 < Digest::Base @intermediate_hash[4] = 0xC3D2E1F0_u32 end - def update(data) - message_array = data.to_slice - message_array.each do |byte| + private def update_impl(data : Bytes) : Nil + data.each do |byte| @message_block[@message_block_index] = byte & 0xFF_u8 @message_block_index += 1 @length_low += 8 @@ -44,7 +50,17 @@ class Digest::SHA1 < Digest::Base end end - def process_message_block + private def final_impl(dst : Bytes) : Nil + pad_message + + @length_low = 0_u32 + @length_high = 0_u32 + {% for i in 0...20 %} + dst[{{i}}] = (@intermediate_hash[{{i >> 2}}] >> 8 * (3 - ({{i & 0x03}}))).to_u8! + {% end %} + end + + private def process_message_block k = {0x5A827999_u32, 0x6ED9EBA1_u32, 0x8F1BBCDC_u32, 0xCA62C1D6_u32} w = uninitialized UInt32[80] @@ -113,27 +129,11 @@ class Digest::SHA1 < Digest::Base @message_block_index = 0 end - def circular_shift(bits, word) + private def circular_shift(bits, word) (word << bits) | (word >> (32 - bits)) end - def final - end - - def result - message_digest = uninitialized UInt8[20] - pad_message - - @length_low = 0_u32 - @length_high = 0_u32 - {% for i in 0...20 %} - message_digest[{{i}}] = (@intermediate_hash[{{i >> 2}}] >> 8 * (3 - ({{i & 0x03}}))).to_u8! - {% end %} - - message_digest - end - - def pad_message + private def pad_message if @message_block_index > 55 @message_block[@message_block_index] = 0x80_u8 @message_block_index += 1 @@ -168,4 +168,8 @@ class Digest::SHA1 < Digest::Base process_message_block end + + def digest_size : Int32 + 20 + end end diff --git a/src/openssl/digest/digest.cr b/src/openssl/digest/digest.cr index c1cb2504c1b5..59b9692c9683 100644 --- a/src/openssl/digest/digest.cr +++ b/src/openssl/digest/digest.cr @@ -1,8 +1,9 @@ require "../lib_crypto" +require "digest/base" require "./digest_base" module OpenSSL - class Digest + class Digest < ::Digest::Base class Error < OpenSSL::Error; end class UnsupportedError < Error; end @@ -38,40 +39,41 @@ module OpenSSL LibCrypto.evp_md_ctx_free(self) end - def clone + def dup ctx = LibCrypto.evp_md_ctx_new if LibCrypto.evp_md_ctx_copy(ctx, @ctx) == 0 LibCrypto.evp_md_ctx_free(ctx) - raise Error.new("Unable to clone digest") + raise Error.new("Unable to dup digest") end Digest.new(@name, ctx) end - def reset + private def reset_impl : Nil if LibCrypto.evp_digestinit_ex(self, to_unsafe_md, nil) != 1 raise Error.new "Digest initialization failed." end - self end - def update(data : String | Slice) - LibCrypto.evp_digestupdate(self, data, data.bytesize) - self + private def update_impl(data : Bytes) : Nil + check_finished + if LibCrypto.evp_digestupdate(self, data, data.bytesize) != 1 + raise Error.new "EVP_DigestUpdate" + end end - protected def finish - size = digest_size - data = Pointer(UInt8).malloc(size) - LibCrypto.evp_digestfinal_ex(@ctx, data, nil) - data.to_slice(size) + private def final_impl(data : Bytes) : Nil + raise ArgumentError.new("data size incorrect") unless data.bytesize == digest_size + if LibCrypto.evp_digestfinal_ex(@ctx, data, nil) != 1 + raise Error.new "EVP_DigestFinal_ex" + end end - def digest_size - LibCrypto.evp_md_size(to_unsafe_md) + def digest_size : Int32 + LibCrypto.evp_md_size(to_unsafe_md).to_i end - def block_size - LibCrypto.evp_md_block_size(to_unsafe_md) + def block_size : Int32 + LibCrypto.evp_md_block_size(to_unsafe_md).to_i end def to_unsafe_md diff --git a/src/openssl/digest/digest_base.cr b/src/openssl/digest/digest_base.cr index 678328e3962d..6a05e31a8973 100644 --- a/src/openssl/digest/digest_base.cr +++ b/src/openssl/digest/digest_base.cr @@ -10,7 +10,7 @@ module OpenSSL end # Reads the io's data and updates the digest with it. - def update(io : IO) : Digest + def update(io : IO) : self buffer = uninitialized UInt8[4096] while (read_bytes = io.read(buffer.to_slice)) > 0 self << buffer.to_slice[0, read_bytes] @@ -19,26 +19,17 @@ module OpenSSL end # :ditto: - def <<(data) : Digest + def <<(data) : self update(data) end - # Clones and finishes the digest. - def digest : Bytes - self.clone.finish - end - # Returns a base64-encoded digest. + @[Deprecated("Use `Base64.strict_encode(final)` instead.")] def base64digest : String Base64.strict_encode(digest) end - # Returns a hexadecimal-encoded digest. - def hexdigest : String - digest.hexstring - end - - # :ditto: + @[Deprecated("Use `io << final.hexstring` instead.")] def to_s(io : IO) : Nil io << hexdigest end diff --git a/src/openssl/digest/digest_io.cr b/src/openssl/digest/digest_io.cr index 3be9a7c049c9..4093725fefcc 100644 --- a/src/openssl/digest/digest_io.cr +++ b/src/openssl/digest/digest_io.cr @@ -12,7 +12,7 @@ module OpenSSL # io = OpenSSL::DigestIO.new(underlying_io, "SHA256") # buffer = Bytes.new(256) # io.read(buffer) - # io.digest.hexstring # => "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + # io.final.hexstring # => "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" # ``` class DigestIO < IO getter io : IO From a3ec85ab8f6d4061540eae29b0f4fb998076532d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 13 May 2020 15:47:42 +0200 Subject: [PATCH 029/263] Refactor to standardize on first argument for methods receiving `IO` (#9134) * Reorganize Enumerable#join specs * Add IO-first overloads to Enumerable#join * Fix deprecated uses of Enumerable#join * Fix deprecated uses of Int#to_s * Add IO-first overload to Int#to_s * Change upcase argument of Int#to_s to a named argument * Add IO-first overloads to String methods * Change to IO-first methods in Crystal::Exception * Add IO-first overload to Time#to_s --- spec/std/enumerable_spec.cr | 54 +++++++++++++++---- spec/std/int_spec.cr | 7 ++- spec/std/string_spec.cr | 21 ++++++++ src/array.cr | 2 +- src/char.cr | 4 +- src/colorize.cr | 8 ++- src/compiler/crystal/codegen/codegen.cr | 2 +- src/compiler/crystal/exception.cr | 10 ++-- src/compiler/crystal/semantic/ast.cr | 2 +- src/compiler/crystal/semantic/call_error.cr | 4 +- src/compiler/crystal/semantic/exception.cr | 16 +++--- src/compiler/crystal/semantic/warnings.cr | 2 +- src/compiler/crystal/syntax/exception.cr | 6 +-- src/compiler/crystal/syntax/to_s.cr | 48 ++++++++--------- src/compiler/crystal/tools/doc/generator.cr | 2 +- src/compiler/crystal/tools/doc/method.cr | 2 +- src/compiler/crystal/tools/doc/type.cr | 16 +++--- src/compiler/crystal/types.cr | 12 ++--- src/deque.cr | 2 +- src/enumerable.cr | 51 +++++++++++++++--- src/fiber.cr | 2 +- src/http/common.cr | 2 +- src/http/server/response.cr | 2 +- src/int.cr | 22 ++++---- src/json/builder.cr | 2 +- src/option_parser.cr | 2 +- src/pointer.cr | 2 +- src/proc.cr | 2 +- src/reference.cr | 4 +- src/semantic_version.cr | 2 +- src/set.cr | 2 +- src/slice.cr | 4 +- src/static_array.cr | 2 +- src/string.cr | 60 +++++++++++++++++---- src/string/formatter.cr | 2 +- src/time.cr | 10 +++- src/tuple.cr | 2 +- src/uri.cr | 4 +- src/uri/encoding.cr | 2 +- src/xml/attributes.cr | 2 +- src/xml/namespace.cr | 2 +- src/xml/node.cr | 2 +- src/xml/node_set.cr | 2 +- 43 files changed, 279 insertions(+), 130 deletions(-) diff --git a/spec/std/enumerable_spec.cr b/spec/std/enumerable_spec.cr index 18dda442134b..b1becdf0334e 100644 --- a/spec/std/enumerable_spec.cr +++ b/spec/std/enumerable_spec.cr @@ -620,26 +620,60 @@ describe "Enumerable" do end end - describe "join" do - it "joins with separator and block" do - str = [1, 2, 3].join(", ") { |x| x + 1 } - str.should eq("2, 3, 4") + describe "#join" do + it "()" do + [1, 2, 3].join.should eq("123") + end + + it "(separator)" do + ["Ruby", "Crystal", "Python"].join(", ").should eq "Ruby, Crystal, Python" end - it "joins without separator and block" do + it "(&)" do str = [1, 2, 3].join { |x| x + 1 } str.should eq("234") end - it "joins with io and block" do + it "(separator, &)" do + str = [1, 2, 3].join(", ") { |x| x + 1 } + str.should eq("2, 3, 4") + end + + it "(io)" do + io = IO::Memory.new + [1, 2, 3].join(io) + io.to_s.should eq("123") + end + + it "(io, separator)" do + io = IO::Memory.new + ["Ruby", "Crystal", "Python"].join(io, ", ") + io.to_s.should eq "Ruby, Crystal, Python" + end + + it "(separator, io) (deprecated)" do + io = IO::Memory.new + ["Ruby", "Crystal", "Python"].join(", ", io) + io.to_s.should eq "Ruby, Crystal, Python" + end + + it "(io, &)" do + io = IO::Memory.new + [1, 2, 3].join(io) { |x, io| io << x + 1 } + io.to_s.should eq("234") + end + + it "(io, separator, &)" do + io = IO::Memory.new + [1, 2, 3].join(io, ", ") { |x, io| io << x + 1 } + io.to_s.should eq("2, 3, 4") + end + + it "(separator, io, &) (deprecated)" do str = IO::Memory.new [1, 2, 3].join(", ", str) { |x, io| io << x + 1 } str.to_s.should eq("2, 3, 4") end - - it "joins with only separator" do - ["Ruby", "Crystal", "Python"].join(", ").should eq "Ruby, Crystal, Python" - end end describe "map" do diff --git a/spec/std/int_spec.cr b/spec/std/int_spec.cr index 8af61717f197..5f4eac7be322 100644 --- a/spec/std/int_spec.cr +++ b/spec/std/int_spec.cr @@ -4,11 +4,13 @@ require "./spec_helper" {% end %} private def to_s_with_io(num) - String.build { |str| num.to_s(str) } + String.build { |io| num.to_s(io) } end private def to_s_with_io(num, base, upcase = false) - String.build { |str| num.to_s(base, str, upcase) } + String.build { |io| num.to_s(io, base, upcase: upcase) } + # Test deprecated overload: + String.build { |io| num.to_s(base, io, upcase) } end describe "Int" do @@ -172,6 +174,7 @@ describe "Int" do it { 1234.to_s(36).should eq("ya") } it { -1234.to_s(36).should eq("-ya") } it { 1234.to_s(16, upcase: true).should eq("4D2") } + it { 1234.to_s(16, true).should eq("4D2") } # Deprecated test it { -1234.to_s(16, upcase: true).should eq("-4D2") } it { 1234.to_s(36, upcase: true).should eq("YA") } it { -1234.to_s(36, upcase: true).should eq("-YA") } diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 029d149ed9a2..f926dc124b23 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -2031,6 +2031,13 @@ describe "String" do it { "12".ljust(7, 'あ').should eq("12あああああ") } describe "to io" do + it { with_io_memory { |io| "123".ljust(io, 2) }.should eq("123") } + it { with_io_memory { |io| "123".ljust(io, 5) }.should eq("123 ") } + it { with_io_memory { |io| "12".ljust(io, 7, '-') }.should eq("12-----") } + it { with_io_memory { |io| "12".ljust(io, 7, 'あ') }.should eq("12あああああ") } + end + + describe "to io (deprecated)" do it { with_io_memory { |io| "123".ljust(2, io) }.should eq("123") } it { with_io_memory { |io| "123".ljust(5, io) }.should eq("123 ") } it { with_io_memory { |io| "12".ljust(7, '-', io) }.should eq("12-----") } @@ -2045,6 +2052,13 @@ describe "String" do it { "12".rjust(7, 'あ').should eq("あああああ12") } describe "to io" do + it { with_io_memory { |io| "123".rjust(io, 2) }.should eq("123") } + it { with_io_memory { |io| "123".rjust(io, 5) }.should eq(" 123") } + it { with_io_memory { |io| "12".rjust(io, 7, '-') }.should eq("-----12") } + it { with_io_memory { |io| "12".rjust(io, 7, 'あ') }.should eq("あああああ12") } + end + + describe "to io (deprecated)" do it { with_io_memory { |io| "123".rjust(2, io) }.should eq("123") } it { with_io_memory { |io| "123".rjust(5, io) }.should eq(" 123") } it { with_io_memory { |io| "12".rjust(7, '-', io) }.should eq("-----12") } @@ -2059,6 +2073,13 @@ describe "String" do it { "12".center(7, 'あ').should eq("ああ12あああ") } describe "to io" do + it { with_io_memory { |io| "123".center(io, 2) }.should eq("123") } + it { with_io_memory { |io| "123".center(io, 5) }.should eq(" 123 ") } + it { with_io_memory { |io| "12".center(io, 7, '-') }.should eq("--12---") } + it { with_io_memory { |io| "12".center(io, 7, 'あ') }.should eq("ああ12あああ") } + end + + describe "to io (deprecated)" do it { with_io_memory { |io| "123".center(2, io) }.should eq("123") } it { with_io_memory { |io| "123".center(5, io) }.should eq(" 123 ") } it { with_io_memory { |io| "12".center(7, '-', io) }.should eq("--12---") } diff --git a/src/array.cr b/src/array.cr index 9729a255ed86..c33bedb434ff 100644 --- a/src/array.cr +++ b/src/array.cr @@ -1875,7 +1875,7 @@ class Array(T) def to_s(io : IO) : Nil executed = exec_recursive(:to_s) do io << '[' - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << ']' end io << "[...]" unless executed diff --git a/src/char.cr b/src/char.cr index e450d4becd7f..c0a184010669 100644 --- a/src/char.cr +++ b/src/char.cr @@ -477,7 +477,7 @@ struct Char dump_or_inspect do |io| if ascii_control? io << "\\u{" - ord.to_s(16, io) + ord.to_s(io, 16) io << '}' else to_s(io) @@ -505,7 +505,7 @@ struct Char dump_or_inspect do |io| if ascii_control? || ord >= 0x80 io << "\\u{" - ord.to_s(16, io) + ord.to_s(io, 16) io << '}' else to_s(io) diff --git a/src/colorize.cr b/src/colorize.cr index 1d6bf38e0a0e..d7a7bd16f550 100644 --- a/src/colorize.cr +++ b/src/colorize.cr @@ -224,12 +224,16 @@ module Colorize blue : UInt8 do def fore(io : IO) : Nil io << "38;2;" - {red, green, blue}.join(';', io, &.to_s io) + io << red << ";" + io << green << ";" + io << blue end def back(io : IO) : Nil io << "48;2;" - {red, green, blue}.join(';', io, &.to_s io) + io << red << ";" + io << green << ";" + io << blue end end end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index 8ee47d118040..f2ba2a364560 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -2212,7 +2212,7 @@ module Crystal str << char else str << '.' - char.ord.to_s(16, str, upcase: true) + char.ord.to_s(str, 16, upcase: true) str << '.' end end diff --git a/src/compiler/crystal/exception.cr b/src/compiler/crystal/exception.cr index 5f3bb580d52a..0394d600f991 100644 --- a/src/compiler/crystal/exception.cr +++ b/src/compiler/crystal/exception.cr @@ -9,14 +9,14 @@ module Crystal @filename : String | VirtualFile | Nil def to_s(io) : Nil - to_s_with_source(nil, io) + to_s_with_source(io, nil) end def warning=(warning) @warning = !!warning end - abstract def to_s_with_source(source, io) + abstract def to_s_with_source(io : IO, source) def to_json(json : JSON::Builder) json.array do @@ -43,7 +43,7 @@ module Crystal def to_s_with_source(source) String.build do |io| - to_s_with_source source, io + to_s_with_source(io, source) end end @@ -77,11 +77,11 @@ module Crystal end class LocationlessException < Exception - def to_s_with_source(source, io) + def to_s_with_source(io : IO, source) io << @message end - def append_to_s(source, io) + def append_to_s(io : IO, source) io << @message end diff --git a/src/compiler/crystal/semantic/ast.cr b/src/compiler/crystal/semantic/ast.cr index 41819a51e2fd..19c84a3a4485 100644 --- a/src/compiler/crystal/semantic/ast.cr +++ b/src/compiler/crystal/semantic/ast.cr @@ -18,7 +18,7 @@ module Crystal String.build do |io| exception = exception_type.for_node(self, message, inner) exception.warning = true - exception.append_to_s(nil, io) + exception.append_to_s(io, nil) end end diff --git a/src/compiler/crystal/semantic/call_error.cr b/src/compiler/crystal/semantic/call_error.cr index 6e79ea10611e..6309874069ce 100644 --- a/src/compiler/crystal/semantic/call_error.cr +++ b/src/compiler/crystal/semantic/call_error.cr @@ -216,7 +216,7 @@ class Crystal::Call str << ".." str << all_arguments_sizes.last else - all_arguments_sizes.join ", ", str + all_arguments_sizes.join str, ", " end str << '+' if min_splat != Int32::MAX @@ -268,7 +268,7 @@ class Crystal::Call msg << " with type" msg << 's' if arg_types.size > 1 || named_args_types msg << ' ' - arg_types.join(", ", msg) + arg_types.join(msg, ", ") end if named_args_types diff --git a/src/compiler/crystal/semantic/exception.cr b/src/compiler/crystal/semantic/exception.cr index 39ba11028a78..223f1fa3acef 100644 --- a/src/compiler/crystal/semantic/exception.cr +++ b/src/compiler/crystal/semantic/exception.cr @@ -91,16 +91,16 @@ module Crystal io.flush end - def to_s_with_source(source, io) - append_to_s source, io + def to_s_with_source(io : IO, source) + append_to_s io, source end - def append_to_s(source, io) + def append_to_s(io : IO, source) inner = @inner unless @error_trace || inner.is_a? MethodTraceException if inner && inner.has_location? - return inner.append_to_s(source, io) + return inner.append_to_s(io, source) end end @@ -134,7 +134,7 @@ module Crystal return unless inner.has_location? io << "\n\n" io << '\n' unless inner.is_a? MethodTraceException - inner.append_to_s source, io + inner.append_to_s io, source end end @@ -183,8 +183,8 @@ module Crystal def to_json_single(json) end - def to_s_with_source(source, io) - append_to_s(source, io) + def to_s_with_source(io : IO, source) + append_to_s(io, source) end def has_trace? @@ -195,7 +195,7 @@ module Crystal @nil_reason || has_trace? && @show end - def append_to_s(source, io) + def append_to_s(io : IO, source) nil_reason = @nil_reason if !@show diff --git a/src/compiler/crystal/semantic/warnings.cr b/src/compiler/crystal/semantic/warnings.cr index 20039040ee47..74642087ab5d 100644 --- a/src/compiler/crystal/semantic/warnings.cr +++ b/src/compiler/crystal/semantic/warnings.cr @@ -15,7 +15,7 @@ module Crystal message = String.build do |io| exception = SyntaxException.new message, location.line_number, location.column_number, location.filename exception.warning = true - exception.append_to_s(nil, io) + exception.append_to_s(io, nil) end end diff --git a/src/compiler/crystal/syntax/exception.cr b/src/compiler/crystal/syntax/exception.cr index 007b77adef8d..9b66a0fd209c 100644 --- a/src/compiler/crystal/syntax/exception.cr +++ b/src/compiler/crystal/syntax/exception.cr @@ -31,7 +31,7 @@ module Crystal end end - def append_to_s(source, io) + def append_to_s(io : IO, source) msg = @message.to_s error_message_lines = msg.lines default_message = "syntax error in #{@filename}:#{@line_number}" @@ -42,8 +42,8 @@ module Crystal io << remaining error_message_lines end - def to_s_with_source(source, io) - append_to_s source, io + def to_s_with_source(io : IO, source) + append_to_s io, source end def deepest_error_message diff --git a/src/compiler/crystal/syntax/to_s.cr b/src/compiler/crystal/syntax/to_s.cr index cfc398c5ac8b..5ba97ee3b6af 100644 --- a/src/compiler/crystal/syntax/to_s.cr +++ b/src/compiler/crystal/syntax/to_s.cr @@ -125,7 +125,7 @@ module Crystal @str << '[' end - node.elements.join(", ", @str, &.accept self) + node.elements.join(@str, ", ", &.accept self) if name @str << '}' @@ -177,7 +177,7 @@ module Crystal def visit(node : NamedTupleLiteral) @str << '{' - node.entries.join(", ", @str) do |entry| + node.entries.join(@str, ", ") do |entry| visit_named_arg_name(entry.key) @str << ": " entry.value.accept self @@ -396,7 +396,7 @@ module Crystal if node.name.ends_with?('=') && node.name[0].ascii_letter? @str << decorate_call(node, node.name.rchop) @str << " = " - node.args.join(", ", @str, &.accept self) + node.args.join(@str, ", ", &.accept self) else @str << decorate_call(node, node.name) @@ -592,9 +592,9 @@ module Crystal end def visit(node : MultiAssign) - node.targets.join(", ", @str, &.accept self) + node.targets.join(@str, ", ", &.accept self) @str << " = " - node.values.join(", ", @str, &.accept self) + node.values.join(@str, ", ", &.accept self) false end @@ -631,7 +631,7 @@ module Crystal @str << "->" if node.def.args.size > 0 @str << '(' - node.def.args.join(", ", @str, &.accept self) + node.def.args.join(@str, ", ", &.accept self) @str << ')' end @str << ' ' @@ -653,7 +653,7 @@ module Crystal if node.args.size > 0 @str << '(' - node.args.join(", ", @str, &.accept self) + node.args.join(@str, ", ", &.accept self) @str << ')' end false @@ -698,7 +698,7 @@ module Crystal if free_vars = node.free_vars @str << " forall " - free_vars.join(", ", @str) + free_vars.join(@str, ", ") end newline @@ -779,7 +779,7 @@ module Crystal def visit(node : MacroFor) @str << "{% for " - node.vars.join(", ", @str, &.accept self) + node.vars.join(@str, ", ", &.accept self) @str << " in " node.exp.accept self @str << " %}" @@ -795,7 +795,7 @@ module Crystal @str << node.name if exps = node.exps @str << '{' - exps.join(", ", @str, &.accept self) + exps.join(@str, ", ", &.accept self) @str << '}' end false @@ -855,7 +855,7 @@ module Crystal def visit(node : ProcNotation) @str << '(' if inputs = node.inputs - inputs.join(", ", @str, &.accept self) + inputs.join(@str, ", ", &.accept self) @str << ' ' end @str << "-> " @@ -872,7 +872,7 @@ module Crystal def visit(node : Path) @str << "::" if node.global? - node.names.join("::", @str) + node.names.join(@str, "::") end def visit(node : Generic) @@ -902,7 +902,7 @@ module Crystal printed_arg = false @str << '(' - node.type_vars.join(", ", @str) do |var| + node.type_vars.join(@str, ", ") do |var| var.accept self printed_arg = true end @@ -947,7 +947,7 @@ module Crystal end def visit(node : Union) - node.types.join(" | ", @str, &.accept self) + node.types.join(@str, " | ", &.accept self) false end @@ -982,7 +982,7 @@ module Crystal @str << keyword("yield") if node.exps.size > 0 @str << ' ' - node.exps.join(", ", @str, &.accept self) + node.exps.join(@str, ", ", &.accept self) end false end @@ -1039,7 +1039,7 @@ module Crystal first = node.elements.first? space = first.is_a?(TupleLiteral) || first.is_a?(NamedTupleLiteral) || first.is_a?(HashLiteral) @str << ' ' if space - node.elements.join(", ", @str, &.accept self) + node.elements.join(@str, ", ", &.accept self) @str << ' ' if space @str << '}' false @@ -1167,7 +1167,7 @@ module Crystal end if node.args.size > 0 @str << '(' - node.args.join(", ", @str) do |arg| + node.args.join(@str, ", ") do |arg| if arg_name = arg.name @str << arg_name << " : " end @@ -1363,7 +1363,7 @@ module Crystal append_indent @str << keyword(node.exhaustive? ? "in" : "when") @str << ' ' - node.conds.join(", ", @str, &.accept self) + node.conds.join(@str, ", ", &.accept self) newline accept_with_indent node.body false @@ -1433,7 +1433,7 @@ module Crystal @str << " :" end @str << ' ' - types.join(" | ", @str, &.accept self) + types.join(@str, " | ", &.accept self) end newline accept_with_indent node.body @@ -1452,7 +1452,7 @@ module Crystal def visit(node : TypeOf) @str << keyword("typeof") @str << '(' - node.expressions.join(", ", @str, &.accept self) + node.expressions.join(@str, ", ", &.accept self) @str << ')' false end @@ -1463,7 +1463,7 @@ module Crystal if !node.args.empty? || node.named_args @str << '(' printed_arg = false - node.args.join(", ", @str) do |arg| + node.args.join(@str, ", ") do |arg| arg.accept self printed_arg = true end @@ -1492,19 +1492,19 @@ module Crystal @str << " :" if outputs = node.outputs @str << ' ' - outputs.join(", ", @str, &.accept self) + outputs.join(@str, ", ", &.accept self) @str << ' ' end @str << ':' if inputs = node.inputs @str << ' ' - inputs.join(", ", @str, &.accept self) + inputs.join(@str, ", ", &.accept self) @str << ' ' end @str << ":" if clobbers = node.clobbers @str << ' ' - clobbers.join(", ", @str, &.inspect @str) + clobbers.join(@str, ", ", &.inspect @str) @str << ' ' end @str << ":" diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 6ebfcc437b9f..77c8a3c4c50d 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -365,7 +365,7 @@ class Crystal::Doc::Generator def isolate_flag_lines(string) flag_regexp = /^ ?(#{FLAGS.join('|')}):?/ String.build do |io| - string.each_line(chomp: false).join("", io) do |line, io| + string.each_line(chomp: false).join(io) do |line, io| if line =~ flag_regexp io << '\n' << line else diff --git a/src/compiler/crystal/tools/doc/method.cr b/src/compiler/crystal/tools/doc/method.cr index 4cc1341e19dd..629845c3e8a5 100644 --- a/src/compiler/crystal/tools/doc/method.cr +++ b/src/compiler/crystal/tools/doc/method.cr @@ -262,7 +262,7 @@ class Crystal::Doc::Method if free_vars = @def.free_vars io << " forall " - free_vars.join(", ", io) + free_vars.join(io, ", ") end io diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index faeac5d7a944..8809a50f6fe2 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -515,7 +515,7 @@ class Crystal::Doc::Type io << node.name end io << '(' - node.type_vars.join(", ", io) do |type_var| + node.type_vars.join(io, ", ") do |type_var| node_to_html type_var, io, links: links end io << ')' @@ -523,7 +523,7 @@ class Crystal::Doc::Type def node_to_html(node : ProcNotation, io, links = true) if inputs = node.inputs - inputs.join(", ", io) do |input| + inputs.join(io, ", ") do |input| node_to_html input, io, links: links end end @@ -544,7 +544,7 @@ class Crystal::Doc::Type end end - node.types.join(" | ", io) do |elem| + node.types.join(io, " | ") do |elem| node_to_html elem, io, links: links end end @@ -597,7 +597,7 @@ class Crystal::Doc::Type separator = " | " end - type.union_types.join(separator, io) do |union_type| + type.union_types.join(io, separator) do |union_type| type_to_html union_type, io, text, links: links end @@ -605,7 +605,7 @@ class Crystal::Doc::Type end def type_to_html(type : Crystal::ProcInstanceType, io, text = nil, links = true) - type.arg_types.join(", ", io) do |arg_type| + type.arg_types.join(io, ", ") do |arg_type| type_to_html arg_type, io, links: links end io << " -> " @@ -615,7 +615,7 @@ class Crystal::Doc::Type def type_to_html(type : Crystal::TupleInstanceType, io, text = nil, links = true) io << '{' - type.tuple_types.join(", ", io) do |tuple_type| + type.tuple_types.join(io, ", ") do |tuple_type| type_to_html tuple_type, io, links: links end io << '}' @@ -623,7 +623,7 @@ class Crystal::Doc::Type def type_to_html(type : Crystal::NamedTupleInstanceType, io, text = nil, links = true) io << '{' - type.entries.join(", ", io) do |entry| + type.entries.join(io, ", ") do |entry| if Symbol.needs_quotes?(entry.name) entry.name.inspect(io) else @@ -655,7 +655,7 @@ class Crystal::Doc::Type io << "" if must_be_included && links && has_link_in_type_vars io << '(' - type.type_vars.values.join(", ", io) do |type_var| + type.type_vars.values.join(io, ", ") do |type_var| case type_var when Var type_to_html type_var.type, io, links: links diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 84ad3e1f3159..3d3038f86133 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -1793,7 +1793,7 @@ module Crystal super if generic_args io << '(' - type_vars.join(", ", io, &.to_s(io)) + type_vars.join(io, ", ", &.to_s(io)) io << ')' end end @@ -1853,7 +1853,7 @@ module Crystal super if generic_args io << '(' - type_vars.join(", ", io, &.to_s(io)) + type_vars.join(io, ", ", &.to_s(io)) io << ')' end end @@ -1987,7 +1987,7 @@ module Crystal if type_var.is_a?(Var) if i == splat_index tuple = type_var.type.as(TupleInstanceType) - tuple.tuple_types.join(", ", io) do |tuple_type| + tuple.tuple_types.join(io, ", ") do |tuple_type| tuple_type = tuple_type.devirtualize unless codegen tuple_type.to_s_with_options(io, codegen: codegen) end @@ -2396,7 +2396,7 @@ module Crystal def to_s_with_options(io : IO, skip_union_parens : Bool = false, generic_args : Bool = true, codegen : Bool = false) : Nil io << "Tuple(" - @tuple_types.join(", ", io) do |tuple_type| + @tuple_types.join(io, ", ") do |tuple_type| tuple_type = tuple_type.devirtualize unless codegen tuple_type.to_s_with_options(io, skip_union_parens: true, codegen: codegen) end @@ -2513,7 +2513,7 @@ module Crystal def to_s_with_options(io : IO, skip_union_parens : Bool = false, generic_args : Bool = true, codegen : Bool = false) : Nil io << "NamedTuple(" - @entries.join(", ", io) do |entry| + @entries.join(io, ", ") do |entry| if Symbol.needs_quotes?(entry.name) entry.name.inspect(io) else @@ -3063,7 +3063,7 @@ module Crystal union_types = @union_types.dup union_types << union_types.delete_at(nil_type_index) end - union_types.join(" | ", io) do |type| + union_types.join(io, " | ") do |type| type = type.devirtualize unless codegen type.to_s_with_options(io, codegen: codegen) end diff --git a/src/deque.cr b/src/deque.cr index e1bbba548c3d..1300149e7aea 100644 --- a/src/deque.cr +++ b/src/deque.cr @@ -336,7 +336,7 @@ class Deque(T) def inspect(io : IO) : Nil executed = exec_recursive(:inspect) do io << "Deque{" - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << '}' end io << "Deque{...}" unless executed diff --git a/src/enumerable.cr b/src/enumerable.cr index f2be7b2a4117..eb407b00cdf8 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -723,7 +723,7 @@ module Enumerable(T) # ``` def join(separator = "") String.build do |io| - join separator, io + join io, separator end end @@ -733,9 +733,9 @@ module Enumerable(T) # ``` # [1, 2, 3, 4, 5].join(", ") { |i| -i } # => "-1, -2, -3, -4, -5" # ``` - def join(separator = "") + def join(separator = "", & : T ->) String.build do |io| - join(separator, io) do |elem| + join(io, separator) do |elem| io << yield elem end end @@ -744,7 +744,7 @@ module Enumerable(T) # Prints to *io* all the elements in the collection, separated by *separator*. # # ``` - # [1, 2, 3, 4, 5].join(", ", STDOUT) + # [1, 2, 3, 4, 5].join(STDOUT, ", ") # ``` # # Prints: @@ -752,17 +752,33 @@ module Enumerable(T) # ```text # 1, 2, 3, 4, 5 # ``` - def join(separator, io) - join(separator, io) do |elem| + def join(io : IO, separator = "") + join(io, separator) do |elem| elem.to_s(io) end end + # Prints to *io* all the elements in the collection, separated by *separator*. + # + # ``` + # [1, 2, 3, 4, 5].join(STDOUT, ", ") + # ``` + # + # Prints: + # + # ```text + # 1, 2, 3, 4, 5 + # ``` + @[Deprecated(%(Use `#join(io : IO, separator = "") instead`))] + def join(separator, io : IO) + join(io, separator) + end + # Prints to *io* the concatenation of the elements, with the possibility of # controlling how the printing is done via a block. # # ``` - # [1, 2, 3, 4, 5].join(", ", STDOUT) { |i, io| io << "(#{i})" } + # [1, 2, 3, 4, 5].join(STDOUT, ", ") { |i, io| io << "(#{i})" } # ``` # # Prints: @@ -770,13 +786,32 @@ module Enumerable(T) # ```text # (1), (2), (3), (4), (5) # ``` - def join(separator, io) + def join(io : IO, separator = "", & : T, IO ->) each_with_index do |elem, i| io << separator if i > 0 yield elem, io end end + # Prints to *io* the concatenation of the elements, with the possibility of + # controlling how the printing is done via a block. + # + # ``` + # [1, 2, 3, 4, 5].join(STDOUT, ", ") { |i, io| io << "(#{i})" } + # ``` + # + # Prints: + # + # ```text + # (1), (2), (3), (4), (5) + # ``` + @[Deprecated(%(Use `#join(io : IO, separator = "", & : T, IO ->) instead`))] + def join(separator, io : IO) + join(io, separator) do |elem, io| + yield elem, io + end + end + # Returns an `Array` with the results of running the block against each element of the collection. # # ``` diff --git a/src/fiber.cr b/src/fiber.cr index 675352e14574..4bb622300402 100644 --- a/src/fiber.cr +++ b/src/fiber.cr @@ -273,7 +273,7 @@ class Fiber def to_s(io : IO) : Nil io << "#<" << self.class.name << ":0x" - object_id.to_s(16, io) + object_id.to_s(io, 16) if name = @name io << ": " << name end diff --git a/src/http/common.cr b/src/http/common.cr index f218c097b6a3..896ba17c751a 100644 --- a/src/http/common.cr +++ b/src/http/common.cr @@ -293,7 +293,7 @@ module HTTP def self.serialize_chunked_body(io, body) buf = uninitialized UInt8[8192] while (buf_length = body.read(buf.to_slice)) > 0 - buf_length.to_s(16, io) + buf_length.to_s(io, 16) io << "\r\n" io.write(buf.to_slice[0, buf_length]) io << "\r\n" diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 2d1c171bb022..c0e0a18d3b42 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -203,7 +203,7 @@ class HTTP::Server ensure_headers_written if @chunked - slice.size.to_s(16, @io) + slice.size.to_s(@io, 16) @io << "\r\n" @io.write(slice) @io << "\r\n" diff --git a/src/int.cr b/src/int.cr index da9692551fc3..50a1c35b498d 100644 --- a/src/int.cr +++ b/src/int.cr @@ -566,15 +566,7 @@ struct Int private DIGITS_UPCASE = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" private DIGITS_BASE62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - def to_s : String - to_s(10) - end - - def to_s(io : IO) : Nil - to_s(10, io) - end - - def to_s(base : Int, upcase : Bool = false) : String + def to_s(base : Int = 10, *, upcase : Bool = false) : String raise ArgumentError.new("Invalid base #{base}") unless 2 <= base <= 36 || base == 62 raise ArgumentError.new("upcase must be false for base 62") if upcase && base == 62 @@ -590,7 +582,12 @@ struct Int end end - def to_s(base : Int, io : IO, upcase : Bool = false) : Nil + @[Deprecated("Use `#to_s(base : Int, *, upcase : Bool = false)` instead")] + def to_s(base : Int, _upcase : Bool) : String + to_s(base, upcase: _upcase) + end + + def to_s(io : IO, base : Int = 10, *, upcase : Bool = false) : Nil raise ArgumentError.new("Invalid base #{base}") unless 2 <= base <= 36 || base == 62 raise ArgumentError.new("upcase must be false for base 62") if upcase && base == 62 @@ -606,6 +603,11 @@ struct Int end end + @[Deprecated("Use `#to_s(io : IO, base : Int, *, upcase : Bool = false)` instead")] + def to_s(base : Int, io : IO, upcase : Bool = false) : Nil + to_s(io, base, upcase: upcase) + end + private def internal_to_s(base, upcase = false) # Given sizeof(self) <= 128 bits, we need at most 128 bytes for a base 2 # representation, plus one byte for the trailing 0. diff --git a/src/json/builder.cr b/src/json/builder.cr index 8f6dc9f79a1d..0d8d2017d1af 100644 --- a/src/json/builder.cr +++ b/src/json/builder.cr @@ -131,7 +131,7 @@ class JSON::Builder io << '0' if ord < 0x1000 io << '0' if ord < 0x100 io << '0' if ord < 0x10 - ord.to_s(16, io) + ord.to_s(io, 16) reader.next_char start_pos = reader.pos next diff --git a/src/option_parser.cr b/src/option_parser.cr index f0a4f1cecc83..7962e776df51 100644 --- a/src/option_parser.cr +++ b/src/option_parser.cr @@ -274,7 +274,7 @@ class OptionParser io << banner io << '\n' end - @flags.join '\n', io + @flags.join io, '\n' end private def append_flag(flag, description) diff --git a/src/pointer.cr b/src/pointer.cr index 87580a2c865a..b808da0ffda8 100644 --- a/src/pointer.cr +++ b/src/pointer.cr @@ -326,7 +326,7 @@ struct Pointer(T) io << ".null" else io << "@0x" - address.to_s(16, io) + address.to_s(io, 16) end end diff --git a/src/proc.cr b/src/proc.cr index 4240842b9da3..b73062f3a5b1 100644 --- a/src/proc.cr +++ b/src/proc.cr @@ -194,7 +194,7 @@ struct Proc io << "#<" io << {{@type.name.stringify}} io << ":0x" - pointer.address.to_s(16, io) + pointer.address.to_s(io, 16) if closure? io << ":closure" end diff --git a/src/reference.cr b/src/reference.cr index d6324c2ff550..8a64357fc112 100644 --- a/src/reference.cr +++ b/src/reference.cr @@ -70,7 +70,7 @@ class Reference # ``` def inspect(io : IO) : Nil io << "#<" << {{@type.name.id.stringify}} << ":0x" - object_id.to_s(16, io) + object_id.to_s(io, 16) executed = exec_recursive(:inspect) do {% for ivar, i in @type.instance_vars %} @@ -129,7 +129,7 @@ class Reference # ``` def to_s(io : IO) : Nil io << "#<" << self.class.name << ":0x" - object_id.to_s(16, io) + object_id.to_s(io, 16) io << '>' end diff --git a/src/semantic_version.cr b/src/semantic_version.cr index 82c2197dbd2c..8828a7d8447c 100644 --- a/src/semantic_version.cr +++ b/src/semantic_version.cr @@ -147,7 +147,7 @@ struct SemanticVersion # semver.prerelease.to_s # => "rc.1" # ``` def to_s(io : IO) : Nil - identifiers.join('.', io) + identifiers.join(io, '.') end # The comparison operator. diff --git a/src/set.cr b/src/set.cr index f17d4cca390d..581df71eed82 100644 --- a/src/set.cr +++ b/src/set.cr @@ -404,7 +404,7 @@ struct Set(T) # Writes a string representation of the set to *io*. def to_s(io : IO) : Nil io << "Set{" - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << '}' end diff --git a/src/slice.cr b/src/slice.cr index 0c011d31a6a7..0fae0e55155b 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -596,11 +596,11 @@ struct Slice(T) if T == UInt8 io << "Bytes[" # Inspect using to_s because we know this is a UInt8. - join ", ", io, &.to_s(io) + join io, ", ", &.to_s(io) io << ']' else io << "Slice[" - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << ']' end end diff --git a/src/static_array.cr b/src/static_array.cr index 40936688fa16..646b31da09a0 100644 --- a/src/static_array.cr +++ b/src/static_array.cr @@ -267,7 +267,7 @@ struct StaticArray(T, N) # ``` def to_s(io : IO) : Nil io << "StaticArray[" - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << ']' end diff --git a/src/string.cr b/src/string.cr index e057e08ee2b1..00075fe93635 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3824,8 +3824,9 @@ class String # "Purple".ljust(8, io) # io.to_s # => "Purple " # ``` + @[Deprecated("Use `#ljust(io :IO, len : Int, char : Char = ' ')` instead")] def ljust(len : Int, io : IO) : Nil - ljust(len, ' ', io) + ljust(io, len) end # Adds instances of *char* to right of the string until it is at least size of *len*, @@ -3833,14 +3834,27 @@ class String # # ``` # io = IO::Memory.new - # "Purple".ljust(8, '-', io) + # "Purple".ljust(io, 8, '-') # io.to_s # => "Purple--" # ``` - def ljust(len : Int, char : Char, io : IO) : Nil + def ljust(io : IO, len : Int, char : Char = ' ') : Nil io << self (len - size).times { io << char } end + # Adds instances of *char* to right of the string until it is at least size of *len*, + # and then appends the result to the given IO. + # + # ``` + # io = IO::Memory.new + # "Purple".ljust(8, '-', io) + # io.to_s # => "Purple--" + # ``` + @[Deprecated("Use `#ljust(io :IO, len : Int, char : Char = ' ')` instead")] + def ljust(len : Int, char : Char, io : IO) : Nil + ljust(io, len, char) + end + # Adds instances of *char* to left of the string until it is at least size of *len*. # # ``` @@ -3860,8 +3874,9 @@ class String # "Purple".rjust(8, io) # io.to_s # => " Purple" # ``` + @[Deprecated("Use `#rjust(io :IO, len : Int, char : Char = ' ')` instead")] def rjust(len : Int, io : IO) : Nil - rjust(len, ' ', io) + rjust(io, len) end # Adds instances of *char* to left of the string until it is at least size of *len*, @@ -3872,11 +3887,24 @@ class String # "Purple".rjust(8, '-', io) # io.to_s # => "--Purple" # ``` - def rjust(len : Int, char : Char, io : IO) : Nil + def rjust(io : IO, len : Int, char : Char = ' ') : Nil (len - size).times { io << char } io << self end + # Adds instances of *char* to left of the string until it is at least size of *len*, + # and then appends the result to the given IO. + # + # ``` + # io = IO::Memory.new + # "Purple".rjust(8, '-', io) + # io.to_s # => "--Purple" + # ``` + @[Deprecated("Use `#rjust(io :IO, len : Int, char : Char = ' ')` instead")] + def rjust(len : Int, char : Char, io : IO) : Nil + rjust(io, len, char) + end + # Adds instances of *char* to left and right of the string until it is at least size of *len*. # # ``` @@ -3897,8 +3925,9 @@ class String # "Purple".center(9, io) # io.to_s # => " Purple " # ``` + @[Deprecated("Use `#center(io :IO, len : Int, char : Char = ' ')` instead")] def center(len : Int, io : IO) : Nil - center(len, ' ', io) + center(io, len) end # Adds instances of *char* to left and right of the string until it is at least size of *len*, @@ -3909,7 +3938,7 @@ class String # "Purple".center(9, '-', io) # io.to_s # => "-Purple--" # ``` - def center(len : Int, char : Char, io : IO) : Nil + def center(io : IO, len : Int, char : Char = ' ') : Nil difference = len - size if difference <= 0 @@ -3925,6 +3954,19 @@ class String right_padding.times { io << char } end + # Adds instances of *char* to left ond right of the string until it is at least size of *len*, + # then appends the result to the given IO. + # + # ``` + # io = IO::Memory.new + # "Purple".center(9, '-', io) + # io.to_s # => "-Purple--" + # ``` + @[Deprecated("Use `#center(io :IO, len : Int, char : Char = ' ')` instead")] + def center(len : Int, char : Char, io : IO) : Nil + center(io, len, char) + end + private def just(len, char, justify) return self if size >= len @@ -4430,7 +4472,7 @@ class String private def dump_hex(char, io) io << "\\x" io << '0' if char < 0x0F - char.to_s(16, io, upcase: true) + char.to_s(io, 16, upcase: true) end private def dump_unicode(char, io) @@ -4439,7 +4481,7 @@ class String io << '0' if char.ord < 0x1000 io << '0' if char.ord < 0x0100 io << '0' if char.ord < 0x0010 - char.ord.to_s(16, io, upcase: true) + char.ord.to_s(io, 16, upcase: true) io << '}' if char.ord > 0xFFFF end diff --git a/src/string/formatter.cr b/src/string/formatter.cr index 3ac9ef6d9f02..34a16cb44907 100644 --- a/src/string/formatter.cr +++ b/src/string/formatter.cr @@ -243,7 +243,7 @@ struct String::Formatter(A) end end - int.to_s(flags.base, @io, upcase: flags.type == 'X') + int.to_s(@io, flags.base, upcase: flags.type == 'X') if flags.right_padding? pad_int int, flags diff --git a/src/time.cr b/src/time.cr index bbeac6258e94..c95890f258b0 100644 --- a/src/time.cr +++ b/src/time.cr @@ -1084,10 +1084,18 @@ struct Time # Formats this `Time` according to the pattern in *format* to the given *io*. # # See `Time::Format` for details. - def to_s(format : String, io : IO) : Nil + def to_s(io : IO, format : String) : Nil Format.new(format).format(self, io) end + # Formats this `Time` according to the pattern in *format* to the given *io*. + # + # See `Time::Format` for details. + @[Deprecated("Use `#to_s(io : IO, format : String)` instead")] + def to_s(format : String, io : IO) : Nil + to_s(io, format) + end + # Format this time using the format specified by [RFC 3339](https://tools.ietf.org/html/rfc3339) ([ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf) profile). # # ``` diff --git a/src/tuple.cr b/src/tuple.cr index 496f5a346e55..6288983dd81e 100644 --- a/src/tuple.cr +++ b/src/tuple.cr @@ -393,7 +393,7 @@ struct Tuple # ``` def to_s(io : IO) : Nil io << '{' - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << '}' end diff --git a/src/uri.cr b/src/uri.cr index 97e76e1b918d..2ccb8af26e17 100644 --- a/src/uri.cr +++ b/src/uri.cr @@ -446,7 +446,7 @@ class URI elsif dst_path.first.includes?(':') # (see RFC2396 Section 5) String.build do |io| io << "./" - dst_path.join('/', io) + dst_path.join(io, '/') end else string = dst_path.join('/') @@ -459,7 +459,7 @@ class URI else String.build do |io| base_path.size.times { io << "../" } - dst_path.join('/', io) + dst_path.join(io, '/') end end end diff --git a/src/uri/encoding.cr b/src/uri/encoding.cr index f04c90450da7..538ae0f85129 100644 --- a/src/uri/encoding.cr +++ b/src/uri/encoding.cr @@ -222,7 +222,7 @@ class URI else io.write_byte '%'.ord.to_u8 io.write_byte '0'.ord.to_u8 if byte < 16 - byte.to_s(16, io, upcase: true) + byte.to_s(io, 16, upcase: true) end end io diff --git a/src/xml/attributes.cr b/src/xml/attributes.cr index 146829aeea93..07226f1e46b2 100644 --- a/src/xml/attributes.cr +++ b/src/xml/attributes.cr @@ -60,7 +60,7 @@ struct XML::Attributes def to_s(io : IO) : Nil io << '[' - join ", ", io, &.inspect(io) + join io, ", ", &.inspect(io) io << ']' end diff --git a/src/xml/namespace.cr b/src/xml/namespace.cr index 8f977dd5817a..5bcd22995fa8 100644 --- a/src/xml/namespace.cr +++ b/src/xml/namespace.cr @@ -21,7 +21,7 @@ struct XML::Namespace def to_s(io : IO) : Nil io << "# Date: Wed, 13 May 2020 14:19:18 -0300 Subject: [PATCH 030/263] Followup of #9134. Missing swap of arguments (#9289) --- src/log/format.cr | 2 +- src/time.cr | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/log/format.cr b/src/log/format.cr index 37facfb47e2e..30275e6284b1 100644 --- a/src/log/format.cr +++ b/src/log/format.cr @@ -81,7 +81,7 @@ class Log # This writes the severity in uppercase and left padded # with enough space so all the severities fit def severity - @entry.severity.label.rjust(7, @io) + @entry.severity.label.rjust(@io, 7) end # Write the source for non-root entries diff --git a/src/time.cr b/src/time.cr index c95890f258b0..c4264019fb21 100644 --- a/src/time.cr +++ b/src/time.cr @@ -1034,13 +1034,13 @@ struct Time # # The name of the location is appended unless it is a fixed zone offset. def inspect(io : IO, with_nanoseconds = true) : Nil - to_s "%F %T", io + to_s io, "%F %T" if with_nanoseconds if @nanoseconds == 0 io << ".0" else - to_s ".%N", io + to_s io, ".%N" end end @@ -1060,7 +1060,7 @@ struct Time # When the location is `UTC`, the offset is replaced with the string `UTC`. # Offset seconds are omitted if `0`. def to_s(io : IO) : Nil - to_s("%F %T ", io) + to_s(io, "%F %T ") if utc? io << "UTC" From 28985640798ae39e6c12204ca7911e1053d2d8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Wed, 13 May 2020 21:44:59 +0200 Subject: [PATCH 031/263] Refactor Docs::Generator source link generation (#9119) * Replace Docs::Generator#repository_name with ProjectInfo#name * Add ProjectInfo#refname * Add ProjectInfo#source_url_pattern * Refactor docs generator to use ProjectInfo#source_url * Add CLI options for refname and url pattern --- Makefile | 2 +- man/crystal.1 | 24 +++ .../crystal/tools/doc/project_info_spec.cr | 162 +++++++++++++++--- spec/compiler/crystal/tools/doc_spec.cr | 45 ----- src/compiler/crystal/command/docs.cr | 8 + src/compiler/crystal/tools/doc/generator.cr | 122 ++----------- src/compiler/crystal/tools/doc/html/type.html | 8 +- src/compiler/crystal/tools/doc/main.cr | 4 +- .../crystal/tools/doc/project_info.cr | 101 ++++++++++- .../crystal/tools/doc/relative_location.cr | 52 ++++++ src/compiler/crystal/tools/doc/type.cr | 4 +- 11 files changed, 342 insertions(+), 190 deletions(-) create mode 100644 src/compiler/crystal/tools/doc/relative_location.cr diff --git a/Makefile b/Makefile index 607dc23c3137..0222da6f4ad5 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ compiler_spec: $(O)/compiler_spec ## Run compiler specs .PHONY: docs docs: ## Generate standard library documentation - $(BUILD_PATH) ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS) --project-name=Crystal --project-version=$(CRYSTAL_VERSION) + $(BUILD_PATH) ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS) --project-name=Crystal --project-version=$(CRYSTAL_VERSION) --source-refname=$(CRYSTAL_CONFIG_BUILD_COMMIT) .PHONY: crystal crystal: $(O)/crystal ## Build the compiler diff --git a/man/crystal.1 b/man/crystal.1 index 5a69bb5b6807..c13644503021 100644 --- a/man/crystal.1 +++ b/man/crystal.1 @@ -176,12 +176,36 @@ Options: .Pp .It Fl -project-name Ar NAME Set the project name. The default value is extracted from shard.yml if available. + In case no default can be found, this option is mandatory. .It Fl -project-version Ar VERSION Set the project version. The default value is extracted from current git commit or shard.yml if available. + In case no default can be found, this option is mandatory. .It Fl -json-config-url Ar URL Set the URL pointing to a config file (used for discovering versions). +.It Fl -source-refname Ar REFNAME +Set source refname (e.g. git tag, commit hash). The default value is extracted from current git commit if available. + +If this option is missing and can't be automatically determined, the generator can't produce source code links. +.It Fl -source-url-pattern Ar URL +Set URL pattern for source code links. The default value is extracted from git remotes ("origin" or first one) if available and the provider's URL pattern is recognized. + +.Pp +Supported replacement tags: +.Pp +.Bl -tag -width "%{refname}" -compact +.It Sy %{refname} +commit reference +.It Sy %{path} +path to source file inside the repository +.It Sy %{filename} +basename of the source file +.It Sy %{line} +line number +.El +.Pp +If this option is missing and can't be automatically determined, the generator can't produce source code links. .It Fl o Ar DIR, Fl -output Ar DIR Set the output directory (default: ./docs). .It Fl b Ar URL, Fl -sitemap-base-url Ar URL diff --git a/spec/compiler/crystal/tools/doc/project_info_spec.cr b/spec/compiler/crystal/tools/doc/project_info_spec.cr index 577f442ce4d8..439639f0b9de 100644 --- a/spec/compiler/crystal/tools/doc/project_info_spec.cr +++ b/spec/compiler/crystal/tools/doc/project_info_spec.cr @@ -24,8 +24,8 @@ describe Crystal::Doc::ProjectInfo do describe "#fill_with_defaults" do it "empty folder" do - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new(nil, nil)) - assert_with_defaults(ProjectInfo.new("foo", "1.0"), ProjectInfo.new("foo", "1.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new(nil, nil, refname: nil)) + assert_with_defaults(ProjectInfo.new("foo", "1.0"), ProjectInfo.new("foo", "1.0", refname: nil)) end context "with shard.yml" do @@ -34,17 +34,17 @@ describe Crystal::Doc::ProjectInfo do end it "no git" do - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "1.0")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) - assert_with_defaults(ProjectInfo.new(nil, "2.0"), ProjectInfo.new("foo", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "1.0", refname: nil)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: nil)) + assert_with_defaults(ProjectInfo.new(nil, "2.0"), ProjectInfo.new("foo", "2.0", refname: nil)) end it "git but no commit" do run_git "init" - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", nil)) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) - assert_with_defaults(ProjectInfo.new(nil, "2.0"), ProjectInfo.new("foo", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", nil, refname: nil)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: nil)) + assert_with_defaults(ProjectInfo.new(nil, "2.0"), ProjectInfo.new("foo", "2.0", refname: nil)) end it "git tagged version" do @@ -53,8 +53,9 @@ describe Crystal::Doc::ProjectInfo do run_git "commit -m 'Initial commit' --no-gpg-sign" run_git "tag v3.0" - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "3.0")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "3.0", refname: "v3.0")) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: "v3.0")) + assert_with_defaults(ProjectInfo.new("bar", "2.0", refname: "12345"), ProjectInfo.new("bar", "2.0", refname: "12345")) end it "git tagged version dirty" do @@ -64,19 +65,21 @@ describe Crystal::Doc::ProjectInfo do run_git "tag v3.0" File.write("foo.txt", "bar") - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "3.0-dev")) - assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "3.0-dev", refname: nil)) + assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1", refname: nil)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: nil)) end it "git non-tagged commit" do run_git "init" run_git "add shard.yml" run_git "commit -m 'Initial commit' --no-gpg-sign" + commit_sha = `git rev-parse HEAD`.chomp - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master")) - assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master", refname: commit_sha)) + assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1", refname: commit_sha)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: commit_sha)) + assert_with_defaults(ProjectInfo.new("bar", "2.0", refname: "12345"), ProjectInfo.new("bar", "2.0", refname: "12345")) end it "git non-tagged commit dirty" do @@ -85,9 +88,20 @@ describe Crystal::Doc::ProjectInfo do run_git "commit -m 'Initial commit' --no-gpg-sign" File.write("foo.txt", "bar") - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master-dev")) - assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master-dev", refname: nil)) + assert_with_defaults(ProjectInfo.new(nil, "1.1"), ProjectInfo.new("foo", "1.1", refname: nil)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: nil)) + end + + it "git with remote" do + run_git "init" + run_git "remote add origin git@github.com:foo/bar" + + url_pattern = "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", nil, refname: nil, source_url_pattern: url_pattern)) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: nil, source_url_pattern: url_pattern)) + assert_with_defaults(ProjectInfo.new(nil, "2.0"), ProjectInfo.new("foo", "2.0", refname: nil, source_url_pattern: url_pattern)) + assert_with_defaults(ProjectInfo.new(nil, "2.0", source_url_pattern: "foo_bar"), ProjectInfo.new("foo", "2.0", refname: nil, source_url_pattern: "foo_bar")) end end @@ -98,9 +112,10 @@ describe Crystal::Doc::ProjectInfo do run_git "commit -m 'Remove shard.yml' --no-gpg-sign" run_git "tag v4.0" - assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new(nil, "4.0")) - assert_with_defaults(ProjectInfo.new("foo", nil), ProjectInfo.new("foo", "4.0")) - assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0")) + assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new(nil, "4.0", refname: "v4.0")) + assert_with_defaults(ProjectInfo.new("foo", nil), ProjectInfo.new("foo", "4.0", refname: "v4.0")) + assert_with_defaults(ProjectInfo.new("bar", "2.0"), ProjectInfo.new("bar", "2.0", refname: "v4.0")) + assert_with_defaults(ProjectInfo.new("bar", "2.0", refname: "12345"), ProjectInfo.new("bar", "2.0", refname: "12345")) end end @@ -149,6 +164,40 @@ describe Crystal::Doc::ProjectInfo do ProjectInfo.find_git_version.should eq "0.1.0" end + describe ".git_remote" do + it "no git workdir" do + ProjectInfo.git_remote.should be_nil + end + + it "no remote" do + run_git "init" + ProjectInfo.git_remote.should be_nil + end + + it "simple origin" do + run_git "init" + run_git "remote add origin https://example.com/foo.git" + ProjectInfo.git_remote.should eq "https://example.com/foo.git" + end + + it "origin plus other" do + run_git "init" + run_git "remote add bar https://example.com/bar.git" + run_git "remote add origin https://example.com/foo.git" + run_git "remote add baz https://example.com/baz.git" + `git remote -v` + ProjectInfo.git_remote.should eq "https://example.com/foo.git" + end + + it "no origin remote" do + run_git "init" + run_git "remote add bar https://example.com/bar.git" + run_git "remote add baz https://example.com/baz.git" + `git remote -v` + ProjectInfo.git_remote.should eq "https://example.com/bar.git" + end + end + describe ".read_shard_properties" do it "no shard.yml" do ProjectInfo.read_shard_properties.should eq({nil, nil}) @@ -202,4 +251,73 @@ describe Crystal::Doc::ProjectInfo do ProjectInfo.read_shard_properties.should eq({nil, nil}) end end + + it ".find_source_url_pattern" do + ProjectInfo.find_source_url_pattern("no a uri").should be_nil + ProjectInfo.find_source_url_pattern("git@example.com:foo/bar").should be_nil + ProjectInfo.find_source_url_pattern("http://example.com/foo/bar").should be_nil + + ProjectInfo.find_source_url_pattern("git@github.com:foo/bar/").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("git@github.com:foo/bar.git").should eq "https://github.com/foo/bar.git/blob/%{refname}/%{path}#L%{line}" + + ProjectInfo.find_source_url_pattern("git@github.com:foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://www.github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://www.github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.git").should eq "https://github.com/foo/bar.git/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.cr").should eq "https://github.com/foo/bar.cr/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.cr.git").should eq "https://github.com/foo/bar.cr.git/blob/%{refname}/%{path}#L%{line}" + + ProjectInfo.find_source_url_pattern("git@gitlab.com:foo/bar").should eq "https://gitlab.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://gitlab.com/foo/bar").should eq "https://gitlab.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + + ProjectInfo.find_source_url_pattern("git@bitbucket.com:foo/bar").should eq "https://bitbucket.com/foo/bar/src/%{refname}/%{path}#%{filename}-%{line}" + ProjectInfo.find_source_url_pattern("http://bitbucket.com/foo/bar").should eq "https://bitbucket.com/foo/bar/src/%{refname}/%{path}#%{filename}-%{line}" + + ProjectInfo.find_source_url_pattern("git@git.sr.ht:~foo/bar").should eq "https://git.sr.ht/~foo/bar/tree/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://git.sr.ht/~foo/bar").should eq "https://git.sr.ht/~foo/bar/tree/%{refname}/%{path}#L%{line}" + end + + describe "#source_url" do + it "fails if refname is missing" do + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info = ProjectInfo.new("test", "v1.0", refname: nil, source_url_pattern: "http://git.example.com/test.git/src/%{refname}/%{path}#L%{line}") + info.source_url(location).should be_nil + end + + it "fails if pattern is missing" do + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info = ProjectInfo.new("test", "v1.0", refname: "master") + info.source_url(location).should be_nil + end + + it "builds url" do + info = ProjectInfo.new("test", "v1.0", refname: "master", source_url_pattern: "http://git.example.com/test.git/src/%{refname}/%{path}#L%{line}") + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info.source_url(location).should eq "http://git.example.com/test.git/src/master/foo/bar.baz#L42" + end + + it "returns nil for empty pattern" do + info = ProjectInfo.new("test", "v1.0", refname: "master", source_url_pattern: "") + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info.source_url(location).should be_nil + end + + it "fails if pattern is missing" do + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info = ProjectInfo.new("test", "v1.0") + info.refname = "master" + info.source_url(location).should be_nil + end + + it "builds url" do + info = ProjectInfo.new("test", "v1.0") + info.refname = "master" + info.source_url_pattern = "http://git.example.com/test.git/src/%{refname}/%{path}#L%{line}" + location = Crystal::Doc::RelativeLocation.new("foo/bar.baz", 42) + info.source_url(location).should eq "http://git.example.com/test.git/src/master/foo/bar.baz#L42" + end + end end diff --git a/spec/compiler/crystal/tools/doc_spec.cr b/spec/compiler/crystal/tools/doc_spec.cr index e11fadc95f98..d6acc34b9f29 100644 --- a/spec/compiler/crystal/tools/doc_spec.cr +++ b/spec/compiler/crystal/tools/doc_spec.cr @@ -1,51 +1,6 @@ require "../../../spec_helper" -private def assert_matches_pattern(url, **options) - match = Crystal::Doc::Generator::GIT_REMOTE_PATTERNS.each_key.compact_map(&.match(url)).first? - if match - options.each { |k, v| match[k.to_s].should eq(v) } - end -end - describe Crystal::Doc::Generator do - describe "GIT_REMOTE_PATTERNS" do - it "matches github repos" do - assert_matches_pattern "https://www.github.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "http://www.github.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "http://github.com/foo/bar", user: "foo", repo: "bar" - - assert_matches_pattern "https://github.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "https://github.com/foo/bar.git", user: "foo", repo: "bar" - assert_matches_pattern "https://github.com/foo/bar.cr", user: "foo", repo: "bar.cr" - assert_matches_pattern "https://github.com/foo/bar.cr.git", user: "foo", repo: "bar.cr" - - assert_matches_pattern "origin\thttps://github.com/foo/bar.cr.git (fetch)\n", user: "foo", repo: "bar.cr" - assert_matches_pattern "origin\tgit@github.com/foo/bar.cr.git (fetch)\n", user: "foo", repo: "bar.cr" - - assert_matches_pattern "https://github.com/fOO-Bar/w00den-baRK.ab.cd", user: "fOO-Bar", repo: "w00den-baRK.ab.cd" - assert_matches_pattern "https://github.com/fOO-Bar/w00den-baRK.ab.cd.git", user: "fOO-Bar", repo: "w00den-baRK.ab.cd" - assert_matches_pattern "https://github.com/foo_bar/_baz-buzz.cx", user: "foo_bar", repo: "_baz-buzz.cx" - end - - it "matches gitlab repos" do - assert_matches_pattern "https://www.gitlab.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "http://www.gitlab.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "http://gitlab.com/foo/bar", user: "foo", repo: "bar" - - assert_matches_pattern "https://gitlab.com/foo/bar", user: "foo", repo: "bar" - assert_matches_pattern "https://gitlab.com/foo/bar.git", user: "foo", repo: "bar" - assert_matches_pattern "https://gitlab.com/foo/bar.cr", user: "foo", repo: "bar.cr" - assert_matches_pattern "https://gitlab.com/foo/bar.cr.git", user: "foo", repo: "bar.cr" - - assert_matches_pattern "origin\thttps://gitlab.com/foo/bar.cr.git (fetch)\n", user: "foo", repo: "bar.cr" - assert_matches_pattern "origin\tgit@gitlab.com/foo/bar.cr.git (fetch)\n", user: "foo", repo: "bar.cr" - - assert_matches_pattern "https://gitlab.com/fOO-Bar/w00den-baRK.ab.cd", user: "fOO-Bar", repo: "w00den-baRK.ab.cd" - assert_matches_pattern "https://gitlab.com/fOO-Bar/w00den-baRK.ab.cd.git", user: "fOO-Bar", repo: "w00den-baRK.ab.cd" - assert_matches_pattern "https://gitlab.com/foo_bar/_baz-buzz.cx", user: "foo_bar", repo: "_baz-buzz.cx" - end - end - describe ".anchor_link" do it "generates the correct anchor link" do Crystal::Doc.anchor_link("anchor").should eq( diff --git a/src/compiler/crystal/command/docs.cr b/src/compiler/crystal/command/docs.cr index c0b49be1cf69..61447c8dc527 100644 --- a/src/compiler/crystal/command/docs.cr +++ b/src/compiler/crystal/command/docs.cr @@ -33,6 +33,14 @@ class Crystal::Command project_info.version = value end + opts.on("--source-refname=REFNAME", "Set source refname (e.g. git tag, commit hash)") do |value| + project_info.refname = value + end + + opts.on("--source-url-pattern=REFNAME", "Set URL pattern for source code links") do |value| + project_info.source_url_pattern = value + end + opts.on("--output=DIR", "-o DIR", "Set the output directory (default: #{output_directory})") do |value| output_directory = value end diff --git a/src/compiler/crystal/tools/doc/generator.cr b/src/compiler/crystal/tools/doc/generator.cr index 77c8a3c4c50d..4b1f7a03a7c7 100644 --- a/src/compiler/crystal/tools/doc/generator.cr +++ b/src/compiler/crystal/tools/doc/generator.cr @@ -2,8 +2,6 @@ class Crystal::Doc::Generator getter program : Program @base_dir : String - @repository : String? = nil - getter repository_name = "" getter project_info # Adding a flag and associated css class will add support in parser @@ -18,17 +16,6 @@ class Crystal::Doc::Generator } FLAGS = FLAG_COLORS.keys - GIT_REMOTE_PATTERNS = { - /github\.com(?:\:|\/)(?(?:\w|-|_)+)\/(?(?:\w|-|_|\.)+?)(?:\.git)?\s/ => { - repository: "https://github.com/%{user}/%{repo}/blob/%{rev}", - repo_name: "github.com/%{user}/%{repo}", - }, - /gitlab\.com(?:\:|\/)(?(?:\w|-|_|\.)+)\/(?(?:\w|-|_|\.)+?)(?:\.git)?\s/ => { - repository: "https://gitlab.com/%{user}/%{repo}/blob/%{rev}", - repo_name: "gitlab.com/%{user}/%{repo}", - }, - } - def self.new(program : Program, included_dirs : Array(String)) new(program, included_dirs, ".", "html", nil, "1.0", "never", ProjectInfo.new("test", "0.0.0-test")) end @@ -40,8 +27,6 @@ class Crystal::Doc::Generator @project_info : ProjectInfo) @base_dir = Dir.current.chomp @types = {} of Crystal::Type => Doc::Type - @repo_name = "" - compute_repository end def run @@ -81,7 +66,7 @@ class Crystal::Doc::Generator def generate_docs_json(program_type, types) readme = read_readme - json = Main.new(readme, Type.new(self, @program), repository_name) + json = Main.new(readme, Type.new(self, @program), project_info) puts json end @@ -98,7 +83,7 @@ class Crystal::Doc::Generator File.write File.join(@output_dir, "index.html"), MainTemplate.new(body, types, project_info) - main_index = Main.new(raw_body, Type.new(self, @program), repository_name) + main_index = Main.new(raw_body, Type.new(self, @program), project_info) File.write File.join(@output_dir, "index.json"), main_index File.write File.join(@output_dir, "search-index.js"), main_index.to_jsonp end @@ -396,112 +381,37 @@ class Crystal::Doc::Generator end end - def compute_repository - # check whether inside git work-tree - `git rev-parse --is-inside-work-tree >/dev/null 2>&1` - return unless $?.success? - - remotes = `git remote -v` - return unless $?.success? - - git_matches = remotes.each_line.compact_map do |line| - GIT_REMOTE_PATTERNS.each_key.compact_map(&.match(line)).first? - end.to_a - - origin = git_matches.find(&.string.starts_with?("origin")) || git_matches.first? - return unless origin - - user = origin["user"] - repo = origin["repo"] - rev = `git rev-parse HEAD`.chomp - - info = GIT_REMOTE_PATTERNS[origin.regex] - @repository = info[:repository] % {user: user, repo: repo, rev: rev} - @repository_name = info[:repo_name] % {user: user, repo: repo} - end - def source_link(node) - location = relative_location node + location = RelativeLocation.from(node, @base_dir) return unless location - - filename = relative_filename location - return unless filename - - "#{@repository}#{filename}#L#{location.line_number}" - end - - def relative_location(node : ASTNode) - relative_location node.location - end - - def relative_location(location : Location?) - return unless location - - repository = @repository - return unless repository - - filename = location.filename - if filename.is_a?(VirtualFile) - location = filename.expanded_location - end - - location - end - - def relative_filename(location) - filename = location.filename - return unless filename.is_a?(String) - return unless filename.starts_with? @base_dir - filename[@base_dir.size..-1] - end - - class RelativeLocation - property show_line_number - getter filename, line_number, url - - def initialize(@filename : String, @line_number : Int32, @url : String?, @show_line_number : Bool) - end - - def to_json(builder : JSON::Builder) - builder.object do - builder.field "filename", filename - builder.field "line_number", line_number - builder.field "url", url - end - end + project_info.source_url(location) end SRC_SEP = "src#{File::SEPARATOR}" def relative_locations(type) - repository = @repository locations = [] of RelativeLocation type.locations.try &.each do |location| - location = relative_location location + location = RelativeLocation.from(location, @base_dir) next unless location - - filename = relative_filename location + filename = location.filename next unless filename - url = "#{repository}#{filename}" if repository - - filename = filename[1..-1] if filename.starts_with? File::SEPARATOR - filename = filename[4..-1] if filename.starts_with? SRC_SEP + url = project_info.source_url(location) + next unless url + location.url = url # Prevent identical link generation in the "Defined in:" section in the docs because of macros - next if locations.any? { |loc| loc.filename == filename && loc.line_number == location.line_number } + next if locations.includes?(location) - show_line_number = locations.any? do |location| - if location.filename == filename - location.show_line_number = true - true - else - false - end + same_file_location = locations.find { |loc| loc.filename == filename } + if same_file_location + location.show_line_number = true + same_file_location.show_line_number = true end - locations << RelativeLocation.new(filename, location.line_number, url, show_line_number) + locations << location end - locations + locations.sort end end diff --git a/src/compiler/crystal/tools/doc/html/type.html b/src/compiler/crystal/tools/doc/html/type.html index 0834d7190c54..9c67e3e8ac39 100644 --- a/src/compiler/crystal/tools/doc/html/type.html +++ b/src/compiler/crystal/tools/doc/html/type.html @@ -54,12 +54,12 @@

Defined in:

<% locations.each do |location| %> - <% if url = location.url %> - - <%= location.filename %><% if location.show_line_number %>:<%= location.line_number %><% end %> + <% if url = project_info.source_url(location) %> + + <%= location.filename_in_project %><% if location.show_line_number %>:<%= location.line_number %><% end %> <% else %> - <%= location.filename %><% if location.show_line_number %>:<%= location.line_number %><% end %> + <%= location.filename_in_project %><% if location.show_line_number %>:<%= location.line_number %><% end %> <% end %>
<% end %> diff --git a/src/compiler/crystal/tools/doc/main.cr b/src/compiler/crystal/tools/doc/main.cr index cfae49d9da47..03fd0fa27a5f 100644 --- a/src/compiler/crystal/tools/doc/main.cr +++ b/src/compiler/crystal/tools/doc/main.cr @@ -1,5 +1,5 @@ module Crystal::Doc - record Main, body : String, program : Type, repository_name : String do + record Main, body : String, program : Type, project_info : ProjectInfo do def to_s(io : IO) : Nil to_json(io) end @@ -18,7 +18,7 @@ module Crystal::Doc def to_json(builder : JSON::Builder) builder.object do - builder.field "repository_name", repository_name + builder.field "repository_name", project_info.name builder.field "body", body builder.field "program", program end diff --git a/src/compiler/crystal/tools/doc/project_info.cr b/src/compiler/crystal/tools/doc/project_info.cr index 3e9bd0e48243..0ebf986e1209 100644 --- a/src/compiler/crystal/tools/doc/project_info.cr +++ b/src/compiler/crystal/tools/doc/project_info.cr @@ -3,11 +3,13 @@ module Crystal::Doc property! name : String property! version : String property json_config_url : String? = nil + property refname : String? = nil + property source_url_pattern : String? = nil - def initialize(@name : String? = nil, @version : String? = nil) + def initialize(@name : String? = nil, @version : String? = nil, @refname : String? = nil, @source_url_pattern : String? = nil) end - def_equals_and_hash @name, @version, @json_config_url + def_equals_and_hash @name, @version, @json_config_url, @refname, @source_url_pattern def crystal_stdlib? name == "Crystal" @@ -20,6 +22,16 @@ module Crystal::Doc end end + if ProjectInfo.git_clean? + self.refname ||= ProjectInfo.git_ref(branch: false) + end + + unless source_url_pattern + if remote = ProjectInfo.git_remote + self.source_url_pattern = ProjectInfo.find_source_url_pattern(remote) + end + end + unless name? && version? shard_name, shard_version = ProjectInfo.read_shard_properties if shard_name && !name? @@ -31,6 +43,16 @@ module Crystal::Doc end end + def source_url(location : RelativeLocation) + refname = self.refname + url_pattern = source_url_pattern + + return unless refname && url_pattern + + url = url_pattern % {refname: refname, path: location.filename, filename: File.basename(location.filename), line: location.line_number} + url.presence + end + def self.git_dir? Process.run("git", ["rev-parse", "--is-inside-work-tree"]).success? end @@ -38,7 +60,7 @@ module Crystal::Doc VERSION_TAG = /^v(\d+[-.][-.a-zA-Z\d]+)$/ def self.find_git_version - if ref = git_ref + if ref = git_ref(branch: true) if ref.matches?(VERSION_TAG) ref = ref.byte_slice(1) end @@ -51,6 +73,56 @@ module Crystal::Doc end end + def self.find_source_url_pattern(remote) + if (at_index = remote.index('@')) && (colon_index = remote.index(':')) && at_index < colon_index + # SSH URI + host = remote[(at_index + 1)...colon_index] + path = remote[(colon_index + 1)..] + else + begin + uri = URI.parse(remote) + rescue URI::Error + return + end + host = uri.host + path = uri.path + end + + path = path.strip("/") + + case host + when "github.com", "www.github.com" + "https://github.com/#{path}/blob/%{refname}/%{path}#L%{line}" + when "gitlab.com", "www.gitlab.com" + "https://gitlab.com/#{path}/blob/%{refname}/%{path}#L%{line}" + when "bitbucket.com", "www.bitbucket.com" + "https://bitbucket.com/#{path}/src/%{refname}/%{path}#%{filename}-%{line}" + when "git.sr.ht" + "https://git.sr.ht/#{path}/tree/%{refname}/%{path}#L%{line}" + else + # Unknown remote host, can't determine source url pattern + end + end + + def self.git_remote + # check whether inside git work-tree + status = Process.run("git", ["rev-parse", "--is-inside-work-tree"]) + return unless status.success? + + io = IO::Memory.new + status = Process.run("git", ["remote", "-v"], output: io) + return unless status.success? + + remotes = io.to_s.lines.select(&.ends_with?(" (fetch)")) + + git_remote = remotes.find(&.starts_with?("origin\t")) || remotes.first? || return + + start_pos = git_remote.index("\t") + end_pos = git_remote.rindex(" ") + return unless start_pos && end_pos + git_remote[(start_pos + 1)...end_pos].presence + end + def self.git_clean? # Use git to determine if index and working directory are clean io = IO::Memory.new @@ -62,7 +134,7 @@ module Crystal::Doc io.bytesize == 0 end - def self.git_ref + def self.git_ref(*, branch) io = IO::Memory.new # Check if current HEAD is tagged status = Process.run("git", ["tag", "--points-at", "HEAD"], output: io) @@ -73,13 +145,26 @@ module Crystal::Doc if tag = tags.first? return tag end - - # Otherwise, return current branch name io.clear - status = Process.run("git", ["rev-parse", "--abbrev-ref", "HEAD"], output: io) + + if branch + # Read current branch name + status = Process.run("git", ["rev-parse", "--abbrev-ref", "HEAD"], output: io) + return unless status.success? + + if branch_name = io.to_s.strip.presence + return branch_name + end + io.clear + end + + # Otherwise, return current commit sha + status = Process.run("git", ["rev-parse", "HEAD"], output: io) return unless status.success? - io.to_s.strip.presence + if sha = io.to_s.strip.presence + return sha + end end def self.read_shard_properties diff --git a/src/compiler/crystal/tools/doc/relative_location.cr b/src/compiler/crystal/tools/doc/relative_location.cr new file mode 100644 index 000000000000..32d43cc1e34c --- /dev/null +++ b/src/compiler/crystal/tools/doc/relative_location.cr @@ -0,0 +1,52 @@ +class Crystal::Doc::RelativeLocation + include Comparable(self) + property show_line_number : Bool = false + + # This property is only used to keep backwards compatibility in JSON output. + property url : String? + + getter filename, line_number + + def initialize(@filename : String, @line_number : Int32) + end + + def_equals_and_hash @filename, @line_number + + def filename_in_project + filename.lchop("src/") + end + + def to_json(builder : JSON::Builder) + builder.object do + builder.field "filename", filename + builder.field "line_number", line_number + builder.field "url", url + end + end + + def <=>(other : self) + cmp = filename <=> other.filename + return cmp unless cmp == 0 + line_number <=> other.line_number + end + + def self.from(node : ASTNode, base_dir : String) + if location = node.location + from(location, base_dir) + end + end + + def self.from(location : Location, base_dir : String) + filename = location.filename + if filename.is_a?(VirtualFile) + location = filename.expanded_location || return + filename = location.filename + end + + return unless filename.is_a?(String) + return unless filename.starts_with? base_dir + filename = filename[(base_dir.size + 1)..] + + new(filename, location.line_number) + end +end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index 8809a50f6fe2..e9b27cfca745 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -758,7 +758,7 @@ class Crystal::Doc::Type end def html_id - "#{@generator.repository_name}/" + ( + "#{@generator.project_info.name}/" + ( if program? "toplevel" elsif namespace = self.namespace @@ -786,7 +786,7 @@ class Crystal::Doc::Type end end builder.field "locations", locations - builder.field "repository_name", @generator.repository_name + builder.field "repository_name", @generator.project_info.name builder.field "program", program? builder.field "enum", enum? builder.field "alias", alias? From 6d0d63f66a796f73df89e7a38c4862d5946e4574 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 14 May 2020 09:05:51 -0300 Subject: [PATCH 032/263] XML: improve xpath regarding namespaces (#9288) * XML: improve xpath regarding namespaces * Add a few type restrictions and remove an unnecessary `to_s` --- spec/std/xml/xpath_spec.cr | 32 +++++++++++++++++++++++++++++++- src/xml/node.cr | 29 ++++++++++++++++++++--------- src/xml/xpath_context.cr | 5 +++-- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/spec/std/xml/xpath_spec.cr b/spec/std/xml/xpath_spec.cr index b58f5e803f05..9f243cf9f2ee 100644 --- a/spec/std/xml/xpath_spec.cr +++ b/spec/std/xml/xpath_spec.cr @@ -75,7 +75,7 @@ module XML doc.xpath_node("//invalid").should be_nil end - it "finds with namespace" do + it "finds with explicit namespace" do doc = XML.parse(%(\ @@ -89,6 +89,22 @@ module XML ns.prefix.should be_nil end + it "finds with implicit (root) namespaces" do + doc = XML.parse(%(\ + + + + + + )) + nodes = doc.xpath("//openSearch:feed/openSearch:something").as(NodeSet) + nodes.size.should eq(1) + nodes[0].name.should eq("something") + ns = nodes[0].namespace.not_nil! + ns.href.should eq("http://a9.com/-/spec/opensearchrss/1.0/") + ns.prefix.should eq("openSearch") + end + it "finds with root namespaces" do doc = XML.parse(%(\ @@ -103,6 +119,20 @@ module XML ns.prefix.should be_nil end + it "finds with root namespaces (using prefix)" do + doc = XML.parse(%(\ + + + + )) + nodes = doc.xpath("//openSearch:feed", namespaces: doc.root.not_nil!.namespaces).as(NodeSet) + nodes.size.should eq(1) + nodes[0].name.should eq("feed") + ns = nodes[0].namespace.not_nil! + ns.href.should eq("http://a9.com/-/spec/opensearchrss/1.0/") + ns.prefix.should eq("openSearch") + end + it "finds with variable binding" do doc = XML.parse(%(\ diff --git a/src/xml/node.cr b/src/xml/node.cr index 14ffe7fea676..7f4e2d10523a 100644 --- a/src/xml/node.cr +++ b/src/xml/node.cr @@ -284,7 +284,7 @@ struct XML::Node end # Returns the namespace for this node or `nil` if not found. - def namespace + def namespace : Namespace? case type when Type::DOCUMENT_NODE, Type::ATTRIBUTE_DECL, Type::DTD_NODE, Type::ELEMENT_DECL nil @@ -301,7 +301,7 @@ struct XML::Node # Default namespaces for ancestors, however, are not. # # See also `#namespaces` - def namespace_scopes + def namespace_scopes : Array(Namespace) scopes = [] of Namespace ns_list = LibXML.xmlGetNsList(@node.value.doc, @node) @@ -326,21 +326,24 @@ struct XML::Node # # NOTE: Note that the keys in this hash XML attributes that would be used to # define this namespace, such as `"xmlns:prefix"`, not just the prefix. - def namespaces + def namespaces : Hash(String, String?) namespaces = {} of String => String? + each_namespace do |namespace| + prefix = namespace.prefix ? "xmlns:#{namespace.prefix}" : "xmlns" + namespaces[prefix] = namespace.href + end + namespaces + end + protected def each_namespace(& : Namespace ->) ns_list = LibXML.xmlGetNsList(@node.value.doc, @node) if ns_list while ns_list.value - namespace = Namespace.new(document, ns_list.value) - prefix = namespace.prefix - namespaces[prefix ? "xmlns:#{prefix}" : "xmlns"] = namespace.href + yield Namespace.new(document, ns_list.value) ns_list += 1 end end - - namespaces end # Returns the address of underlying `LibXML::Node*` in memory. @@ -483,7 +486,15 @@ struct XML::Node # Raises `XML::Error` on evaluation error. def xpath(path, namespaces = nil, variables = nil) ctx = XPathContext.new(self) - ctx.register_namespaces namespaces if namespaces + + if namespaces + ctx.register_namespaces namespaces + else + root.try &.each_namespace do |namespace| + ctx.register_namespace namespace.prefix || "xmlns", namespace.href + end + end + ctx.register_variables variables if variables ctx.evaluate(path) end diff --git a/src/xml/xpath_context.cr b/src/xml/xpath_context.cr index 25af8e92683d..f6509237e023 100644 --- a/src/xml/xpath_context.cr +++ b/src/xml/xpath_context.cr @@ -41,8 +41,9 @@ struct XML::XPathContext end end - def register_namespace(prefix, uri) - LibXML.xmlXPathRegisterNs(self, prefix.to_s, uri.to_s) + def register_namespace(prefix : String, uri : String?) + prefix = prefix.lchop("xmlns:") + LibXML.xmlXPathRegisterNs(self, prefix, uri.to_s) end def register_variables(variables) From 19e21728a9cd5f0bb804f62af481dfe89faf20da Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 14 May 2020 09:07:12 -0300 Subject: [PATCH 033/263] Compiler: cast fun function pointer to Proc (#9287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Compiler: cast fun function pointer to Proc * Compiler: when assigning nil to a pointer/proc struct field, don't check closure * Remove unneeded raise in spec * Update spec/compiler/codegen/closure_spec.cr Co-authored-by: Jonne Haß Co-authored-by: Jonne Haß --- spec/compiler/codegen/closure_spec.cr | 9 +++---- spec/compiler/codegen/proc_spec.cr | 28 ++++++++++++++++++++++ src/compiler/crystal/codegen/codegen.cr | 10 ++++---- src/compiler/crystal/codegen/fun.cr | 11 +++++++++ src/compiler/crystal/codegen/primitives.cr | 6 ++--- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/spec/compiler/codegen/closure_spec.cr b/spec/compiler/codegen/closure_spec.cr index 90e71c36317a..784b4e2cfe49 100644 --- a/spec/compiler/codegen/closure_spec.cr +++ b/spec/compiler/codegen/closure_spec.cr @@ -700,13 +700,8 @@ describe "Code gen: closure" do it "allows passing an external function along" do codegen(%( - lib LibC - fun exit(c : Int32) : NoReturn - end + require "prelude" - def raise(a) : NoReturn - LibC.exit(1) - end lib LibA fun a(a : Void* -> Void*) @@ -716,7 +711,9 @@ describe "Code gen: closure" do LibA.a(a) end )) + end + it "allows passing an external function along (2)" do codegen(%( lib LibFoo struct S diff --git a/spec/compiler/codegen/proc_spec.cr b/spec/compiler/codegen/proc_spec.cr index 390d49ac235e..2bbe672b0322 100644 --- a/spec/compiler/codegen/proc_spec.cr +++ b/spec/compiler/codegen/proc_spec.cr @@ -819,4 +819,32 @@ describe "Code gen: proc" do a )).to_i.should eq(3) end + + it "calls function pointer" do + run(%( + require "prelude" + + fun foo(f : Int32 -> Int32) : Int32 + f.call(1) + end + + foo(->(x : Int32) { x &+ 1 }) + )).to_i.should eq(2) + end + + it "casts from function pointer to proc" do + codegen(%( + fun a(a : Void* -> Void*) + Pointer(Proc((Void* -> Void*), Void*)).new(0_u64).value.call(a) + end + )) + end + + it "takes pointerof function pointer" do + codegen(%( + fun a(a : Void* -> Void*) + pointerof(a).value.call(Pointer(Void).new(0_u64)) + end + )) + end end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index f2ba2a364560..a0456e9b7925 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -1531,12 +1531,10 @@ module Crystal end def check_proc_is_not_closure(value, type) - if value.type == llvm_typer.proc_type - check_fun_name = "~check_proc_is_not_closure" - func = @main_mod.functions[check_fun_name]? || create_check_proc_is_not_closure_fun(check_fun_name) - func = check_main_fun check_fun_name, func - value = call func, [value] of LLVM::Value - end + check_fun_name = "~check_proc_is_not_closure" + func = @main_mod.functions[check_fun_name]? || create_check_proc_is_not_closure_fun(check_fun_name) + func = check_main_fun check_fun_name, func + value = call func, [value] of LLVM::Value bit_cast value, llvm_proc_type(type) end diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index 07cea2419a01..e8e8c548eaed 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -512,12 +512,23 @@ class Crystal::CodeGenVisitor context.vars[arg.name] = LLVMVar.new(value, var_type) return else + # If an argument is a Proc inside a C function, we need to cast it to Proc + fun_proc = var_type.is_a?(ProcInstanceType) && target_def.is_a?(External) + # We don't need to create a copy of the argument if it's never # assigned a value inside the function. needs_copy = target_def_var.try &.assigned_to? + needs_copy ||= fun_proc + if needs_copy pointer = alloca(llvm_type(var_type), arg.name) pointer = declare_debug_for_function_argument(arg.name, var_type, index + 1, pointer, location) unless target_def.naked? + + if fun_proc + value = bit_cast(value, llvm_context.void_pointer) + value = make_fun(var_type, value, llvm_context.void_pointer.null) + end + context.vars[arg.name] = LLVMVar.new(pointer, var_type) if arg.type.passed_by_value? && !context.fun.attributes(index + 1).by_val? diff --git a/src/compiler/crystal/codegen/primitives.cr b/src/compiler/crystal/codegen/primitives.cr index 39bfe6d53836..94fc88c44810 100644 --- a/src/compiler/crystal/codegen/primitives.cr +++ b/src/compiler/crystal/codegen/primitives.cr @@ -860,12 +860,10 @@ class Crystal::CodeGenVisitor scope = context.type.as(NonGenericClassType) field_type = scope.instance_vars[var_name].type - # Check nil to pointer + # Check assigning nil to a field of type pointer or Proc if node.type.nil_type? && (field_type.pointer? || field_type.proc?) call_arg = llvm_c_type(field_type).null - end - - if field_type.proc? + elsif field_type.proc? call_arg = check_proc_is_not_closure(call_arg, field_type) end From d270ebae7986497c5c074ead00b3fef3748606ce Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 14 May 2020 10:33:55 -0300 Subject: [PATCH 034/263] Log: Rename Severity::Warning to Severity::Warn (#9293) This make will stop accepting :warning and LOG_LEVEL=WARNING in favor of :warn and LOG_LEVEL=WARN. --- spec/std/log/broadcast_backend_spec.cr | 6 +++--- spec/std/log/builder_spec.cr | 8 ++++---- spec/std/log/env_config_spec.cr | 4 ++-- spec/std/log/format_spec.cr | 6 +++--- spec/std/log/io_backend_spec.cr | 4 ++-- spec/std/log/log_spec.cr | 14 +++++++------- spec/std/log/main_spec.cr | 8 ++++---- src/compiler/crystal.cr | 2 +- src/log.cr | 2 +- src/log/entry.cr | 18 +++++++++--------- src/log/format.cr | 12 ++++++------ src/log/log.cr | 3 +-- 12 files changed, 43 insertions(+), 44 deletions(-) diff --git a/spec/std/log/broadcast_backend_spec.cr b/spec/std/log/broadcast_backend_spec.cr index fa18ff5fa72f..28b88291855e 100644 --- a/spec/std/log/broadcast_backend_spec.cr +++ b/spec/std/log/broadcast_backend_spec.cr @@ -68,15 +68,15 @@ describe Log::BroadcastBackend do it "single backend" do main = Log::BroadcastBackend.new - main.append(Log::MemoryBackend.new, s(:warning)) + main.append(Log::MemoryBackend.new, s(:warn)) - main.min_level.should eq(s(:warning)) + main.min_level.should eq(s(:warn)) end it "multiple backends" do main = Log::BroadcastBackend.new main.append(Log::MemoryBackend.new, s(:info)) - main.append(Log::MemoryBackend.new, s(:warning)) + main.append(Log::MemoryBackend.new, s(:warn)) main.min_level.should eq(s(:info)) end diff --git a/spec/std/log/builder_spec.cr b/spec/std/log/builder_spec.cr index 8f97f3b3efb1..d91453044b60 100644 --- a/spec/std/log/builder_spec.cr +++ b/spec/std/log/builder_spec.cr @@ -105,13 +105,13 @@ describe Log::Builder do builder = Log::Builder.new a = Log::MemoryBackend.new builder.bind("*", :fatal, a) - builder.bind("db.*", :warning, a) + builder.bind("db.*", :warn, a) builder.bind("db", :error, a) builder.bind("db.pool", :none, a) builder.for("").level.should eq(s(:fatal)) builder.for("db").level.should eq(s(:error)) - builder.for("db.query").level.should eq(s(:warning)) + builder.for("db.query").level.should eq(s(:warn)) builder.for("db.pool").level.should eq(s(:none)) end @@ -130,10 +130,10 @@ describe Log::Builder do log.level.should eq(s(:none)) a = Log::MemoryBackend.new - builder.bind("*", :warning, a) + builder.bind("*", :warn, a) log.backend.should be(a) - log.level.should eq(s(:warning)) + log.level.should eq(s(:warn)) end it "removes all logs backends on .clear" do diff --git a/spec/std/log/env_config_spec.cr b/spec/std/log/env_config_spec.cr index 8eb505a5645d..a98167fea23a 100644 --- a/spec/std/log/env_config_spec.cr +++ b/spec/std/log/env_config_spec.cr @@ -63,9 +63,9 @@ describe "Log.setup_from_env" do it "is used if no LOG_LEVEL is set" do with_env "LOG_LEVEL": nil do builder = Log::Builder.new - Log.setup_from_env(builder: builder, default_level: :warning) + Log.setup_from_env(builder: builder, default_level: :warn) - builder.for("").initial_level.should eq(s(:warning)) + builder.for("").initial_level.should eq(s(:warn)) end end diff --git a/spec/std/log/format_spec.cr b/spec/std/log/format_spec.cr index af9045f35195..c8b9ead1f5db 100644 --- a/spec/std/log/format_spec.cr +++ b/spec/std/log/format_spec.cr @@ -79,9 +79,9 @@ class Log TestFormatter.format(Entry.new("source", :error, "Oh, no", Log::Metadata.empty, exception), io) io.rewind - io.gets.should eq(" INFO [source] test message ({\"a\" => 1, \"b\" => 2})") - io.gets.should eq(" INFO test message") - io.gets.should eq(" ERROR [source] test Oh, no") + io.gets.should eq(" INFO [source] test message ({\"a\" => 1, \"b\" => 2})") + io.gets.should eq(" INFO test message") + io.gets.should eq(" ERROR [source] test Oh, no") io.gets_to_end.should eq(exception.inspect_with_backtrace) end end diff --git a/spec/std/log/io_backend_spec.cr b/spec/std/log/io_backend_spec.cr index 051df3cc7d5f..56a1a4dca46b 100644 --- a/spec/std/log/io_backend_spec.cr +++ b/spec/std/log/io_backend_spec.cr @@ -23,7 +23,7 @@ describe Log::IOBackend do logger.level = s(:debug) logger.debug { "debug:show" } - logger.level = s(:warning) + logger.level = s(:warn) logger.debug { "debug:skip:again" } logger.info { "info:skip" } logger.error { "error:show" } @@ -61,7 +61,7 @@ describe Log::IOBackend do logger = io_logger(stdout: w, source: "db.pool") logger.warn { "message" } - r.gets(chomp: false).should match(/.+? WARNING - db.pool: message\n/) + r.gets(chomp: false).should match(/.+? WARN - db.pool: message\n/) end end diff --git a/spec/std/log/log_spec.cr b/spec/std/log/log_spec.cr index fe0d98637ab8..aeb8f294d01d 100644 --- a/spec/std/log/log_spec.cr +++ b/spec/std/log/log_spec.cr @@ -23,8 +23,8 @@ describe Log do s(:trace).should be < s(:debug) s(:debug).should be < s(:info) s(:info).should be < s(:notice) - s(:notice).should be < s(:warning) - s(:warning).should be < s(:error) + s(:notice).should be < s(:warn) + s(:warn).should be < s(:error) s(:error).should be < s(:fatal) s(:fatal).should be < s(:none) end @@ -34,7 +34,7 @@ describe Log do Log::Severity.parse("debug").should eq s(:debug) Log::Severity.parse("info").should eq s(:info) Log::Severity.parse("notice").should eq s(:notice) - Log::Severity.parse("warning").should eq s(:warning) + Log::Severity.parse("warn").should eq s(:warn) Log::Severity.parse("error").should eq s(:error) Log::Severity.parse("fatal").should eq s(:fatal) Log::Severity.parse("none").should eq s(:none) @@ -43,7 +43,7 @@ describe Log do Log::Severity.parse("DEBUG").should eq s(:debug) Log::Severity.parse("INFO").should eq s(:info) Log::Severity.parse("NOTICE").should eq s(:notice) - Log::Severity.parse("WARNING").should eq s(:warning) + Log::Severity.parse("WARN").should eq s(:warn) Log::Severity.parse("ERROR").should eq s(:error) Log::Severity.parse("FATAL").should eq s(:fatal) Log::Severity.parse("NONE").should eq s(:none) @@ -52,7 +52,7 @@ describe Log do it "filter messages to the backend above level only" do backend = Log::MemoryBackend.new - log = Log.new("a", backend, :warning) + log = Log.new("a", backend, :warn) log.trace { "trace message" } log.debug { "debug message" } @@ -63,7 +63,7 @@ describe Log do log.fatal { "fatal message" } backend.entries.map { |e| {e.severity, e.message} }.should eq([ - {s(:warning), "warning message"}, + {s(:warn), "warning message"}, {s(:error), "error message"}, {s(:fatal), "fatal message"}, ]) @@ -71,7 +71,7 @@ describe Log do it "level can be changed" do backend = Log::MemoryBackend.new - log = Log.new("a", backend, :warning) + log = Log.new("a", backend, :warn) log.level = :error diff --git a/spec/std/log/main_spec.cr b/spec/std/log/main_spec.cr index 4abfc88000cc..9b305d2f55ce 100644 --- a/spec/std/log/main_spec.cr +++ b/spec/std/log/main_spec.cr @@ -32,16 +32,16 @@ describe Log do top = Log.for("qux", :info) top.level.should eq(Log::Severity::Info) - Log.for("qux", :warning) - top.level.should eq(Log::Severity::Warning) + Log.for("qux", :warn) + top.level.should eq(Log::Severity::Warn) end it "can build nested with level override" do foo_bar = Log.for("foo").for("bar", :info) foo_bar.level.should eq(Log::Severity::Info) - Log.for("foo.bar", :warning) - foo_bar.level.should eq(Log::Severity::Warning) + Log.for("foo.bar", :warn) + foo_bar.level.should eq(Log::Severity::Warn) end it "can build for module type" do diff --git a/src/compiler/crystal.cr b/src/compiler/crystal.cr index beefe336ffb1..1d14ec386bc6 100644 --- a/src/compiler/crystal.cr +++ b/src/compiler/crystal.cr @@ -6,6 +6,6 @@ require "log" require "./crystal/**" -Log.setup_from_env(default_level: :warning, default_sources: "crystal.*") +Log.setup_from_env(default_level: :warn, default_sources: "crystal.*") Crystal::Command.run diff --git a/src/log.cr b/src/log.cr index c4b7a6feaf64..5b5f7acb8913 100644 --- a/src/log.cr +++ b/src/log.cr @@ -108,7 +108,7 @@ # Log.setup |c| # backend = Log::IOBackend.new # -# c.bind "*", :warning, backend +# c.bind "*", :warn, backend # c.bind "db.*", :debug, backend # c.bind "*", :error, ElasticSearchBackend.new("http://localhost:9200") # end diff --git a/src/log/entry.cr b/src/log/entry.cr index 4603c764baed..185d2b62e0f2 100644 --- a/src/log/entry.cr +++ b/src/log/entry.cr @@ -9,7 +9,7 @@ enum Log::Severity # Used for normal but significant conditions. Notice # Used for conditions that can potentially cause application oddities, but that can be automatically recovered. - Warning + Warn # Used for any error that is fatal to the operation, but not to the service or application. Error # Used for any error that is forcing a shutdown of the service or application @@ -19,14 +19,14 @@ enum Log::Severity def label case self - when Trace then "TRACE" - when Debug then "DEBUG" - when Info then "INFO" - when Notice then "NOTICE" - when Warning then "WARNING" - when Error then "ERROR" - when Fatal then "FATAL" - when None then "NONE" + when Trace then "TRACE" + when Debug then "DEBUG" + when Info then "INFO" + when Notice then "NOTICE" + when Warn then "WARN" + when Error then "ERROR" + when Fatal then "FATAL" + when None then "NONE" else raise "unreachable" end diff --git a/src/log/format.cr b/src/log/format.cr index 30275e6284b1..57ffe7346ebd 100644 --- a/src/log/format.cr +++ b/src/log/format.cr @@ -46,8 +46,8 @@ class Log # end # # Log.setup(:info, Log::IOBackend.new(formatter: MyFormat)) - # Log.info { "Hello" } # => - INFO: Hello - # Log.error { "Oh, no!" } # => - ERROR: Oh, no! + # Log.info { "Hello" } # => - INFO: Hello + # Log.error { "Oh, no!" } # => - ERROR: Oh, no! # ``` # # There is also a helper macro to generate these formatters. Here's @@ -81,7 +81,7 @@ class Log # This writes the severity in uppercase and left padded # with enough space so all the severities fit def severity - @entry.severity.label.rjust(@io, 7) + @entry.severity.label.rjust(@io, 6) end # Write the source for non-root entries @@ -180,17 +180,17 @@ end # # It writes log entries with the following format: # ``` -# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything +# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything # ``` # # When the entries have context data it's also written to the output: # ``` -# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything -- {"data" => 123} +# 2020-05-07T17:40:07.994508000Z INFO - my.source: Initializing everything -- {"data" => 123} # ``` # # Exceptions are written in a separate line: # ``` -# 2020-05-07T17:40:07.994508000Z ERROR - my.source: Something failed +# 2020-05-07T17:40:07.994508000Z ERROR - my.source: Something failed # Oh, no (Exception) # from ... # ``` diff --git a/src/log/log.cr b/src/log/log.cr index e2573f56ff3a..5c93d14cfcd5 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -38,11 +38,10 @@ class Log debug: Severity::Debug, info: Severity::Info, notice: Severity::Notice, - warn: Severity::Warning, + warn: Severity::Warn, error: Severity::Error, fatal: Severity::Fatal, } %} - # Logs a message if the logger's current severity is lower or equal to `{{severity}}`. def {{method.id}}(*, exception : Exception? = nil) return unless backend = @backend From 46aad29871739b0d95e8e22bf5bfa68edf4c6dd5 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 14 May 2020 10:34:14 -0300 Subject: [PATCH 035/263] Avoid requiring non std-lib spec spec_helper in hooks_spec (#9294) * Avoid requiring non std-lib spec spec_helper in hooks_spec * Refactor: DRY * Completely remove top level spec_helper from std specs --- spec/std/exception/call_stack_spec.cr | 18 +++----------- spec/std/exception_spec.cr | 14 +---------- spec/std/io/file_descriptor_spec.cr | 4 ++-- spec/std/kernel_spec.cr | 32 ++++++++++++------------- spec/std/process_spec.cr | 3 +-- spec/std/spec/hooks_spec.cr | 4 ++-- spec/std/spec_helper.cr | 34 +++++++++++++++++++++++++++ 7 files changed, 59 insertions(+), 50 deletions(-) diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index efb1bd5f68bc..ee42e4d77b5c 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -1,17 +1,5 @@ require "../spec_helper" -private def compile_and_run_file(source_file) - with_tempfile("executable_file") do |executable_file| - Process.run("bin/crystal", ["build", "--debug", "-o", executable_file, source_file]) - File.exists?(executable_file).should be_true - - output, error = IO::Memory.new, IO::Memory.new - Process.run executable_file, output: output, error: error - - {output.to_s, error.to_s} - end -end - describe "Backtrace" do it "prints file line:colunm" do source_file = datapath("backtrace_sample") @@ -22,7 +10,7 @@ describe "Backtrace" do current_dir += File::SEPARATOR unless current_dir.ends_with?(File::SEPARATOR) source_file = source_file.lchop(current_dir) - output, _ = compile_and_run_file(source_file) + _, output, _ = compile_and_run_file(source_file) # resolved file line:column output.should match(/#{source_file}:3:10 in 'callee1'/) @@ -40,7 +28,7 @@ describe "Backtrace" do it "prints exception backtrace to stderr" do sample = datapath("exception_backtrace_sample") - output, error = compile_and_run_file(sample) + _, output, error = compile_and_run_file(sample) output.to_s.empty?.should be_true error.to_s.should contain("IndexError") @@ -49,7 +37,7 @@ describe "Backtrace" do it "prints crash backtrace to stderr" do sample = datapath("crash_backtrace_sample") - output, error = compile_and_run_file(sample) + _, output, error = compile_and_run_file(sample) output.to_s.empty?.should be_true error.to_s.should contain("Invalid memory access") diff --git a/spec/std/exception_spec.cr b/spec/std/exception_spec.cr index 72840c47c442..fc4aa1ae19eb 100644 --- a/spec/std/exception_spec.cr +++ b/spec/std/exception_spec.cr @@ -1,17 +1,5 @@ require "./spec_helper" -private def compile_and_run_file(source_file) - with_tempfile("executable_file") do |executable_file| - Process.run("bin/crystal", ["build", "--release", "-o", executable_file, source_file]) - File.exists?(executable_file).should be_true - - output, error = IO::Memory.new, IO::Memory.new - Process.run executable_file, output: output, error: error - - {output.to_s, error.to_s} - end -end - private class FooError < Exception def message "#{super || ""} -- bar!" @@ -51,7 +39,7 @@ describe "Exception" do it "collect memory within ensure block" do sample = datapath("collect_within_ensure") - output, error = compile_and_run_file(sample) + _, output, error = compile_and_run_file(sample, ["--release"]) output.to_s.empty?.should be_true error.to_s.should contain("Unhandled exception: Oh no! (Exception)") diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index 37ff29a5ed60..566b8236ddc0 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -1,9 +1,9 @@ -require "../../spec_helper" +require "../spec_helper" describe IO::FileDescriptor do it "reopen STDIN with the right mode" do code = %q(puts "#{STDIN.blocking} #{STDIN.info.type}") - build(code) do |binpath| + compile_source(code) do |binpath| `#{binpath} < #{binpath}`.chomp.should eq("true File") `echo "" | #{binpath}`.chomp.should eq("false Pipe") end diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index db4a57fab485..b8aa3f9c3af0 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -1,14 +1,14 @@ require "spec" -require "../spec_helper" +require "./spec_helper" describe "exit" do it "exits normally with status 0" do - status, _ = build_and_run "exit" + status, _ = compile_and_run_source "exit" status.success?.should be_true end it "exits with given error code" do - status, _ = build_and_run "exit 42" + status, _ = compile_and_run_source "exit 42" status.success?.should be_false status.exit_code.should eq(42) end @@ -16,7 +16,7 @@ end describe "at_exit" do it "runs handlers on normal program ending" do - status, output = build_and_run <<-CODE + status, output = compile_and_run_source <<-CODE at_exit do puts "handler code" end @@ -27,7 +27,7 @@ describe "at_exit" do end it "runs handlers on explicit program ending" do - status, output = build_and_run <<-'CODE' + status, output = compile_and_run_source <<-'CODE' at_exit do |exit_code| puts "handler code, exit code: #{exit_code}" end @@ -40,7 +40,7 @@ describe "at_exit" do end it "runs handlers in reverse order" do - status, output = build_and_run <<-CODE + status, output = compile_and_run_source <<-CODE at_exit do puts "first handler code" end @@ -59,7 +59,7 @@ describe "at_exit" do end it "runs all handlers maximum once" do - status, output = build_and_run <<-CODE + status, output = compile_and_run_source <<-CODE at_exit do puts "first handler code" end @@ -86,7 +86,7 @@ describe "at_exit" do end it "allows handlers to change the exit code with explicit `exit` call" do - status, output = build_and_run <<-'CODE' + status, output = compile_and_run_source <<-'CODE' at_exit do |exit_code| puts "first handler code, exit code: #{exit_code}" end @@ -114,7 +114,7 @@ describe "at_exit" do end it "allows handlers to change the exit code with explicit `exit` call (2)" do - status, output = build_and_run <<-'CODE' + status, output = compile_and_run_source <<-'CODE' at_exit do |exit_code| puts "first handler code, exit code: #{exit_code}" end @@ -144,7 +144,7 @@ describe "at_exit" do end it "changes final exit code when an handler raises an error" do - status, output, error = build_and_run <<-'CODE' + status, output, error = compile_and_run_source <<-'CODE' at_exit do |exit_code| puts "first handler code, exit code: #{exit_code}" end @@ -173,7 +173,7 @@ describe "at_exit" do end it "errors when used in an at_exit handler" do - status, output, error = build_and_run <<-CODE + status, output, error = compile_and_run_source <<-CODE at_exit do at_exit {} end @@ -184,7 +184,7 @@ describe "at_exit" do end it "shows unhandled exceptions after at_exit handlers" do - status, _, error = build_and_run <<-CODE + status, _, error = compile_and_run_source <<-CODE at_exit do STDERR.puts "first handler code" end @@ -205,7 +205,7 @@ describe "at_exit" do end it "can get unhandled exception in at_exit handler" do - status, _, error = build_and_run <<-CODE + status, _, error = compile_and_run_source <<-CODE at_exit do |_, ex| STDERR.puts ex.try &.message end @@ -223,7 +223,7 @@ end describe "seg fault" do it "reports SIGSEGV" do - status, _, error = build_and_run <<-'CODE' + status, _, error = compile_and_run_source <<-'CODE' puts Pointer(Int64).null.value CODE @@ -241,7 +241,7 @@ describe "seg fault" do # the default stack size is 0.5G. Setting a # smaller stack size with `ulimit -s 8192` # will address this. - status, _, error = build_and_run <<-'CODE' + status, _, error = compile_and_run_source <<-'CODE' def foo y = StaticArray(Int8,512).new(0) foo @@ -255,7 +255,7 @@ describe "seg fault" do {% end %} it "detects stack overflow on a fiber stack" do - status, _, error = build_and_run <<-'CODE' + status, _, error = compile_and_run_source <<-'CODE' def foo y = StaticArray(Int8,512).new(0) foo diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 135f6d259d4f..1fb65ffcc09d 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -1,7 +1,6 @@ require "spec" require "process" require "./spec_helper" -require "../spec_helper" private def exit_code_command(code) {% if flag?(:win32) %} @@ -138,7 +137,7 @@ describe Process do end pending_win32 "chroot raises when unprivileged" do - status, output = build_and_run <<-'CODE' + status, output = compile_and_run_source <<-'CODE' begin Process.chroot("/usr") puts "FAIL" diff --git a/spec/std/spec/hooks_spec.cr b/spec/std/spec/hooks_spec.cr index 4dac431b7e9b..7a7caf8a94c5 100644 --- a/spec/std/spec/hooks_spec.cr +++ b/spec/std/spec/hooks_spec.cr @@ -1,9 +1,9 @@ -require "../../spec_helper" +require "./spec_helper" describe Spec do describe "hooks" do it "runs in correct order" do - run(<<-CR).to_string.lines[..-5].should eq <<-OUT.lines + compile_and_run_source(<<-CR)[1].lines[..-5].should eq <<-OUT.lines require "prelude" require "spec" diff --git a/spec/std/spec_helper.cr b/spec/std/spec_helper.cr index b8f2704f08ea..faee17463a46 100644 --- a/spec/std/spec_helper.cr +++ b/spec/std/spec_helper.cr @@ -91,3 +91,37 @@ def spawn_and_check(before : Proc(_), file = __FILE__, line = __LINE__, &block : fail "Failed to stress expected path", file, line end end + +def compile_file(source_file, flags = %w(--debug)) + with_tempfile("executable_file") do |executable_file| + Process.run("bin/crystal", ["build"] + flags + ["-o", executable_file, source_file]) + File.exists?(executable_file).should be_true + + yield executable_file + end +end + +def compile_source(source, flags = %w(--debug)) + with_tempfile("source_file") do |source_file| + File.write(source_file, source) + compile_file(source_file, flags) do |executable_file| + yield executable_file + end + end +end + +def compile_and_run_file(source_file, flags = %w(--debug)) + compile_file(source_file) do |executable_file| + output, error = IO::Memory.new, IO::Memory.new + status = Process.run executable_file, output: output, error: error + + {status, output.to_s, error.to_s} + end +end + +def compile_and_run_source(source, flags = %w(--debug)) + with_tempfile("source_file") do |source_file| + File.write(source_file, source) + compile_and_run_file(source_file, flags) + end +end From 7f399f0fa740d3df63c3d7d2d14b822c916c63d6 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Thu, 14 May 2020 10:34:36 -0300 Subject: [PATCH 036/263] Use `_realpath$DARWIN_EXTSN` function in Darwin to fix issue with Catalina (#9296) --- src/lib_c/x86_64-darwin/c/stdlib.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib_c/x86_64-darwin/c/stdlib.cr b/src/lib_c/x86_64-darwin/c/stdlib.cr index d3325e2defd0..e7d1e35d2ecb 100644 --- a/src/lib_c/x86_64-darwin/c/stdlib.cr +++ b/src/lib_c/x86_64-darwin/c/stdlib.cr @@ -17,7 +17,7 @@ lib LibC fun mkstemps(x0 : Char*, x1 : Int) : Int fun putenv(x0 : Char*) : Int fun realloc(x0 : Void*, x1 : SizeT) : Void* - fun realpath(x0 : Char*, x1 : Char*) : Char* + fun realpath = "realpath$DARWIN_EXTSN"(x0 : Char*, x1 : Char*) : Char* fun setenv(x0 : Char*, x1 : Char*, x2 : Int) : Int fun strtof(x0 : Char*, x1 : Char**) : Float fun strtod(x0 : Char*, x1 : Char**) : Double From 2f34d0b9d323400ab4284004499b9c064b676233 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 14 May 2020 09:42:53 -0400 Subject: [PATCH 037/263] Add IO overloads to various String case methods (#9236) * Add IO overloads to various String case methods * Replace with_io_memory with String.build * Restore ascii optimization for non IO overloads --- spec/std/string_spec.cr | 183 ++++++++++++++++-------- src/string.cr | 304 ++++++++++++++++++++++++---------------- 2 files changed, 305 insertions(+), 182 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index f926dc124b23..44af6621348b 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -530,7 +530,7 @@ describe "String" do end end - describe "downcase" do + describe "#downcase" do it { "HELLO!".downcase.should eq("hello!") } it { "HELLO MAN!".downcase.should eq("hello man!") } it { "ÁÉÍÓÚĀ".downcase.should eq("áéíóúā") } @@ -541,9 +541,22 @@ describe "String" do it { "ff".downcase(Unicode::CaseOptions::Fold).should eq("ff") } it { "tschüß".downcase(Unicode::CaseOptions::Fold).should eq("tschüss") } it { "ΣίσυφοςfiÆ".downcase(Unicode::CaseOptions::Fold).should eq("σίσυφοσfiæ") } + + describe "with IO" do + it { String.build { |io| "HELLO!".downcase io }.should eq "hello!" } + it { String.build { |io| "HELLO MAN!".downcase io }.should eq "hello man!" } + it { String.build { |io| "ÁÉÍÓÚĀ".downcase io }.should eq "áéíóúā" } + it { String.build { |io| "AEIİOU".downcase io, Unicode::CaseOptions::Turkic }.should eq "aeıiou" } + it { String.build { |io| "ÁEÍOÚ".downcase io, Unicode::CaseOptions::ASCII }.should eq "ÁeÍoÚ" } + it { String.build { |io| "İ".downcase io }.should eq "i̇" } + it { String.build { |io| "Baffle".downcase io, Unicode::CaseOptions::Fold }.should eq "baffle" } + it { String.build { |io| "ff".downcase io, Unicode::CaseOptions::Fold }.should eq "ff" } + it { String.build { |io| "tschüß".downcase io, Unicode::CaseOptions::Fold }.should eq "tschüss" } + it { String.build { |io| "ΣίσυφοςfiÆ".downcase io, Unicode::CaseOptions::Fold }.should eq "σίσυφοσfiæ" } + end end - describe "upcase" do + describe "#upcase" do it { "hello!".upcase.should eq("HELLO!") } it { "hello man!".upcase.should eq("HELLO MAN!") } it { "áéíóúā".upcase.should eq("ÁÉÍÓÚĀ") } @@ -553,17 +566,37 @@ describe "String" do it { "baffle".upcase.should eq("BAFFLE") } it { "ff".upcase.should eq("FF") } it { "ňž".upcase.should eq("ŇŽ") } # #7922 + + describe "with IO" do + it { String.build { |io| "hello!".upcase io }.should eq "HELLO!" } + it { String.build { |io| "hello man!".upcase io }.should eq "HELLO MAN!" } + it { String.build { |io| "áéíóúā".upcase io }.should eq "ÁÉÍÓÚĀ" } + it { String.build { |io| "aeıiou".upcase io, Unicode::CaseOptions::Turkic }.should eq "AEIİOU" } + it { String.build { |io| "áeíoú".upcase io, Unicode::CaseOptions::ASCII }.should eq "áEíOú" } + it { String.build { |io| "aeiou".upcase io, Unicode::CaseOptions::Turkic }.should eq "AEİOU" } + it { String.build { |io| "baffle".upcase io }.should eq "BAFFLE" } + it { String.build { |io| "ff".upcase io }.should eq "FF" } + it { String.build { |io| "ňž".upcase io }.should eq "ŇŽ" } + end end - describe "capitalize" do + describe "#capitalize" do it { "HELLO!".capitalize.should eq("Hello!") } it { "HELLO MAN!".capitalize.should eq("Hello man!") } it { "".capitalize.should eq("") } it { "fflİ".capitalize.should eq("FFLi̇") } it { "iO".capitalize(Unicode::CaseOptions::Turkic).should eq("İo") } + + describe "with IO" do + it { String.build { |io| "HELLO!".capitalize io }.should eq "Hello!" } + it { String.build { |io| "HELLO MAN!".capitalize io }.should eq "Hello man!" } + it { String.build { |io| "".capitalize io }.should be_empty } + it { String.build { |io| "fflİ".capitalize io }.should eq "FFLi̇" } + it { String.build { |io| "iO".capitalize io, Unicode::CaseOptions::Turkic }.should eq "İo" } + end end - describe "titleize" do + describe "#titleize" do it { "hEllO tAb\tworld".titleize.should eq("Hello Tab\tWorld") } it { " spaces before".titleize.should eq(" Spaces Before") } it { "testa-se muito".titleize.should eq("Testa-se Muito") } @@ -571,6 +604,16 @@ describe "String" do it { " spáçes before".titleize.should eq(" Spáçes Before") } it { "testá-se múitô".titleize.should eq("Testá-se Múitô") } it { "iO iO".titleize(Unicode::CaseOptions::Turkic).should eq("İo İo") } + + describe "with IO" do + it { String.build { |io| "hEllO tAb\tworld".titleize io }.should eq "Hello Tab\tWorld" } + it { String.build { |io| " spaces before".titleize io }.should eq " Spaces Before" } + it { String.build { |io| "testa-se muito".titleize io }.should eq "Testa-se Muito" } + it { String.build { |io| "hÉllÕ tAb\tworld".titleize io }.should eq "Héllõ Tab\tWorld" } + it { String.build { |io| " spáçes before".titleize io }.should eq " Spáçes Before" } + it { String.build { |io| "testá-se múitô".titleize io }.should eq "Testá-se Múitô" } + it { String.build { |io| "iO iO".titleize io, Unicode::CaseOptions::Turkic }.should eq "İo İo" } + end end describe "chomp" do @@ -1869,32 +1912,58 @@ describe "String" do end end - it "does underscore" do - "Foo".underscore.should eq("foo") - "FooBar".underscore.should eq("foo_bar") - "ABCde".underscore.should eq("ab_cde") - "FOO_bar".underscore.should eq("foo_bar") - "Char_S".underscore.should eq("char_s") - "Char_".underscore.should eq("char_") - "C_".underscore.should eq("c_") - "HTTP".underscore.should eq("http") - "HTTP_CLIENT".underscore.should eq("http_client") - "CSS3".underscore.should eq("css3") - "HTTP1.1".underscore.should eq("http1.1") - "3.14IsPi".underscore.should eq("3.14_is_pi") - "I2C".underscore.should eq("i2_c") - end - - it "does camelcase" do - "foo".camelcase.should eq("Foo") - "foo_bar".camelcase.should eq("FooBar") - "foo".camelcase(lower: true).should eq("foo") - "foo_bar".camelcase(lower: true).should eq("fooBar") - - "Foo".camelcase.should eq("Foo") - "Foo_bar".camelcase.should eq("FooBar") - "Foo".camelcase(lower: true).should eq("foo") - "Foo_bar".camelcase(lower: true).should eq("fooBar") + describe "#underscore" do + it { "Foo".underscore.should eq "foo" } + it { "FooBar".underscore.should eq "foo_bar" } + it { "ABCde".underscore.should eq "ab_cde" } + it { "FOO_bar".underscore.should eq "foo_bar" } + it { "Char_S".underscore.should eq "char_s" } + it { "Char_".underscore.should eq "char_" } + it { "C_".underscore.should eq "c_" } + it { "HTTP".underscore.should eq "http" } + it { "HTTP_CLIENT".underscore.should eq "http_client" } + it { "CSS3".underscore.should eq "css3" } + it { "HTTP1.1".underscore.should eq "http1.1" } + it { "3.14IsPi".underscore.should eq "3.14_is_pi" } + it { "I2C".underscore.should eq "i2_c" } + + describe "with IO" do + it { String.build { |io| "Foo".underscore io }.should eq "foo" } + it { String.build { |io| "FooBar".underscore io }.should eq "foo_bar" } + it { String.build { |io| "ABCde".underscore io }.should eq "ab_cde" } + it { String.build { |io| "FOO_bar".underscore io }.should eq "foo_bar" } + it { String.build { |io| "Char_S".underscore io }.should eq "char_s" } + it { String.build { |io| "Char_".underscore io }.should eq "char_" } + it { String.build { |io| "C_".underscore io }.should eq "c_" } + it { String.build { |io| "HTTP".underscore io }.should eq "http" } + it { String.build { |io| "HTTP_CLIENT".underscore io }.should eq "http_client" } + it { String.build { |io| "CSS3".underscore io }.should eq "css3" } + it { String.build { |io| "HTTP1.1".underscore io }.should eq "http1.1" } + it { String.build { |io| "3.14IsPi".underscore io }.should eq "3.14_is_pi" } + it { String.build { |io| "I2C".underscore io }.should eq "i2_c" } + end + end + + describe "#camelcase" do + it { "foo".camelcase.should eq "Foo" } + it { "foo_bar".camelcase.should eq "FooBar" } + it { "foo".camelcase(lower: true).should eq "foo" } + it { "foo_bar".camelcase(lower: true).should eq "fooBar" } + it { "Foo".camelcase.should eq "Foo" } + it { "Foo_bar".camelcase.should eq "FooBar" } + it { "Foo".camelcase(lower: true).should eq "foo" } + it { "Foo_bar".camelcase(lower: true).should eq "fooBar" } + + describe "with IO" do + it { String.build { |io| "foo".camelcase io }.should eq "Foo" } + it { String.build { |io| "foo_bar".camelcase io }.should eq "FooBar" } + it { String.build { |io| "foo".camelcase io, lower: true }.should eq "foo" } + it { String.build { |io| "foo_bar".camelcase io, lower: true }.should eq "fooBar" } + it { String.build { |io| "Foo".camelcase io }.should eq "Foo" } + it { String.build { |io| "Foo_bar".camelcase io }.should eq "FooBar" } + it { String.build { |io| "Foo".camelcase io, lower: true }.should eq "foo" } + it { String.build { |io| "Foo_bar".camelcase io, lower: true }.should eq "fooBar" } + end end it "answers ascii_only?" do @@ -2031,17 +2100,17 @@ describe "String" do it { "12".ljust(7, 'あ').should eq("12あああああ") } describe "to io" do - it { with_io_memory { |io| "123".ljust(io, 2) }.should eq("123") } - it { with_io_memory { |io| "123".ljust(io, 5) }.should eq("123 ") } - it { with_io_memory { |io| "12".ljust(io, 7, '-') }.should eq("12-----") } - it { with_io_memory { |io| "12".ljust(io, 7, 'あ') }.should eq("12あああああ") } + it { String.build { |io| "123".ljust(io, 2) }.should eq("123") } + it { String.build { |io| "123".ljust(io, 5) }.should eq("123 ") } + it { String.build { |io| "12".ljust(io, 7, '-') }.should eq("12-----") } + it { String.build { |io| "12".ljust(io, 7, 'あ') }.should eq("12あああああ") } end describe "to io (deprecated)" do - it { with_io_memory { |io| "123".ljust(2, io) }.should eq("123") } - it { with_io_memory { |io| "123".ljust(5, io) }.should eq("123 ") } - it { with_io_memory { |io| "12".ljust(7, '-', io) }.should eq("12-----") } - it { with_io_memory { |io| "12".ljust(7, 'あ', io) }.should eq("12あああああ") } + it { String.build { |io| "123".ljust(2, io) }.should eq("123") } + it { String.build { |io| "123".ljust(5, io) }.should eq("123 ") } + it { String.build { |io| "12".ljust(7, '-', io) }.should eq("12-----") } + it { String.build { |io| "12".ljust(7, 'あ', io) }.should eq("12あああああ") } end end @@ -2052,17 +2121,17 @@ describe "String" do it { "12".rjust(7, 'あ').should eq("あああああ12") } describe "to io" do - it { with_io_memory { |io| "123".rjust(io, 2) }.should eq("123") } - it { with_io_memory { |io| "123".rjust(io, 5) }.should eq(" 123") } - it { with_io_memory { |io| "12".rjust(io, 7, '-') }.should eq("-----12") } - it { with_io_memory { |io| "12".rjust(io, 7, 'あ') }.should eq("あああああ12") } + it { String.build { |io| "123".rjust(io, 2) }.should eq("123") } + it { String.build { |io| "123".rjust(io, 5) }.should eq(" 123") } + it { String.build { |io| "12".rjust(io, 7, '-') }.should eq("-----12") } + it { String.build { |io| "12".rjust(io, 7, 'あ') }.should eq("あああああ12") } end describe "to io (deprecated)" do - it { with_io_memory { |io| "123".rjust(2, io) }.should eq("123") } - it { with_io_memory { |io| "123".rjust(5, io) }.should eq(" 123") } - it { with_io_memory { |io| "12".rjust(7, '-', io) }.should eq("-----12") } - it { with_io_memory { |io| "12".rjust(7, 'あ', io) }.should eq("あああああ12") } + it { String.build { |io| "123".rjust(2, io) }.should eq("123") } + it { String.build { |io| "123".rjust(5, io) }.should eq(" 123") } + it { String.build { |io| "12".rjust(7, '-', io) }.should eq("-----12") } + it { String.build { |io| "12".rjust(7, 'あ', io) }.should eq("あああああ12") } end end @@ -2073,17 +2142,17 @@ describe "String" do it { "12".center(7, 'あ').should eq("ああ12あああ") } describe "to io" do - it { with_io_memory { |io| "123".center(io, 2) }.should eq("123") } - it { with_io_memory { |io| "123".center(io, 5) }.should eq(" 123 ") } - it { with_io_memory { |io| "12".center(io, 7, '-') }.should eq("--12---") } - it { with_io_memory { |io| "12".center(io, 7, 'あ') }.should eq("ああ12あああ") } + it { String.build { |io| "123".center(io, 2) }.should eq("123") } + it { String.build { |io| "123".center(io, 5) }.should eq(" 123 ") } + it { String.build { |io| "12".center(io, 7, '-') }.should eq("--12---") } + it { String.build { |io| "12".center(io, 7, 'あ') }.should eq("ああ12あああ") } end describe "to io (deprecated)" do - it { with_io_memory { |io| "123".center(2, io) }.should eq("123") } - it { with_io_memory { |io| "123".center(5, io) }.should eq(" 123 ") } - it { with_io_memory { |io| "12".center(7, '-', io) }.should eq("--12---") } - it { with_io_memory { |io| "12".center(7, 'あ', io) }.should eq("ああ12あああ") } + it { String.build { |io| "123".center(2, io) }.should eq("123") } + it { String.build { |io| "123".center(5, io) }.should eq(" 123 ") } + it { String.build { |io| "12".center(7, '-', io) }.should eq("--12---") } + it { String.build { |io| "12".center(7, 'あ', io) }.should eq("ああ12あああ") } end end @@ -2656,9 +2725,3 @@ describe "String" do end end end - -private def with_io_memory - io = IO::Memory.new - yield io - io.to_s -end diff --git a/src/string.cr b/src/string.cr index 00075fe93635..cc45f7e41a89 100644 --- a/src/string.cr +++ b/src/string.cr @@ -1056,60 +1056,77 @@ class String end end - def unsafe_byte_at(index) + # Returns the byte at the given *index* without bounds checking. + def unsafe_byte_at(index : Int) : UInt8 to_unsafe[index] end - # Returns a new `String` with each uppercase letter replaced with its lowercase - # counterpart. + # Returns a new `String` with each uppercase letter replaced with its lowercase counterpart. # # ``` # "hEllO".downcase # => "hello" # ``` - def downcase(options = Unicode::CaseOptions::None) + def downcase(options : Unicode::CaseOptions = :none) : String return self if empty? if ascii_only? && (options.none? || options.ascii?) - String.new(bytesize) do |buffer| + return String.new(bytesize) do |buffer| bytesize.times do |i| - buffer[i] = to_unsafe[i].unsafe_chr.downcase.ord.to_u8 + buffer[i] = unsafe_byte_at(i).unsafe_chr.downcase.ord.to_u8 end {@bytesize, @length} end - else - String.build(bytesize) do |io| - each_char do |char| - char.downcase(options) do |res| - io << res - end - end + end + + String.build(bytesize) { |io| downcase io, options } + end + + # Writes a downcased version of `self` to the given *io*. + # + # ``` + # io = IO::Memory.new + # "hEllO".downcase io + # io.to_s # => hello + # ``` + def downcase(io : IO, options : Unicode::CaseOptions = :none) : Nil + each_char do |char| + char.downcase(options) do |res| + io << res end end end - # Returns a new `String` with each lowercase letter replaced with its uppercase - # counterpart. + # Returns a new `String` with each lowercase letter replaced with its uppercase counterpart. # # ``` # "hEllO".upcase # => "HELLO" # ``` - def upcase(options = Unicode::CaseOptions::None) + def upcase(options : Unicode::CaseOptions = :none) : String return self if empty? if ascii_only? && (options.none? || options.ascii?) - String.new(bytesize) do |buffer| + return String.new(bytesize) do |buffer| bytesize.times do |i| - buffer[i] = to_unsafe[i].unsafe_chr.upcase.ord.to_u8 + buffer[i] = unsafe_byte_at(i).unsafe_chr.upcase.ord.to_u8 end {@bytesize, @length} end - else - String.build(bytesize) do |io| - each_char do |char| - char.upcase(options) do |res| - io << res - end - end + end + + String.build(bytesize) { |io| upcase io, options } + end + + # Writes a upcased version of `self` to the given *io*. + # + # ``` + # io = IO::Memory.new + # "hEllO".upcase io + # io.to_s # => HELLO + # ``` + def upcase(io : IO, options : Unicode::CaseOptions = :none) : Nil + each_char do |char| + char.upcase(options) do |res| + io << res end end end @@ -1120,29 +1137,40 @@ class String # ``` # "hEllO".capitalize # => "Hello" # ``` - def capitalize(options = Unicode::CaseOptions::None) + def capitalize(options : Unicode::CaseOptions = :none) : String return self if empty? if ascii_only? && (options.none? || options.ascii?) - String.new(bytesize) do |buffer| + return String.new(bytesize) do |buffer| bytesize.times do |i| - if i == 0 - buffer[i] = to_unsafe[i].unsafe_chr.upcase.ord.to_u8 - else - buffer[i] = to_unsafe[i].unsafe_chr.downcase.ord.to_u8 - end + byte = if i.zero? + unsafe_byte_at(i).unsafe_chr.upcase.ord.to_u8 + else + unsafe_byte_at(i).unsafe_chr.downcase.ord.to_u8 + end + + buffer[i] = byte end {@bytesize, @length} end - else - String.build(bytesize) do |io| - each_char_with_index do |char, i| - if i == 0 - char.upcase(options) { |c| io << c } - else - char.downcase(options) { |c| io << c } - end - end + end + + String.build(bytesize) { |io| capitalize io, options } + end + + # Writes a capitalized version of `self` to the given *io*. + # + # ``` + # io = IO::Memory.new + # "hEllO".capitalize io + # io.to_s # => Hello + # ``` + def capitalize(io : IO, options : Unicode::CaseOptions = :none) : Nil + each_char_with_index do |char, i| + if i.zero? + char.upcase(options) { |c| io << c } + else + char.downcase(options) { |c| io << c } end end end @@ -1155,28 +1183,40 @@ class String # " spaces before".titleize # => " Spaces Before" # "x-men: the last stand".titleize # => "X-men: The Last Stand" # ``` - def titleize(options = Unicode::CaseOptions::None) + def titleize(options : Unicode::CaseOptions = :none) : String return self if empty? - upcase_next = true if ascii_only? && (options.none? || options.ascii?) - String.new(bytesize) do |buffer| + upcase_next = true + + return String.new(bytesize) do |buffer| bytesize.times do |i| - char = to_unsafe[i].unsafe_chr + char = unsafe_byte_at(i).unsafe_chr replaced_char = upcase_next ? char.upcase : char.downcase buffer[i] = replaced_char.ord.to_u8 upcase_next = char.whitespace? end {@bytesize, @length} end - else - String.build(bytesize) do |io| - each_char_with_index do |char, i| - replaced_char = upcase_next ? char.upcase(options) : char.downcase(options) - io << replaced_char - upcase_next = char.whitespace? - end - end + end + + String.build(bytesize) { |io| titleize io, options } + end + + # Writes a titleized version of `self` to the given *io*. + # + # ``` + # io = IO::Memory.new + # "x-men: the last stand".titleize io + # io.to_s # => X-men: The Last Stand + # ``` + def titleize(io : IO, options : Unicode::CaseOptions = :none) : Nil + upcase_next = true + + each_char_with_index do |char, i| + replaced_char = upcase_next ? char.upcase(options) : char.downcase(options) + io << replaced_char + upcase_next = char.whitespace? end end @@ -3672,74 +3712,83 @@ class String # "3.14IsPi".underscore # => "3.14_is_pi" # "InterestingImage".underscore(Unicode::CaseOptions::Turkic) # => "ınteresting_ımage" # ``` - def underscore(options : Unicode::CaseOptions = Unicode::CaseOptions::None) + def underscore(options : Unicode::CaseOptions = :none) : String + String.build(bytesize + 10) { |io| underscore io, options } + end + + # Writes an underscored version of `self` to the given *io*. + # + # ``` + # io = IO::Memory.new + # "DoesWhatItSaysOnTheTin".underscore io + # io.to_s # => "does_what_it_says_on_the_tin" + # ``` + def underscore(io : IO, options : Unicode::CaseOptions = :none) : Nil first = true last_is_downcase = false last_is_upcase = false last_is_digit = false - mem = nil + mem : Char? = nil - String.build(bytesize + 10) do |str| - each_char do |char| - digit = char.ascii_number? + each_char do |char| + digit = char.ascii_number? - if options.none? - downcase = digit || char.ascii_lowercase? - upcase = char.ascii_uppercase? - else - downcase = digit || char.lowercase? - upcase = char.uppercase? - end + if options.none? + downcase = digit || char.ascii_lowercase? + upcase = char.ascii_uppercase? + else + downcase = digit || char.lowercase? + upcase = char.uppercase? + end - if first - str << char.downcase(options) - elsif last_is_downcase && upcase - if mem - # This is the case of A1Bcd, we need to put 'mem' (not to need to convert as downcase - # ^ - # because 'mem' is digit surely) before putting this char as downcase. - str << mem - mem = nil - end - # This is the case of AbcDe, we need to put an underscore before the 'D' - # ^ - str << '_' - str << char.downcase(options) - elsif (last_is_upcase || last_is_digit) && (upcase || digit) - # This is the case of 1) A1Bcd, 2) A1BCd or 3) A1B_cd:if the next char is upcase (case 1) we need - # ^ ^ ^ - # 1) we need to append this char as downcase - # 2) we need to append an underscore and then the char as downcase, so we save this char - # in 'mem' and decide later - # 3) we need to append this char as downcase and then a single underscore - if mem - # case 2 - str << mem.downcase(options) - end - mem = char - else - if mem - if char == '_' - # case 3 - elsif last_is_upcase && downcase - # case 1 - str << '_' - end - str << mem.downcase(options) - mem = nil + if first + io << char.downcase(options) + elsif last_is_downcase && upcase + if mem + # This is the case of A1Bcd, we need to put 'mem' (not to need to convert as downcase + # ^ + # because 'mem' is digit surely) before putting this char as downcase. + io << mem + mem = nil + end + # This is the case of AbcDe, we need to put an underscore before the 'D' + # ^ + io << '_' + io << char.downcase(options) + elsif (last_is_upcase || last_is_digit) && (upcase || digit) + # This is the case of 1) A1Bcd, 2) A1BCd or 3) A1B_cd:if the next char is upcase (case 1) we need + # ^ ^ ^ + # 1) we need to append this char as downcase + # 2) we need to append an underscore and then the char as downcase, so we save this char + # in 'mem' and decide later + # 3) we need to append this char as downcase and then a single underscore + if mem + # case 2 + io << mem.downcase(options) + end + mem = char + else + if mem + if char == '_' + # case 3 + elsif last_is_upcase && downcase + # case 1 + io << '_' end - - str << char.downcase(options) + io << mem.downcase(options) + mem = nil end - last_is_downcase = downcase - last_is_upcase = upcase - last_is_digit = digit - first = false + io << char.downcase(options) end - str << mem.downcase(options) if mem + last_is_downcase = downcase + last_is_upcase = upcase + last_is_digit = digit + first = false end + + io << mem.downcase(options) if mem end # Converts underscores to camelcase boundaries. @@ -3751,26 +3800,37 @@ class String # "empire_state_building".camelcase(lower: true) # => "empireStateBuilding" # "isolated_integer".camelcase(options: Unicode::CaseOptions::Turkic) # => "İsolatedİnteger" # ``` - def camelcase(options : Unicode::CaseOptions = Unicode::CaseOptions::None, *, lower : Bool = false) + def camelcase(options : Unicode::CaseOptions = Unicode::CaseOptions::None, *, lower : Bool = false) : String return self if empty? + String.build(bytesize) { |io| camelcase io, options, lower: lower } + end + + # Writes an camelcased version of `self` to the given *io*. + # + # If *lower* is true, lower camelcase will be written (the first letter is downcased). + # + # ``` + # io = IO::Memory.new + # "eiffel_tower".camelcase io + # io.to_s # => "EiffelTower" + # ``` + def camelcase(io : IO, options : Unicode::CaseOptions = Unicode::CaseOptions::None, *, lower : Bool = false) : Nil first = true last_is_underscore = false - String.build(bytesize) do |str| - each_char do |char| - if first - str << (lower ? char.downcase(options) : char.upcase(options)) - elsif char == '_' - last_is_underscore = true - elsif last_is_underscore - str << char.upcase(options) - last_is_underscore = false - else - str << char - end - first = false + each_char do |char| + if first + io << (lower ? char.downcase(options) : char.upcase(options)) + elsif char == '_' + last_is_underscore = true + elsif last_is_underscore + io << char.upcase(options) + last_is_underscore = false + else + io << char end + first = false end end From b977a952e142efe5ee63e65e7820eec2b55a0c07 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Thu, 14 May 2020 09:43:48 -0400 Subject: [PATCH 038/263] Escape CDATA end sequences (#9230) * Escape CDATA end sequences * Update src/xml/builder.cr Co-authored-by: Sijawusz Pur Rahnama * Add backticks to other CDATA reference Co-authored-by: Sijawusz Pur Rahnama --- spec/std/xml/builder_spec.cr | 30 ++++++++++++++++++++---------- src/xml/builder.cr | 12 +++++++++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/spec/std/xml/builder_spec.cr b/spec/std/xml/builder_spec.cr index 24bc21ef802f..10ca184fe06e 100644 --- a/spec/std/xml/builder_spec.cr +++ b/spec/std/xml/builder_spec.cr @@ -170,19 +170,29 @@ describe XML::Builder do end end - it "writes cdata" do - assert_built(%{\n\n}) do |xml| - element("foo") do - cdata("hello") + describe "#cdata" do + it "writes cdata" do + assert_built(%{\n\n}) do |xml| + element("foo") do + cdata("hello") + end end end - end - it "writes cdata with block" do - assert_built(%{\n\n}) do |xml| - element("foo") do - cdata do - text "hello" + it "escapes ]]> sequences" do + assert_built(%{\nTwo]]]]>Three]]>\n}) do |xml| + element("foo") do + cdata("One]]>Two]]>Three") + end + end + end + + it "writes cdata with block" do + assert_built(%{\n\n}) do |xml| + element("foo") do + cdata do + text "hello" + end end end end diff --git a/src/xml/builder.cr b/src/xml/builder.cr index ba8716a920bf..b5e30addcd97 100644 --- a/src/xml/builder.cr +++ b/src/xml/builder.cr @@ -5,6 +5,9 @@ # without a matching `start_element`, or trying to use # a non-string value as an object's field name) struct XML::Builder + private CDATA_END = "]]>" + private CDATA_ESCAPE = "]]]]>" + @box : Void* # Creates a builder that writes to the given *io*. @@ -180,14 +183,17 @@ struct XML::Builder # Emits the start of a `CDATA` section, invokes the block # and then emits the end of the `CDATA` section. - def cdata + # + # NOTE: `CDATA` end sequences written within the block + # need to be escaped manually. + def cdata(&) start_cdata yield.tap { end_cdata } end - # Emits a `CDATA` section. + # Emits a `CDATA` section. Escapes nested `CDATA` end sequences. def cdata(text : String) : Nil - call WriteCDATA, string_to_unsafe(text) + call WriteCDATA, string_to_unsafe(text.gsub(CDATA_END, CDATA_ESCAPE)) end # Emits the start of a comment. From 1e19a230e54b26c025c001a4af467bc28519fb8f Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Fri, 15 May 2020 21:05:17 +0900 Subject: [PATCH 039/263] Add `StringLiteral#titleize` macro method (#9269) * Add `StringLiteral#titleize` macro method This also adds `SymbolLiteral#titleize` and `MacroId#titleize`. Other case conversion methods (`upcase`, `camelcase`, etc) are avialble in macro. So, `#titleize` should not be exception. One use-case is generating error message from type name. * Add specs for `SymbolLiteral#titleize` and `MacroId#titleize` --- spec/compiler/macro/macro_methods_spec.cr | 6 ++++++ src/compiler/crystal/macros.cr | 12 ++++++++++++ src/compiler/crystal/macros/methods.cr | 2 ++ 3 files changed, 20 insertions(+) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index c7341f877b2a..cd4afd780b27 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -430,6 +430,10 @@ module Crystal assert_macro "", %({{"FooBar".underscore}}), [] of ASTNode, %("foo_bar") end + it "executes titleize" do + assert_macro "", %({{"hello world".titleize}}), [] of ASTNode, %("Hello World") + end + it "executes to_i" do assert_macro "", %({{"1234".to_i}}), [] of ASTNode, %(1234) end @@ -477,6 +481,7 @@ module Crystal assert_macro "x", %({{x.starts_with?("hel")}}), [MacroId.new("hello")] of ASTNode, %(true) assert_macro "x", %({{x.chomp}}), [MacroId.new("hello\n")] of ASTNode, %(hello) assert_macro "x", %({{x.upcase}}), [MacroId.new("hello")] of ASTNode, %(HELLO) + assert_macro "x", %({{x.titleize}}), [MacroId.new("hello world")] of ASTNode, %(Hello World) assert_macro "x", %({{x.includes?("el")}}), [MacroId.new("hello")] of ASTNode, %(true) assert_macro "x", %({{x.includes?("he")}}), [MacroId.new("hello")] of ASTNode, %(true) assert_macro "x", %({{x.includes?("EL")}}), [MacroId.new("hello")] of ASTNode, %(false) @@ -519,6 +524,7 @@ module Crystal assert_macro "x", %({{x.starts_with?("hel")}}), ["hello".symbol] of ASTNode, %(true) assert_macro "x", %({{x.chomp}}), [SymbolLiteral.new("hello\n")] of ASTNode, %(:hello) assert_macro "x", %({{x.upcase}}), ["hello".symbol] of ASTNode, %(:HELLO) + assert_macro "x", %({{x.titleize}}), ["hello world".symbol] of ASTNode, %(:"Hello World") assert_macro "x", %({{x.includes?("el")}}), ["hello".symbol] of ASTNode, %(true) assert_macro "x", %({{x.includes?("he")}}), ["hello".symbol] of ASTNode, %(true) assert_macro "x", %({{x.includes?("EL")}}), ["hello".symbol] of ASTNode, %(false) diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index 02ea666951a5..57962d295792 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -466,6 +466,10 @@ module Crystal::Macros def strip : StringLiteral end + # Similar to `String#titleize`. + def titleize : StringLiteral + end + # Similar to `String#to_i`. def to_i(base = 10) end @@ -567,6 +571,10 @@ module Crystal::Macros def strip : SymbolLiteral end + # Similar to `String#titleize`. + def titleize : SymbolLiteral + end + # Similar to `String#tr`. def tr(from : StringLiteral, to : StringLiteral) : SymbolLiteral end @@ -1727,6 +1735,10 @@ module Crystal::Macros def strip : MacroId end + # Similar to `String#titleize`. + def titleize : MacroId + end + # Similar to `String#tr`. def tr(from : StringLiteral, to : StringLiteral) : MacroId end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 6937d06116f9..2000a2284349 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -760,6 +760,8 @@ module Crystal end when "strip" interpret_argless_method(method, args) { StringLiteral.new(@value.strip) } + when "titleize" + interpret_argless_method(method, args) { StringLiteral.new(@value.titleize) } when "to_i" case args.size when 0 From d49375b181b9803a5a405b9b04846dce6aec2870 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Fri, 15 May 2020 08:05:36 -0400 Subject: [PATCH 040/263] `TypeNode` methods to check what "type" the node is (#9270) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add methods to `TypeNode` to check if its a module, class, or struct Adds specs and updates docs for `#nilable?`, `#abstract?` and `#union?` * Fix typo in struct? documentation Co-authored-by: Jonne Haß Co-authored-by: Jonne Haß --- spec/compiler/macro/macro_methods_spec.cr | 297 +++++++++++++++++++++- src/compiler/crystal/macros.cr | 85 ++++++- src/compiler/crystal/macros/methods.cr | 6 + 3 files changed, 377 insertions(+), 11 deletions(-) diff --git a/spec/compiler/macro/macro_methods_spec.cr b/spec/compiler/macro/macro_methods_spec.cr index cd4afd780b27..0739ad312fe6 100644 --- a/spec/compiler/macro/macro_methods_spec.cr +++ b/spec/compiler/macro/macro_methods_spec.cr @@ -1658,15 +1658,300 @@ module Crystal end end - it "executes nilable? (false)" do - assert_macro("x", "{{x.nilable?}}", "false") do |program| - [TypeNode.new(program.string)] of ASTNode + describe "#abstract?" do + it NonGenericModuleType do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + mod = NonGenericModuleType.new(program, program, "SomeModule") + + [TypeNode.new(mod)] of ASTNode + end + end + + it GenericModuleType do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + generic_mod = GenericModuleType.new(program, program, "SomeGenericModule", ["T"]) + + [TypeNode.new(generic_mod)] of ASTNode + end + end + + describe NonGenericClassType do + describe "class" do + it "abstract" do + assert_macro("type", "{{type.abstract?}}", "true") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.abstract = true + + [TypeNode.new(klass)] of ASTNode + end + end + + it "non-abstract" do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + + [TypeNode.new(klass)] of ASTNode + end + end + end + + describe "struct" do + it "abstract" do + assert_macro("type", "{{type.abstract?}}", "true") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.abstract = true + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + + it "non-abstract" do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + end + + describe GenericClassType do + describe "class" do + it "abstract" do + assert_macro("type", "{{type.abstract?}}", "true") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.abstract = true + + [TypeNode.new(klass)] of ASTNode + end + end + + it "non-abstract" do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + + [TypeNode.new(klass)] of ASTNode + end + end + end + + describe "struct" do + it "abstract" do + assert_macro("type", "{{type.abstract?}}", "true") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.abstract = true + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + + it "non-abstract" do + assert_macro("type", "{{type.abstract?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + end + end + + describe "#union?" do + it true do + assert_macro("x", "{{x.union?}}", "true") do |program| + [TypeNode.new(program.union_of(program.string, program.nil))] of ASTNode + end + end + + it false do + assert_macro("x", "{{x.union?}}", "false") do |program| + [TypeNode.new(program.string)] of ASTNode + end + end + end + + describe "#module?" do + it NonGenericModuleType do + assert_macro("type", "{{type.module?}}", "true") do |program| + mod = NonGenericModuleType.new(program, program, "SomeModule") + + [TypeNode.new(mod)] of ASTNode + end + end + + it GenericModuleType do + assert_macro("type", "{{type.module?}}", "true") do |program| + generic_mod = GenericModuleType.new(program, program, "SomeGenericModule", ["T"]) + + [TypeNode.new(generic_mod)] of ASTNode + end + end + + describe NonGenericClassType do + it "class" do + assert_macro("type", "{{type.module?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.module?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + + describe GenericClassType do + it "class" do + assert_macro("type", "{{type.module?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.module?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + end + + describe "#class?" do + it NonGenericModuleType do + assert_macro("type", "{{type.class?}}", "false") do |program| + mod = NonGenericModuleType.new(program, program, "SomeModule") + + [TypeNode.new(mod)] of ASTNode + end + end + + it GenericModuleType do + assert_macro("type", "{{type.class?}}", "false") do |program| + generic_mod = GenericModuleType.new(program, program, "SomeGenericModule", ["T"]) + + [TypeNode.new(generic_mod)] of ASTNode + end + end + + describe NonGenericClassType do + it "class" do + assert_macro("type", "{{type.class?}}", "true") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.class?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + + describe GenericClassType do + it "class" do + assert_macro("type", "{{type.class?}}", "true") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.class?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end end end - it "executes nilable? (true)" do - assert_macro("x", "{{x.nilable?}}", "true") do |program| - [TypeNode.new(program.union_of(program.string, program.nil))] of ASTNode + describe "#struct?" do + it NonGenericModuleType do + assert_macro("type", "{{type.struct?}}", "false") do |program| + mod = NonGenericModuleType.new(program, program, "SomeModule") + + [TypeNode.new(mod)] of ASTNode + end + end + + it GenericModuleType do + assert_macro("type", "{{type.struct?}}", "false") do |program| + generic_mod = GenericModuleType.new(program, program, "SomeGenericModule", ["T"]) + + [TypeNode.new(generic_mod)] of ASTNode + end + end + + describe NonGenericClassType do + it "class" do + assert_macro("type", "{{type.struct?}}", "false") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.struct?}}", "true") do |program| + klass = NonGenericClassType.new(program, program, "SomeType", program.reference) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + + describe GenericClassType do + it "class" do + assert_macro("type", "{{type.struct?}}", "false") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + + [TypeNode.new(klass)] of ASTNode + end + end + + it "struct" do + assert_macro("type", "{{type.struct?}}", "true") do |program| + klass = GenericClassType.new(program, program, "SomeGenericType", program.reference, ["T"]) + klass.struct = true + + [TypeNode.new(klass)] of ASTNode + end + end + end + end + describe "#nilable?" do + it false do + assert_macro("x", "{{x.nilable?}}", "false") do |program| + [TypeNode.new(program.string)] of ASTNode + end + end + + it true do + assert_macro("x", "{{x.nilable?}}", "true") do |program| + [TypeNode.new(program.union_of(program.string, program.nil))] of ASTNode + end end end diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index 57962d295792..ad26ceb38429 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -1750,24 +1750,99 @@ module Crystal::Macros # Represents a type in the program, like `Int32` or `String`. class TypeNode < ASTNode - # Returns `true` if this type is abstract. + # Returns `true` if `self` is abstract, otherwise `false`. + # + # ``` + # module One; end + # + # abstract struct Two; end + # + # class Three; end + # + # abstract class Four; end + # + # {{One.abstract?}} # => false + # {{Two.abstract?}} # => true + # {{Three.abstract?}} # => false + # {{Four.abstract?}} # => true + # ``` def abstract? : BoolLiteral end - # Returns `true` if this type is a union type, `false` otherwise. + # Returns `true` if `self` is a union type, otherwise `false`. + # + # See also: `#union_types`. # - # See also: `union_types`. + # ``` + # {{String.union?}} # => false + # {{String?.union?}} # => true + # {{Union(String, Bool).union?}} # => true + # ``` def union? : BoolLiteral end - # Returns `true` if this type is nilable (if it has `Nil` amongst its types). + # Returns `true` if `self` is nilable (if it has `Nil` amongst its types), otherwise `false`. + # + # ``` + # {{String.nilable?}} # => false + # {{String?.nilable?}} # => true + # {{Union(String, Bool, Nil).nilable?}} # => true + # ``` def nilable? : BoolLiteral end + # Returns `true` if `self` is a `module`, otherwise `false`. + # + # ``` + # module One; end + # + # class Two; end + # + # struct Three; end + # + # {{One.module?}} # => true + # {{Two.module?}} # => false + # {{Three.module?}} # => false + # ``` + def module? : BoolLiteral + end + + # Returns `true` if `self` is a `class`, otherwise `false`. + # + # ``` + # module One; end + # + # class Two; end + # + # struct Three; end + # + # {{One.class?}} # => false + # {{Two.class?}} # => true + # {{Three.class?}} # => false + # ``` + def class? : BoolLiteral + end + + # Returns `true` if `self` is a `struct`, otherwise `false`. + # + # ``` + # module One; end + # + # class Two; end + # + # struct Three; end + # + # {{One.struct?}} # => false + # {{Two.struct?}} # => false + # {{Three.struct?}} # => true + # ``` + def struct? : BoolLiteral + end + # Returns the types forming a union type, if this is a union type. # Otherwise returns this single type inside an array literal (so you can safely call `union_types` on any type and treat all types uniformly). # - # See also: `union?`. + # See also: `#union?`. def union_types : ArrayLiteral(TypeNode) end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 2000a2284349..98d8a08163e9 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -1565,6 +1565,12 @@ module Crystal interpret_argless_method(method, args) { BoolLiteral.new(type.abstract?) } when "union?" interpret_argless_method(method, args) { BoolLiteral.new(type.is_a?(UnionType)) } + when "module?" + interpret_argless_method(method, args) { BoolLiteral.new(type.module?) } + when "class?" + interpret_argless_method(method, args) { BoolLiteral.new(type.class? && !type.struct?) } + when "struct?" + interpret_argless_method(method, args) { BoolLiteral.new(type.class? && type.struct?) } when "nilable?" interpret_argless_method(method, args) { BoolLiteral.new(type.nilable?) } when "union_types" From 60bdcf0002dd26be6092eec216edb1d838ec5b8e Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Sat, 16 May 2020 13:47:36 +0200 Subject: [PATCH 041/263] Resolve the always-present Windows linker's warnings (#9307) * Resolve the always-present Windows linker's warnings Currently when building on Windows, the version of MSVC linker is always printed first, then the list of all inputs, then a warning for every object file because it ends with .o, not .obj. So, use the .obj extension and disable the default outputs. So errors are still printed, but normally the output is empty. * Also switch the file extension for llvm_ext.obj --- .github/workflows/win.yml | 7 ++++--- .gitignore | 1 + src/compiler/crystal/codegen/link.cr | 4 ++++ src/compiler/crystal/compiler.cr | 17 ++++++++++------- src/llvm/lib_llvm_ext.cr | 6 +++++- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index cd7d17412754..e0e950ad7c51 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -16,12 +16,13 @@ jobs: - name: Cross-compile Crystal run: | LLVM_TARGETS=X86 bin/crystal build --cross-compile --target x86_64-pc-windows-msvc src/compiler/crystal.cr -Dwithout_playground + mv crystal.o crystal.obj || true # TODO: Remove this after 0.35.0 - name: Upload Crystal object file uses: actions/upload-artifact@v1 with: name: objs - path: crystal.o + path: crystal.obj windows-job: needs: linux-job @@ -146,10 +147,10 @@ jobs: name: objs - name: Build LLVM extensions run: | - cl /MT /c src\llvm\ext\llvm_ext.cc -I llvm\include /Fosrc\llvm\ext\llvm_ext.o + cl /MT /c src\llvm\ext\llvm_ext.cc -I llvm\include /Fosrc\llvm\ext\llvm_ext.obj - name: Link Crystal executable run: | - Invoke-Expression "cl objs\crystal.o /Fecrystal-cross src\llvm\ext\llvm_ext.o $(llvm\bin\llvm-config.exe --libs) libs\pcre.lib libs\gc.lib advapi32.lib libcmt.lib legacy_stdio_definitions.lib /F10000000" + Invoke-Expression "cl objs\crystal.obj /Fecrystal-cross src\llvm\ext\llvm_ext.obj $(llvm\bin\llvm-config.exe --libs) libs\pcre.lib libs\gc.lib advapi32.lib libcmt.lib legacy_stdio_definitions.lib /F10000000" - name: Re-build Crystal run: | diff --git a/.gitignore b/.gitignore index 2ce57d81ab1d..ed709c9b2cda 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ all_spec /tmp /docs/ /src/llvm/ext/llvm_ext.o +/src/llvm/ext/llvm_ext.obj /src/llvm/ext/llvm_ext.dwo /src/ext/*.o /src/ext/libcrystal.a diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index cf3f381f806d..70ed81bdab2a 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -86,6 +86,10 @@ module Crystal end class Program + def object_extension + has_flag?("windows") ? ".obj" : ".o" + end + def lib_flags has_flag?("windows") ? lib_flags_windows : lib_flags_posix end diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 7c0833329cfc..dbea26202368 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -274,7 +274,7 @@ module Crystal units = llvm_modules.map do |type_name, info| llvm_mod = info.mod llvm_mod.target = target_triple - CompilationUnit.new(self, type_name, llvm_mod, output_dir, bc_flags_changed) + CompilationUnit.new(self, program, type_name, llvm_mod, output_dir, bc_flags_changed) end if @cross_compile @@ -304,7 +304,7 @@ module Crystal private def cross_compile(program, units, output_filename) unit = units.first llvm_mod = unit.llvm_mod - object_name = "#{output_filename}.o" + object_name = output_filename + program.object_extension optimize llvm_mod if @release @@ -327,7 +327,7 @@ module Crystal # Execute and expand `subcommands`. lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } if expand - args = %(#{object_names.join(" ")} "/Fe#{output_filename}" #{lib_flags} #{@link_flags}) + args = %(/nologo #{object_names.join(" ")} "/Fe#{output_filename}" #{lib_flags} #{@link_flags}) cmd = "#{CL} #{args}" if cmd.to_utf16.size > 32000 @@ -597,9 +597,10 @@ module Crystal getter original_name getter llvm_mod getter? reused_previous_compilation = false + @object_extension : String - def initialize(@compiler : Compiler, @name : String, @llvm_mod : LLVM::Module, - @output_dir : String, @bc_flags_changed : Bool) + def initialize(@compiler : Compiler, program : Program, @name : String, + @llvm_mod : LLVM::Module, @output_dir : String, @bc_flags_changed : Bool) @name = "_main" if @name == "" @original_name = @name @name = String.build do |str| @@ -621,6 +622,8 @@ module Crystal # 17 chars from name + 1 (dash) + 32 (md5) = 50 @name = "#{@name[0..16]}-#{Digest::MD5.hexdigest(@name)}" end + + @object_extension = program.object_extension end def compile @@ -721,7 +724,7 @@ module Crystal llvm_mod.print_to_file "#{output_filename}.ll" end if emit_target.obj? - FileUtils.cp(object_name, "#{output_filename}.o") + FileUtils.cp(object_name, output_filename + @object_extension) end end @@ -730,7 +733,7 @@ module Crystal end def object_filename - "#{@name}.o" + @name + @object_extension end def temporary_object_name diff --git a/src/llvm/lib_llvm_ext.cr b/src/llvm/lib_llvm_ext.cr index 07227aa2907c..b7bae30dcd4d 100644 --- a/src/llvm/lib_llvm_ext.cr +++ b/src/llvm/lib_llvm_ext.cr @@ -1,5 +1,9 @@ require "./lib_llvm" -@[Link(ldflags: "#{__DIR__}/ext/llvm_ext.o")] +{% if flag?(:win32) %} + @[Link(ldflags: "#{__DIR__}/ext/llvm_ext.obj")] +{% else %} + @[Link(ldflags: "#{__DIR__}/ext/llvm_ext.o")] +{% end %} lib LibLLVMExt alias Char = LibC::Char alias Int = LibC::Int From b199a819916d7790eaa1ecc686a623e60c1df17e Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 19 May 2020 03:58:49 +0900 Subject: [PATCH 042/263] Parser: rewrite type parser (#9208) * Parser: rewrite type parser For some reasons, the parser for types and generic type path literals is known as one of the most complex part of the parser. (Another one is `do ... end` block parsing.) - **too much sharing**: the parser for a single type, type arguments, proc type arguments, tuple element types and others are unified to just one method. This code-sharing is excessive. They are not share syntax really, so there are many conditions and branches, thus it is hard to follow code flow. - **historical naming**: a type path (or part ot this) is called as `ident` for historical reason. However `ident` means different today. As the above reasons, gazillion bugs live in there unfortunately. For instance, they are valid type restriction for now: - `*Foo` (orphan splat) - `(Foo, Bar) | Baz` (comma + union type) - `{(Foo, Bar), Baz}` (comma + tuple type) This commit refines type grammer and rewrite parsers. * Update src/compiler/crystal/syntax/parser.cr Co-authored-by: Brian J. Cardiff * Use Generic and Union for nilable type properly https://github.com/crystal-lang/crystal/pull/9208#issuecomment-625817943 and, some bugs are fixed. * Remove unnecessary if Co-authored-by: Brian J. Cardiff --- spec/compiler/formatter/formatter_spec.cr | 3 + spec/compiler/parser/parser_spec.cr | 13 + spec/compiler/semantic/restrictions_spec.cr | 10 - src/compiler/crystal/syntax/parser.cr | 698 +++++++++----------- src/compiler/crystal/tools/formatter.cr | 15 +- 5 files changed, 355 insertions(+), 384 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 16f943dd3a14..c5dacd27d36c 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -848,6 +848,7 @@ describe Crystal::Formatter do assert_format "if 1\n ((1) + 2)\nend" assert_format "def foo(x : self ?) \n end", "def foo(x : self?)\nend" + assert_format "def foo(x : (self)?)\nend" assert_format " macro foo\n end\n\n :+", "macro foo\n end\n\n:+" assert_format "[\n1, # a\n2, # b\n 3 # c\n]", "[\n 1, # a\n 2, # b\n 3, # c\n]" @@ -1030,6 +1031,7 @@ describe Crystal::Formatter do assert_format "foo : self?" assert_format "foo : self? | A" + assert_format "foo : (self)?" assert_format "foo : (A) | D" assert_format "foo : (F(A)) | D" @@ -1041,6 +1043,7 @@ describe Crystal::Formatter do assert_format "module Readline\n @@completion_proc : (String -> Array(String)?) | (String -> Array(String)) | Nil\nend" assert_format "alias A = (B(C, (C | D)) | E)" assert_format "alias A = ((B(C | D) | E) | F)" + assert_format "alias A = ({A, (B)})" assert_format "foo : A(B)\nbar : C" assert_format "foo : (A -> B)\nbar : C" diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 73e0a648b763..01487730ac10 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -554,6 +554,7 @@ module Crystal it_parses "Foo(x: U)", Generic.new("Foo".path, [] of ASTNode, named_args: [NamedArgument.new("x", "U".path)]) it_parses "Foo(x: U, y: V)", Generic.new("Foo".path, [] of ASTNode, named_args: [NamedArgument.new("x", "U".path), NamedArgument.new("y", "V".path)]) + it_parses "Foo(X: U, Y: V)", Generic.new("Foo".path, [] of ASTNode, named_args: [NamedArgument.new("X", "U".path), NamedArgument.new("Y", "V".path)]) assert_syntax_error "Foo(T, x: U)" assert_syntax_error "Foo(x: T y: U)" @@ -562,6 +563,7 @@ module Crystal it_parses "Foo({x: X})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path)])] of ASTNode) it_parses "Foo({x: X, y: Y})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path), NamedArgument.new("y", "Y".path)])] of ASTNode) + it_parses "Foo({X: X, Y: Y})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("X", "X".path), NamedArgument.new("Y", "Y".path)])] of ASTNode) it_parses "Foo(T, {x: X})", Generic.new("Foo".path, ["T".path, Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path)])] of ASTNode) assert_syntax_error "Foo({x: X, x: Y})", "duplicated key: x" @@ -1883,6 +1885,17 @@ module Crystal it_parses %(annotation Foo\nend\nrequire "bar"), [AnnotationDef.new("Foo".path), Require.new("bar")] + assert_syntax_error "def foo(x : *Int32); end", "invalid type splat" + assert_syntax_error "def foo(x : (*Int32)); end", "invalid type splat" + assert_syntax_error "def foo(x : Int32, Int32); end" + assert_syntax_error "def foo(x : (Int32, Int32)); end" + assert_syntax_error "def foo(x : (Int32, Int32) | Int32); end" + assert_syntax_error "def foo(x : Int32 | (Int32, Int32)); end" + assert_syntax_error "def foo(x : {Int32, (Int32, Int32)}); end" + assert_syntax_error "def foo(x : 1); end" + assert_syntax_error "def foo(x : {sizeof(Int32), 2}); end" + assert_syntax_error "def foo(x : Array({sizeof(Int32), 2})); end" + it "gets corrects of ~" do node = Parser.parse("\n ~1") loc = node.location.not_nil! diff --git a/spec/compiler/semantic/restrictions_spec.cr b/spec/compiler/semantic/restrictions_spec.cr index 00784dea5c1b..2542fab3ab95 100644 --- a/spec/compiler/semantic/restrictions_spec.cr +++ b/spec/compiler/semantic/restrictions_spec.cr @@ -680,16 +680,6 @@ describe "Restrictions" do )) { types["Parent"].metaclass.virtual_type! } end - it "doesn't crash on invalid splat restriction (#3698)" do - assert_error %( - def foo(arg : *String) - end - - foo(1) - ), - "no overload matches" - end - it "errors if using free var without forall" do assert_error %( def foo(x : T) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 2c4a6a3bf80c..1795cbc07ba9 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -386,7 +386,7 @@ module Crystal ) push_var atomic next_token_skip_space - type = parse_single_type + type = parse_bare_proc_type atomic = UninitializedVar.new(atomic, type).at(location) return atomic else @@ -835,12 +835,12 @@ module Crystal if @token.type == :"(" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type skip_space_or_newline check :")" next_token_skip_space else - type = parse_single_type + type = parse_union_type end IsA.new(atomic, type) @@ -851,13 +851,13 @@ module Crystal if @token.type == :"(" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type skip_space_or_newline check :")" end_location = token_end_location next_token_skip_space else - type = parse_single_type(allow_commas: false) + type = parse_union_type end_location = token_end_location end @@ -942,7 +942,7 @@ module Crystal when :"{%" parse_percent_macro_control when :"::" - parse_ident_or_global_call + parse_generic_or_global_call when :"->" parse_fun_literal when :"@[" @@ -1156,7 +1156,7 @@ module Crystal set_visibility parse_var_or_call end when :CONST - parse_ident_or_literal + parse_generic_or_custom_literal when :INSTANCE_VAR if @in_macro_expression && @token.value == "@type" @is_macro_def = true @@ -1186,7 +1186,7 @@ module Crystal def parse_type_declaration(var) next_token_skip_space_or_newline - var_type = parse_single_type(allow_splat: true) + var_type = parse_bare_proc_type skip_space if @token.type == :"=" next_token_skip_space_or_newline @@ -1230,12 +1230,12 @@ module Crystal end end - def parse_ident_or_literal - ident = parse_ident - parse_custom_literal ident + def parse_generic_or_custom_literal + type = parse_generic(expression: true) + parse_custom_literal type end - def parse_custom_literal(ident) + def parse_custom_literal(type) skip_space if @token.type == :"{" @@ -1249,16 +1249,17 @@ module Crystal case tuple_or_hash when TupleLiteral - ary = ArrayLiteral.new(tuple_or_hash.elements, name: ident).at(tuple_or_hash.location) + ary = ArrayLiteral.new(tuple_or_hash.elements, name: type).at(tuple_or_hash.location) return ary when HashLiteral - tuple_or_hash.name = ident + tuple_or_hash.name = type return tuple_or_hash else raise "BUG: tuple_or_hash should be tuple or hash, not #{tuple_or_hash}" end end - ident + + type end def check_not_inside_def(message) @@ -1433,7 +1434,7 @@ module Crystal def parse_rescue_types types = [] of ASTNode while true - types << parse_ident + types << parse_generic skip_space if @token.type == :"|" next_token_skip_space @@ -1623,7 +1624,7 @@ module Crystal superclass = Self.new.at(@token.location) next_token else - superclass = parse_ident + superclass = parse_generic end end skip_statement_end @@ -1858,7 +1859,7 @@ module Crystal if @token.type == :":" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type end if @token.type == :"," @@ -1900,7 +1901,7 @@ module Crystal end end when :CONST - obj = parse_ident + obj = parse_generic check :"." name = consume_def_or_macro_name next_token_skip_space @@ -1914,7 +1915,7 @@ module Crystal if @token.type == :"(" next_token_skip_space - types = parse_types + types = parse_union_types(:")") check :")" next_token_skip_space else @@ -2278,7 +2279,7 @@ module Crystal next_token_skip_space if @token.keyword?(:of) next_token_skip_space_or_newline - of = parse_single_type + of = parse_bare_proc_type ArrayLiteral.new(of: of).at_end(of) else raise "for empty arrays use '[] of ElementType'", line, column @@ -2317,7 +2318,7 @@ module Crystal of = nil if @token.keyword?(:of) next_token_skip_space_or_newline - of = parse_single_type + of = parse_bare_proc_type end_location = of.end_location elsif exps.size == 0 raise "for empty arrays use '[] of ElementType'", line, column @@ -2472,10 +2473,10 @@ module Crystal if allow_of if @token.keyword?(:of) next_token_skip_space_or_newline - of_key = parse_single_type + of_key = parse_bare_proc_type check :"=>" next_token_skip_space_or_newline - of_value = parse_single_type + of_value = parse_bare_proc_type of = HashLiteral::Entry.new(of_key, of_value) end_location = of_value.end_location end @@ -2879,7 +2880,7 @@ module Crystal name.end_location = token_end_location next_token_skip_space else - name = parse_ident + name = parse_generic end klass.new name @@ -3371,7 +3372,7 @@ module Crystal end_location = token_end_location if @token.type == :CONST - receiver = parse_ident(allow_type_vars: false) + receiver = parse_path elsif @token.type == :IDENT check_valid_def_name name = @token.value.to_s @@ -3518,7 +3519,7 @@ module Crystal if @token.type == :":" next_token_skip_space - return_type = parse_single_type + return_type = parse_bare_proc_type end_location = return_type.end_location end @@ -3699,7 +3700,7 @@ module Crystal next_token end - restriction = parse_single_type(allow_splat: !splat_restriction) + restriction = parse_bare_proc_type if splat_restriction restriction = splat ? Splat.new(restriction) : DoubleSplat.new(restriction) @@ -3769,7 +3770,7 @@ module Crystal location = @token.location - type_spec = parse_single_type(allow_splat: true) + type_spec = parse_bare_proc_type end block_arg = Arg.new(arg_name, restriction: type_spec).at(name_location) @@ -4640,7 +4641,7 @@ module Crystal end end - def parse_ident_or_global_call + def parse_generic_or_global_call location = @token.location next_token_skip_space_or_newline @@ -4648,46 +4649,187 @@ module Crystal when :IDENT set_visibility parse_var_or_call global: true when :CONST - ident = parse_ident_after_colons location, - global: true, - allow_type_vars: true, - parse_nilable: true + ident = parse_generic global: true, location: location, expression: true parse_custom_literal ident else unexpected_token end end - def parse_ident(allow_type_vars = true, parse_nilable = true) + # Parse a **bare** proc type like `A, B, C -> D`. + # Generally it is entry point of type parsing and + # it is used on the context expected type (e.g. type restrictions, rhs of `alias` and more) + def parse_bare_proc_type + type = parse_type_splat { parse_union_type } + + # To determine to consume comma, looking-ahead is needed. + # Consider `[ [] of Int32, Foo.new ]`, we want to parse it as `[ ([] of Int32), Foo.new ]` of course. + # If the parser consumes comma afrer Int32 quickly, it may cause parsing error. + unless @token.type == :"->" || (@token.type == :"," && type_start?) + if type.is_a?(Splat) + raise "invalid type splat", type.location.not_nil! + end + return type + end + + input_types = [type] + if @token.type != :"->" + loop do + next_token_skip_space_or_newline + input_types << parse_type_splat { parse_union_type } + break unless @token.type == :"," && type_start? + end + end + + parse_proc_type_output(input_types, input_types.first.location) + end + + def parse_union_type + type = parse_atomic_type_with_suffix + return type unless @token.type == :| + + types = [type] + while @token.type == :| + next_token_skip_space_or_newline + types << parse_atomic_type_with_suffix + end + + Union.new(types).at(types.first).at_end(types.last) + end + + def parse_atomic_type_with_suffix + type = parse_atomic_type + parse_type_suffix type + end + + def parse_atomic_type location = @token.location - global = false + case @token.type + when :IDENT + case @token.value + when :self + next_token_skip_space + Self.new.at(location) + when "self?" + next_token_skip_space + make_nilable_type Self.new.at(location) + when :typeof + parse_typeof + else + unexpected_token + end + when :UNDERSCORE + next_token_skip_space + Underscore.new.at(location) + when :CONST, :"::" + parse_generic + when :"{" + next_token_skip_space_or_newline + if named_tuple_start? || @token.type == :DELIMITER_START + type = make_named_tuple_type parse_named_type_args(:"}") + else + type = make_tuple_type parse_union_types(:"}") + end + check :"}" + next_token_skip_space + type + when :"->" + parse_proc_type_output(nil, location) + when :"(" + next_token_skip_space_or_newline + type = parse_type_splat { parse_union_type } + if @token.type == :")" + next_token_skip_space + if @token.type == :"->" # `(A) -> B` case + type = parse_proc_type_output([type], location) + elsif type.is_a?(Splat) + raise "invalid type splat", type.location.not_nil! + end + else + input_types = [type] + while @token.type == :"," + next_token_skip_space_or_newline + break if @token.type == :")" # allow trailing comma + input_types << parse_type_splat { parse_union_type } + end + if @token.type == :"->" # `(A, B, C -> D)` case + type = parse_proc_type_output(input_types, input_types.first.location) + check :")" + next_token_skip_space + else # `(A, B, C) -> D` case + check :")" + next_token_skip_space + type = parse_proc_type_output(input_types, location) + end + end + type + else + unexpected_token + end + end - if @token.type == :UNDERSCORE - return node_and_next_token Underscore.new.at(location) + def parse_union_types(end_token) + types = [parse_union_type] + while @token.type == :"," + next_token_skip_space_or_newline + break if @token.type == end_token # allow trailing comma + types << parse_union_type end + types + end + + # Parse generic type path like `A::B(C, D)?`. + # This method is used to parse not only a type, but also an expression represents type. + # And it also consumes prefix `::` to specify global path. + def parse_generic(expression = false) + location = @token.location + global = false if @token.type == :"::" - global = true next_token_skip_space_or_newline + global = true end - check :CONST - parse_ident_after_colons(location, global, allow_type_vars, parse_nilable) + parse_generic global, location, expression end - def parse_path - name = parse_ident(allow_type_vars: false, parse_nilable: false) - raise "BUG: expected a Path" unless name.is_a?(Path) - name + def parse_generic(global, location, expression) + path = parse_path(global, location) + type = parse_type_args(path) + + # Nilable suffixes without any spaces are consumed here + # for expression represents nilable type. Typically such an expression + # is appeared in macro expression. (e.g. `{% if T <= Int32? %} ... {% end %}`) + # Note that the parser cannot consume any spaces because it conflicts ternary operator. + while expression && @token.type == :"?" + next_token + type = make_nilable_expression(type) + end + + skip_space + + type end - def parse_ident_after_colons(location, global, allow_type_vars, parse_nilable) - start_line = location.line_number - start_column = location.column_number + # Parse type path. + # It also consumes prefix `::` to specify global path. + def parse_path + location = @token.location + + global = false + if @token.type == :"::" + next_token_skip_space_or_newline + global = true + end + + path = parse_path(global, @token.location) + skip_space + path + end - names = [] of String - names << @token.value.to_s + def parse_path(global, location) + names = [check_const] end_location = token_end_location @wants_regex = false @@ -4696,58 +4838,44 @@ module Crystal next_token_skip_space_or_newline names << check_const end_location = token_end_location + @wants_regex = false next_token end - const = Path.new(names, global).at(location) - const.end_location = end_location - - token_location = @token.location - if token_location && token_location.line_number == start_line - const.name_size = token_location.column_number - start_column - end + Path.new(names, global).at(location).at_end(end_location) + end - if allow_type_vars && @token.type == :"(" - next_token_skip_space_or_newline + def parse_type_args(name) + return name unless @token.type == :"(" - if named_tuple_start? || @token.type == :DELIMITER_START - types = [] of ASTNode - named_args = parse_type_named_args(:")") - else - types = parse_types allow_primitives: true, allow_splat: true - if types.empty? - raise "must specify at least one type var" - end - named_args = nil + next_token_skip_space_or_newline + args = [] of ASTNode + if named_tuple_start? || string_literal_start? + named_args = parse_named_type_args(:")") + else + args << parse_type_splat { parse_type_arg } + while @token.type == :"," + next_token_skip_space_or_newline + break if @token.type == :")" # allow trailing comma + args << parse_type_splat { parse_type_arg } end - next_token if @token.type == :"," - - skip_space_or_newline - check :")" - const = Generic.new(const, types, named_args).at(location) - const.end_location = token_end_location - - next_token - end - - if parse_nilable - while @token.type == :"?" - const = Generic.new(Path.global("Union").at(const), [ - const, Path.global("Nil").at(const), - ] of ASTNode) - const.question = true - next_token + has_int = args.any? { |arg| arg.is_a?(NumberLiteral) || arg.is_a?(SizeOf) || arg.is_a?(InstanceSizeOf) || arg.is_a?(OffsetOf) } + if @token.type == :"->" && !has_int + args = [parse_proc_type_output(args, args.first.location)] of ASTNode end end - skip_space + skip_space_or_newline + check :")" + end_location = token_end_location + next_token - const + Generic.new(name, args, named_args).at(name).at_end(end_location) end - def parse_type_named_args(end_token) + def parse_named_type_args(end_token) named_args = [] of NamedArgument while @token.type != end_token @@ -4767,7 +4895,7 @@ module Crystal check :":" next_token_skip_space_or_newline - type = parse_single_type(allow_commas: false) + type = parse_bare_proc_type skip_space named_args << NamedArgument.new(name, type) @@ -4784,308 +4912,148 @@ module Crystal named_args end - def parse_types(allow_primitives = false, allow_splat = false) - type = parse_type(allow_primitives: allow_primitives, allow_splat: allow_splat, inside_paren: true) - case type - when Array - type - when ASTNode - [type] of ASTNode - else - raise "BUG" - end - end - - def parse_single_type(allow_primitives = false, allow_commas = true, allow_splat = false) - location = @token.location - type = parse_type(allow_primitives: allow_primitives, allow_commas: allow_commas, allow_splat: allow_splat) - case type - when Array - raise "unexpected ',' in type (use parentheses to disambiguate)", location - when ASTNode - type - else - raise "BUG" - end - end - - def parse_type(allow_primitives, allow_commas = true, allow_splat = false, inside_paren = false) + def parse_type_splat location = @token.location - if @token.type == :"->" - input_types = nil - else - input_types = parse_type_union(allow_primitives, allow_splat) - input_types = [input_types] unless input_types.is_a?(Array) - while allow_commas && @token.type == :"," && ( - (allow_primitives && next_comes_type_or_int) || - (!allow_primitives && next_comes_type) || - (inside_paren && next_comes_curly) - ) - next_token_skip_space_or_newline - if @token.type == :"->" - next_types = parse_type(false) - case next_types - when Array - input_types.concat next_types - when ASTNode - input_types << next_types - end - next - else - type_union = parse_type_union(allow_primitives, allow_splat) - if type_union.is_a?(Array) - input_types.concat type_union - else - input_types << type_union - end - end - end - end - - if @token.type == :"->" - next_token_skip_space - case @token.type - when :"=", :",", :")", :"}", :";", :NEWLINE - return_type = nil - else - type_union = parse_type_union(allow_primitives, allow_splat) - if type_union.is_a?(Array) - raise "can't return more than more type", location.line_number, location.column_number - else - return_type = type_union - end - end - ProcNotation.new(input_types, return_type).at(location) - else - input_types = input_types.not_nil! - if input_types.size == 1 - input_types.first - else - input_types - end - end - end - - def parse_type_union(allow_primitives, allow_splat) - types = [] of ASTNode - parse_type_with_suffix(types, allow_primitives, allow_splat) - if @token.type == :"|" - while @token.type == :"|" - next_token_skip_space_or_newline - parse_type_with_suffix(types, allow_primitives, allow_splat) - end - - if types.size == 1 - types.first - else - Union.new(types).at(types.first.location) - end - elsif types.size == 1 - types.first - else - types - end - end - - def parse_type_with_suffix(types, allow_primitives, allow_splat) splat = false - if allow_splat && @token.type == :"*" + if @token.type == :"*" + next_token_skip_space_or_newline splat = true - next_token end - location = @token.location + type = yield + type = Splat.new(type).at(location) if splat + type + end - if @token.type == :IDENT && @token.value == "self?" - type = Self.new.at(location) - type = Union.new([type, Path.global("Nil")] of ASTNode).at(location) + def parse_type_arg + if @token.type == :NUMBER + num = NumberLiteral.new(@token.value.to_s, @token.number_kind).at(@token.location) next_token_skip_space - elsif @token.keyword?(:self) - type = Self.new.at(location) - next_token_skip_space - else - case @token.type - when :"{" - next_token_skip_space_or_newline - - if named_tuple_start? || @token.type == :DELIMITER_START - named_args = parse_type_named_args(:"}") - else - type = parse_type(allow_primitives) - end - - # Allow a trailing comma - if @token.type == :"," - next_token_skip_space_or_newline - end - - check :"}" - next_token_skip_space - - if named_args - type = make_named_tuple_type(named_args).at(location) - else - case type - when Array - type = make_tuple_type(type).at(location) - when ASTNode - type = make_tuple_type([type] of ASTNode).at(location) - else - raise "BUG" - end - end - when :"(" - next_token_skip_space_or_newline - type = parse_type(allow_primitives, allow_splat: allow_splat) - check :")" - next_token_skip_space - case type - when Array - types.concat type - return - when ASTNode - # skip - else - raise "BUG" - end - else - if allow_primitives - if @token.type == :NUMBER - num = NumberLiteral.new(@token.value.to_s, @token.number_kind).at(@token.location) - types << node_and_next_token(num) - skip_space - return types - end - end - - type = parse_simple_type - end + return num end - type = Splat.new(type).at(location) if splat - types << parse_type_suffix(type) - end - - def parse_simple_type case @token - when .keyword?(:typeof) - type = parse_typeof when .keyword?(:sizeof) - type = parse_sizeof + parse_sizeof when .keyword?(:instance_sizeof) - type = parse_instance_sizeof + parse_instance_sizeof when .keyword?(:offsetof) - type = parse_offsetof + parse_offsetof else - type = parse_ident(parse_nilable: false) + parse_union_type end - skip_space - type end def parse_type_suffix(type) - while true + loop do case @token.type + when :"." + next_token_skip_space_or_newline + check_ident :class + next_token_skip_space + type = Metaclass.new(type).at(type) when :"?" - type = Union.new([type, Path.global("Nil")] of ASTNode).at(type.location) next_token_skip_space + type = make_nilable_type(type) when :"*" - type = make_pointer_type(type).at(type.location) next_token_skip_space + type = make_pointer_type(type) when :"**" - type = make_pointer_type(make_pointer_type(type)).at(type.location) next_token_skip_space + type = make_pointer_type(make_pointer_type(type)) when :"[" - next_token_skip_space - size = parse_single_type allow_primitives: true + next_token_skip_space_or_newline + size = parse_type_arg + skip_space_or_newline check :"]" - @wants_regex = false - next_token_skip_space - type = make_static_array_type(type, size).at(type.location) - when :"." - next_token - check_ident :class - type = Metaclass.new(type).at(type.location) next_token_skip_space + type = make_static_array_type(type, size) else - break + return type end end - type end - def parse_typeof - location = @token.location + def parse_proc_type_output(input_types, location) + has_output_type = type_start?(consume_newlines: false) + check :"->" next_token_skip_space - check :"(" - next_token_skip_space_or_newline - if @token.type == :")" - raise "missing typeof argument" - end - exps = [] of ASTNode - while @token.type != :")" - exps << parse_op_assign - if @token.type == :"," - next_token_skip_space_or_newline - else - skip_space_or_newline - check :")" - end + if has_output_type + skip_space_or_newline + output_type = parse_union_type end - end_location = token_end_location - next_token_skip_space + ProcNotation.new(input_types, output_type).at(location) + end - TypeOf.new(exps).at(location).at_end(end_location) + def make_nilable_type(type) + Union.new([type, Path.global("Nil").at(type)]).at(type) end - def next_comes_type - next_comes_type_or_int allow_int: false + def make_nilable_expression(type) + type = Generic.new(Path.global("Union").at(type), [type, Path.global("Nil").at(type)]).at(type) + type.question = true + type end - def next_comes_type_or_int(allow_int = true) - old_pos, old_line, old_column = current_pos, @line_number, @column_number + def make_pointer_type(type) + Generic.new(Path.global("Pointer").at(type), [type] of ASTNode).at(type) + end + + def make_static_array_type(type, size) + Generic.new(Path.global("StaticArray").at(type), [type, size] of ASTNode).at(type) + end + + def make_tuple_type(types) + Generic.new(Path.global("Tuple"), types) + end + + def make_named_tuple_type(named_args) + Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: named_args) + end + # Looks ahead next tokens to check whether they indicate type. + def type_start?(consume_newlines = true) + old_pos, old_line, old_column = current_pos, @line_number, @column_number @temp_token.copy_from(@token) - next_token_skip_space_or_newline + if consume_newlines + next_token_skip_space_or_newline + else + next_token_skip_space + end - while @token.type == :"{" || @token.type == :"(" + while @token.type == :"(" || @token.type == :"{" next_token_skip_space_or_newline end + # TODO: the below conditions are not complete, and there are many false-positive or true-negative examples. + # For example, `[ [] of Int32, Foo::Bar.new ]` should be parsed to `[ ([] of Int32), Foo::Bar.new ]`, + # however, the current implementation mistakes `Foo::Bar` as type name, so parsing is failed. + begin case @token.type - when :CONST - next_token_skip_space - if @token.type == :"." - next_token_skip_space - @token.keyword?(:class) - else - true - end - when :UNDERSCORE - true - when :"->" - true - when :"*" - next_token - @token.type == :CONST - when :NUMBER - allow_int && @token.number_kind == :i32 when :IDENT case @token.value - when :typeof, :self, :sizeof, :instance_sizeof, :offsetof + when :typeof, :self, "self?" true else false end - when :"::" + when :CONST + return false if named_tuple_start? next_token_skip_space + return true unless @token.type == :"." + next_token_skip_space_or_newline + @token.keyword?(:class) + when :"::" + next_token @token.type == :CONST + when :UNDERSCORE, :"->" + true else false end @@ -5095,35 +5063,31 @@ module Crystal end end - def next_comes_curly - old_pos, old_line, old_column = current_pos, @line_number, @column_number - - @temp_token.copy_from(@token) + def parse_typeof + location = @token.location + next_token_skip_space + check :"(" next_token_skip_space_or_newline + if @token.type == :")" + raise "missing typeof argument" + end - curly = @token.type == :"{" - - @token.copy_from(@temp_token) - self.current_pos, @line_number, @column_number = old_pos, old_line, old_column - - curly - end - - def make_pointer_type(node) - Generic.new(Path.global("Pointer").at(node), [node] of ASTNode).at(node) - end - - def make_static_array_type(type, size) - Generic.new(Path.global("StaticArray").at(type), [type, size] of ASTNode).at(type.location).at(type) - end + exps = [] of ASTNode + while @token.type != :")" + exps << parse_op_assign + if @token.type == :"," + next_token_skip_space_or_newline + else + skip_space_or_newline + check :")" + end + end - def make_tuple_type(types) - Generic.new(Path.global("Tuple"), types) - end + end_location = token_end_location + next_token_skip_space - def make_named_tuple_type(named_args) - Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: named_args) + TypeOf.new(exps).at(location).at_end(end_location) end def parse_visibility_modifier(modifier) @@ -5380,7 +5344,7 @@ module Crystal unexpected_token end when :CONST - ident = parse_ident + ident = parse_path(global: false, location: @token.location) skip_space check :"=" next_token_skip_space_or_newline @@ -5399,7 +5363,7 @@ module Crystal end check :":" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type if name[0].ascii_uppercase? raise "external variables must start with lowercase, use for example `$#{name.underscore} = #{name} : #{type}`", location @@ -5471,17 +5435,15 @@ module Crystal next_token_skip_space_or_newline check :":" next_token_skip_space_or_newline - arg_type = parse_single_type + arg_type = parse_bare_proc_type skip_space_or_newline args << Arg.new(arg_name, nil, arg_type).at(arg_location) push_var_name arg_name if require_body else - arg_types = parse_types - arg_types.each do |arg_type_2| - args << Arg.new("", nil, arg_type_2).at(arg_type_2.location) - end + arg_type = parse_union_type + args << Arg.new("", nil, arg_type).at(arg_type.location) end if @token.type == :"," @@ -5497,7 +5459,7 @@ module Crystal if @token.type == :":" next_token_skip_space_or_newline - return_type = parse_single_type + return_type = parse_bare_proc_type end skip_statement_end @@ -5538,7 +5500,7 @@ module Crystal check :"=" next_token_skip_space_or_newline - value = parse_single_type + value = parse_bare_proc_type skip_space alias_node = Alias.new(name, value) @@ -5581,7 +5543,7 @@ module Crystal next_token_skip_space_or_newline location = @token.location - exp = parse_single_type.at(location) + exp = parse_bare_proc_type.at(location) skip_space_or_newline @@ -5598,7 +5560,7 @@ module Crystal next_token_skip_space_or_newline type_location = @token.location - type = parse_single_type.at(type_location) + type = parse_bare_proc_type.at(type_location) skip_space check :"," @@ -5626,7 +5588,7 @@ module Crystal check :"=" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type skip_space typedef = TypeDef.new name, type @@ -5701,7 +5663,7 @@ module Crystal check :":" next_token_skip_space_or_newline - type = parse_single_type + type = parse_bare_proc_type skip_statement_end @@ -5722,7 +5684,7 @@ module Crystal case @token.type when :":" next_token_skip_space_or_newline - base_type = parse_single_type + base_type = parse_bare_proc_type skip_statement_end when :";", :NEWLINE skip_statement_end diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 6422c2ca7bae..0e1abd5bf71e 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -1112,6 +1112,7 @@ module Crystal if node.question? node.type_vars[0].accept self + skip_space write_token :"?" return false end @@ -1232,22 +1233,22 @@ module Crystal # Restore the old parentheses count @paren_count = old_paren_count - check_close_paren - false + ensure + check_close_paren end def visit(node : Union) + check_open_paren + if @token.type == :IDENT && @token.value == "self?" && node.types.size == 2 && - node.types.any?(&.is_a?(Self)) && - node.types.any? { |t| t.to_s == "::Nil" } + node.types[0].is_a?(Self) && node.types[1].to_s == "::Nil" write "self?" next_token + check_close_paren return false end - check_open_paren - paren_count = @paren_count column = @column @@ -2339,7 +2340,9 @@ module Crystal end def visit(node : Self) + check_open_paren write_keyword :self + check_close_paren false end From 7f132502126adcc9f3a432bef336ca62a788fda7 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 18 May 2020 15:59:36 -0300 Subject: [PATCH 043/263] Make IO#skip IO#write returns the number of bytes it skipped/written (#9233) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * IO#skip returns the number of bytes it skipped * make IO#skip return UInt64 * Allow Int arguments in IO#skip * Add specs assertions for skip / skip_to_end * Refactor: delegate IO::Stapled#skip_to_end directly to reader * Make IO#write return bytes written * Add specs, make printf return written bytes * Avoid using the BytesCounter wrapper Return bytes only in write_* operations. printf, puts, etc return Nil * Use &+ * Remove CHECK comments Co-authored-by: Carl Hörberg --- spec/std/http/chunked_content_spec.cr | 2 +- spec/std/http/request_spec.cr | 3 +- spec/std/http/server/response_spec.cr | 3 +- spec/std/io/buffered_spec.cr | 4 +- spec/std/io/io_spec.cr | 41 +++++++++++++++++--- spec/std/io/memory_spec.cr | 6 +-- spec/std/io/sized_spec.cr | 5 ++- spec/std/io/stapled_spec.cr | 16 ++++++++ spec/support/io.cr | 3 +- src/compress/deflate/writer.cr | 6 ++- src/compress/gzip/writer.cr | 6 ++- src/compress/zip/checksum_writer.cr | 4 +- src/compress/zlib/writer.cr | 6 ++- src/float.cr | 2 +- src/http/content.cr | 10 +++-- src/http/server/response.cr | 4 +- src/http/web_socket/protocol.cr | 6 ++- src/int.cr | 2 +- src/io.cr | 37 ++++++++++-------- src/io/buffered.cr | 30 +++++++++----- src/io/byte_format.cr | 56 ++++++++++++++------------- src/io/delimited.cr | 2 +- src/io/encoding.cr | 6 ++- src/io/hexdump.cr | 4 +- src/io/memory.cr | 14 ++++--- src/io/multi_writer.cr | 6 ++- src/io/sized.cr | 5 ++- src/io/stapled.cr | 13 +++++-- src/openssl/digest/digest_io.cr | 4 +- src/openssl/ssl/socket.cr | 1 - src/string/builder.cr | 6 +-- 31 files changed, 203 insertions(+), 110 deletions(-) diff --git a/spec/std/http/chunked_content_spec.cr b/spec/std/http/chunked_content_spec.cr index 2a4e43a3827c..c97cef02ca1f 100644 --- a/spec/std/http/chunked_content_spec.cr +++ b/spec/std/http/chunked_content_spec.cr @@ -34,7 +34,7 @@ describe HTTP::ChunkedContent do mem = IO::Memory.new("4\r\n123\n\r\n0\r\n\r\n") content = HTTP::ChunkedContent.new(mem) - content.skip(2) + content.skip(2).should eq(2) content.read_char.should eq('3') expect_raises(IO::EOFError) do diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index e06a4c193dc4..5c53426a0e9a 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -6,7 +6,8 @@ private class EmptyIO < IO 0 end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 + slice.size.to_u64 end end diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index 33bd0aa70ada..ff35b45527aa 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -12,10 +12,11 @@ private class ReverseResponseOutput < IO def initialize(@output : IO) end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 slice.reverse_each do |byte| @output.write_byte(byte) end + slice.size.to_u64 end def read(slice : Bytes) diff --git a/spec/std/io/buffered_spec.cr b/spec/std/io/buffered_spec.cr index a2d615fa6ccd..7c3514cbaaa4 100644 --- a/spec/std/io/buffered_spec.cr +++ b/spec/std/io/buffered_spec.cr @@ -429,14 +429,14 @@ describe "IO::Buffered" do it "skips" do str = IO::Memory.new("123456789") io = BufferedWrapper.new(str) - io.skip(3) + io.skip(3).should eq(3) io.read_char.should eq('4') end it "skips big" do str = IO::Memory.new(("a" * 10_000) + "b") io = BufferedWrapper.new(str) - io.skip(10_000) + io.skip(10_000).should eq(10_000) io.read_char.should eq('b') end diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index b49e6a0fd992..60413fcf995a 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -46,7 +46,7 @@ private class SimpleIOMemory < IO count end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 count = slice.size new_bytesize = bytesize + count if new_bytesize > @capacity @@ -56,7 +56,7 @@ private class SimpleIOMemory < IO slice.copy_to(@buffer + @bytesize, count) @bytesize += count - nil + slice.size.to_u64 end def to_slice @@ -99,7 +99,8 @@ private class OneByOneIO < IO 1 end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 + slice.size.to_u64 end end @@ -507,7 +508,7 @@ describe IO do it "skips a few bytes" do io = SimpleIOMemory.new io << "hello world" - io.skip(6) + io.skip(6).should eq(6) io.gets_to_end.should eq("world") end @@ -522,14 +523,14 @@ describe IO do it "skips more than 4096 bytes" do io = SimpleIOMemory.new io << "a" * 4100 - io.skip(4099) + io.skip(4099).should eq(4099) io.gets_to_end.should eq("a") end it "skips to end" do io = SimpleIOMemory.new io << "hello" - io.skip_to_end + io.skip_to_end.should eq(5) io.read_byte.should be_nil end @@ -543,6 +544,34 @@ describe IO do end end end + + describe "counts written bytes" do + it "directly" do + with_tempfile("create.txt") do |path| + File.open(path, "w") do |io| + io.write("hello world".to_slice).should eq(11) + io.write_utf8("mañana".to_slice).should eq(7) + end + end + end + + pending_win32 "with encoding" do + with_tempfile("create.txt") do |path| + File.open(path, "w", File::DEFAULT_CREATE_PERMISSIONS, "CP1252") do |io| + # In UTF-8 ñ will use 2 bytes + io.write_utf8("mañana".to_slice).should eq(6) + end + end + end + + it "with byte format" do + io = SimpleIOMemory.new + + io.write_bytes(1u64).should eq(8) + io.write_bytes(1u32).should eq(4) + io.write_bytes(1u8).should eq(1) + end + end end pending_win32 describe: "encoding" do diff --git a/spec/std/io/memory_spec.cr b/spec/std/io/memory_spec.cr index 2e19e5caec17..1aca014e7eef 100644 --- a/spec/std/io/memory_spec.cr +++ b/spec/std/io/memory_spec.cr @@ -353,11 +353,11 @@ describe IO::Memory do it "skips" do io = IO::Memory.new("hello") - io.skip(2) + io.skip(2).should eq(2) io.gets_to_end.should eq("llo") io.rewind - io.skip(5) + io.skip(5).should eq(5) io.gets_to_end.should eq("") io.rewind @@ -369,7 +369,7 @@ describe IO::Memory do it "skips_to_end" do io = IO::Memory.new("hello") - io.skip_to_end + io.skip_to_end.should eq(5) io.gets_to_end.should eq("") end diff --git a/spec/std/io/sized_spec.cr b/spec/std/io/sized_spec.cr index 8338063ff9db..46cb7d625e4f 100644 --- a/spec/std/io/sized_spec.cr +++ b/spec/std/io/sized_spec.cr @@ -5,7 +5,8 @@ private class NoPeekIO < IO 0 end - def write(bytes : Bytes) : Nil + def write(bytes : Bytes) : UInt64 + 0u64 end def peek @@ -138,7 +139,7 @@ describe "IO::Sized" do it "skips" do io = IO::Memory.new "123456789" sized = IO::Sized.new(io, read_size: 6) - sized.skip(3) + sized.skip(3).should eq(3) sized.read_char.should eq('4') expect_raises(IO::EOFError) do diff --git a/spec/std/io/stapled_spec.cr b/spec/std/io/stapled_spec.cr index ef27598d9d57..261dfcea11bb 100644 --- a/spec/std/io/stapled_spec.cr +++ b/spec/std/io/stapled_spec.cr @@ -76,6 +76,22 @@ describe IO::Stapled do io.peek.should eq Bytes.empty end + it "#skip delegates to reader" do + reader = IO::Memory.new "cletus" + io = IO::Stapled.new reader, IO::Memory.new + io.peek.should eq "cletus".to_slice + io.skip(4).should eq(4) + io.peek.should eq "us".to_slice + end + + it "#skip_to_end delegates to reader" do + reader = IO::Memory.new "cletus" + io = IO::Stapled.new reader, IO::Memory.new + io.peek.should eq "cletus".to_slice + io.skip_to_end.should eq(6) + io.peek.should eq Bytes.empty + end + describe ".pipe" do it "creates a bidirectional pipe" do a, b = IO::Stapled.pipe diff --git a/spec/support/io.cr b/spec/support/io.cr index 07d3afa8624a..2eb785856c32 100644 --- a/spec/support/io.cr +++ b/spec/support/io.cr @@ -8,9 +8,10 @@ class RaiseIOError < IO raise IO::Error.new("...") end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 @writes += 1 raise IO::Error.new("...") if @raise_on_write + slice.size.to_u64 end def flush diff --git a/src/compress/deflate/writer.cr b/src/compress/deflate/writer.cr index 68d7efff5e88..b32a3af8b300 100644 --- a/src/compress/deflate/writer.cr +++ b/src/compress/deflate/writer.cr @@ -43,14 +43,16 @@ class Compress::Deflate::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_open - return if slice.empty? + return 0u64 if slice.empty? @stream.avail_in = slice.size @stream.next_in = slice consume_output LibZ::Flush::NO_FLUSH + + slice.size.to_u64 end # See `IO#flush`. diff --git a/src/compress/gzip/writer.cr b/src/compress/gzip/writer.cr index f6ebb8a58d5a..8998ee291e81 100644 --- a/src/compress/gzip/writer.cr +++ b/src/compress/gzip/writer.cr @@ -68,10 +68,10 @@ class Compress::Gzip::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_open - return if slice.empty? + return 0u64 if slice.empty? flate_io = write_header flate_io.write(slice) @@ -82,6 +82,8 @@ class Compress::Gzip::Writer < IO # Using wrapping addition here because isize is only 32 bits wide but # uncompressed data size can be bigger. @isize &+= slice.size + + slice.size.to_u64 end # Flushes data, forcing writing the gzip header if no diff --git a/src/compress/zip/checksum_writer.cr b/src/compress/zip/checksum_writer.cr index 77853958a53d..179bfabdcb19 100644 --- a/src/compress/zip/checksum_writer.cr +++ b/src/compress/zip/checksum_writer.cr @@ -13,8 +13,8 @@ module Compress::Zip raise IO::Error.new "Can't read from Zip::Writer entry" end - def write(slice : Bytes) : Nil - return if slice.empty? + def write(slice : Bytes) : UInt64 + return 0u64 if slice.empty? @count += slice.size @crc32 = Digest::CRC32.update(slice, @crc32) if @compute_crc32 diff --git a/src/compress/zlib/writer.cr b/src/compress/zlib/writer.cr index 455ac8df6395..b4e4479314a9 100644 --- a/src/compress/zlib/writer.cr +++ b/src/compress/zlib/writer.cr @@ -44,15 +44,17 @@ class Compress::Zlib::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_open - return if slice.empty? + return 0u64 if slice.empty? write_header unless @wrote_header @flate_io.write(slice) @adler32 = Digest::Adler32.update(slice, @adler32) + + slice.size.to_u64 end # Flushes data, forcing writing the zlib header if no diff --git a/src/float.cr b/src/float.cr index bc4fdf0c2540..23eda5aeb44a 100644 --- a/src/float.cr +++ b/src/float.cr @@ -93,7 +93,7 @@ struct Float # Writes this float to the given *io* in the given *format*. # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) + def to_io(io : IO, format : IO::ByteFormat) : UInt64 format.encode(self, io) end diff --git a/src/http/content.cr b/src/http/content.cr index e7dd653845c2..09aa6b2e0017 100644 --- a/src/http/content.cr +++ b/src/http/content.cr @@ -41,7 +41,7 @@ module HTTP super end - def skip(bytes_count) + def skip(bytes_count : Int) : UInt64 ensure_send_continue super end @@ -73,7 +73,7 @@ module HTTP @io.peek end - def skip(bytes_count) + def skip(bytes_count : Int) : UInt64 ensure_send_continue @io.skip(bytes_count) end @@ -164,14 +164,18 @@ module HTTP peek end - def skip(bytes_count) + def skip(bytes_count : Int) : UInt64 + bytes_count = bytes_count.to_u64 ensure_send_continue + if bytes_count <= @chunk_remaining @io.skip(bytes_count) @chunk_remaining -= bytes_count else super end + + bytes_count end # Checks if the last read consumed a chunk and we diff --git a/src/http/server/response.cr b/src/http/server/response.cr index c0e0a18d3b42..eab4a03a7907 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -80,8 +80,8 @@ class HTTP::Server end # See `IO#write(slice)`. - def write(slice : Bytes) : Nil - return if slice.empty? + def write(slice : Bytes) : UInt64 + return 0u64 if slice.empty? @output.write(slice) end diff --git a/src/http/web_socket/protocol.cr b/src/http/web_socket/protocol.cr index 302eb2541237..077e99da545b 100644 --- a/src/http/web_socket/protocol.cr +++ b/src/http/web_socket/protocol.cr @@ -52,8 +52,8 @@ class HTTP::WebSocket::Protocol @pos = 0 end - def write(slice : Bytes) : Nil - return if slice.empty? + def write(slice : Bytes) : UInt64 + return 0u64 if slice.empty? count = Math.min(@buffer.size - @pos, slice.size) (@buffer + @pos).copy_from(slice.to_unsafe, count) @@ -66,6 +66,8 @@ class HTTP::WebSocket::Protocol if count < slice.size write(slice + count) end + + slice.size.to_u64 end def read(slice : Bytes) diff --git a/src/int.cr b/src/int.cr index 50a1c35b498d..ec5efb4f78bc 100644 --- a/src/int.cr +++ b/src/int.cr @@ -638,7 +638,7 @@ struct Int # Writes this integer to the given *io* in the given *format*. # # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) + def to_io(io : IO, format : IO::ByteFormat) : UInt64 format.encode(self, io) end diff --git a/src/io.cr b/src/io.cr index a334bfc187da..df29eab4e666 100644 --- a/src/io.cr +++ b/src/io.cr @@ -14,7 +14,7 @@ require "c/errno" # these two methods: # # * `read(slice : Bytes)`: read at most *slice.size* bytes from IO into *slice* and return the number of bytes read -# * `write(slice : Bytes)`: write the whole *slice* into the IO +# * `write(slice : Bytes)`: write the whole *slice* into the IO and return the number of bytes written # # For example, this is a simple `IO` on top of a `Bytes`: # @@ -29,10 +29,10 @@ require "c/errno" # slice.size # end # -# def write(slice : Bytes) : Nil +# def write(slice : Bytes) : UInt64 # slice.size.times { |i| @slice[i] = slice[i] } # @slice += slice.size -# nil +# slice.size.to_u64 # end # end # @@ -100,7 +100,7 @@ abstract class IO # io.write(slice) # io.to_s # => "abcd" # ``` - abstract def write(slice : Bytes) : Nil + abstract def write(slice : Bytes) : UInt64 # Closes this `IO`. # @@ -268,7 +268,6 @@ abstract class IO # :ditto: def printf(format_string, args : Array | Tuple) : Nil String::Formatter(typeof(args)).new(format_string, args, self).format - nil end # Reads a single byte from this `IO`. Returns `nil` if there is no more @@ -465,13 +464,12 @@ abstract class IO end # Writes a slice of UTF-8 encoded bytes to this `IO`, using the current encoding. - def write_utf8(slice : Bytes) + def write_utf8(slice : Bytes) : UInt64 if encoder = encoder() encoder.write(self, slice) else write(slice) end - nil end private def encoder @@ -814,22 +812,27 @@ abstract class IO # io.gets # => "world" # io.skip(1) # raises IO::EOFError # ``` - def skip(bytes_count : Int) : Nil + def skip(bytes_count : Int) : UInt64 + bytes_count = bytes_count.to_u64 + remaining = bytes_count buffer = uninitialized UInt8[4096] - while bytes_count > 0 - read_count = read(buffer.to_slice[0, Math.min(bytes_count, 4096)]) + while remaining > 0 + read_count = read(buffer.to_slice[0, Math.min(remaining, 4096)]) raise IO::EOFError.new if read_count == 0 - - bytes_count -= read_count + remaining -= read_count end + bytes_count end # Reads and discards bytes from `self` until there # are no more bytes. - def skip_to_end : Nil + def skip_to_end : UInt64 + bytes_count = 0_u64 buffer = uninitialized UInt8[4096] - while read(buffer.to_slice) > 0 + while (len = read(buffer.to_slice)) > 0 + bytes_count &+= len end + bytes_count end # Writes a single byte into this `IO`. @@ -839,7 +842,7 @@ abstract class IO # io.write_byte 97_u8 # io.to_s # => "a" # ``` - def write_byte(byte : UInt8) + def write_byte(byte : UInt8) : UInt64 x = byte write Slice.new(pointerof(x), 1) end @@ -847,7 +850,7 @@ abstract class IO # Writes the given object to this `IO` using the specified *format*. # # This ends up invoking `object.to_io(self, format)`, so any object defining a - # `to_io(io : IO, format : IO::ByteFormat = IO::ByteFormat::SystemEndian)` + # `to_io(io : IO, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : UInt64` # method can be written in this way. # # See `Int#to_io` and `Float#to_io`. @@ -858,7 +861,7 @@ abstract class IO # io.rewind # io.gets(4) # => "\u{4}\u{3}\u{2}\u{1}" # ``` - def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) + def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : UInt64 object.to_io(self, format) end diff --git a/src/io/buffered.cr b/src/io/buffered.cr index 1ed9b132d8f5..f38b953f3a30 100644 --- a/src/io/buffered.cr +++ b/src/io/buffered.cr @@ -110,30 +110,36 @@ module IO::Buffered end # :nodoc: - def skip(bytes_count) : Nil + def skip(bytes_count : Int) : UInt64 + bytes_count = bytes_count.to_u64 check_open if bytes_count <= @in_buffer_rem.size @in_buffer_rem += bytes_count - return + return bytes_count end - bytes_count -= @in_buffer_rem.size + remaining = bytes_count + remaining -= @in_buffer_rem.size @in_buffer_rem = Bytes.empty - super(bytes_count) + super(remaining) + bytes_count end # Buffered implementation of `IO#write(slice)`. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 + # NOTE: It returns the bytes written without differencing whether + # they are kept in the buffer or sent to the underlying IO. check_open - return if slice.empty? + return 0u64 if slice.empty? count = slice.size if sync? - return unbuffered_write(slice) + unbuffered_write(slice) + return slice.size.to_u64 end if flush_on_newline? @@ -149,7 +155,8 @@ module IO::Buffered if count >= @buffer_size flush - return unbuffered_write slice[0, count] + unbuffered_write slice[0, count] + return slice.size.to_u64 end if count > @buffer_size - @out_count @@ -158,11 +165,12 @@ module IO::Buffered slice.copy_to(out_buffer + @out_count, count) @out_count += count - nil + + slice.size.to_u64 end # :nodoc: - def write_byte(byte : UInt8) + def write_byte(byte : UInt8) : UInt64 check_open if sync? @@ -178,6 +186,8 @@ module IO::Buffered if flush_on_newline? && byte === '\n' flush end + + 1u64 end # Turns on/off `IO` **write** buffering. When *sync* is set to `true`, no buffering diff --git a/src/io/byte_format.cr b/src/io/byte_format.cr index 6f6b6a1082cd..e6f774106bd6 100644 --- a/src/io/byte_format.cr +++ b/src/io/byte_format.cr @@ -33,27 +33,27 @@ # io.to_slice # => Bytes[0x34, 0x12] # ``` module IO::ByteFormat - abstract def encode(int : Int8, io : IO) - abstract def encode(int : UInt8, io : IO) - abstract def encode(int : Int16, io : IO) - abstract def encode(int : UInt16, io : IO) - abstract def encode(int : Int32, io : IO) - abstract def encode(int : UInt32, io : IO) - abstract def encode(int : Int64, io : IO) - abstract def encode(int : UInt64, io : IO) - abstract def encode(int : Int128, io : IO) - abstract def encode(int : UInt128, io : IO) - - abstract def encode(int : Int8, bytes : Bytes) - abstract def encode(int : UInt8, bytes : Bytes) - abstract def encode(int : Int16, bytes : Bytes) - abstract def encode(int : UInt16, bytes : Bytes) - abstract def encode(int : Int32, bytes : Bytes) - abstract def encode(int : UInt32, bytes : Bytes) - abstract def encode(int : Int64, bytes : Bytes) - abstract def encode(int : UInt64, bytes : Bytes) - abstract def encode(int : Int128, bytes : Bytes) - abstract def encode(int : UInt128, bytes : Bytes) + abstract def encode(int : Int8, io : IO) : UInt64 + abstract def encode(int : UInt8, io : IO) : UInt64 + abstract def encode(int : Int16, io : IO) : UInt64 + abstract def encode(int : UInt16, io : IO) : UInt64 + abstract def encode(int : Int32, io : IO) : UInt64 + abstract def encode(int : UInt32, io : IO) : UInt64 + abstract def encode(int : Int64, io : IO) : UInt64 + abstract def encode(int : UInt64, io : IO) : UInt64 + abstract def encode(int : Int128, io : IO) : UInt64 + abstract def encode(int : UInt128, io : IO) : UInt64 + + abstract def encode(int : Int8, bytes : Bytes) : UInt64 + abstract def encode(int : UInt8, bytes : Bytes) : UInt64 + abstract def encode(int : Int16, bytes : Bytes) : UInt64 + abstract def encode(int : UInt16, bytes : Bytes) : UInt64 + abstract def encode(int : Int32, bytes : Bytes) : UInt64 + abstract def encode(int : UInt32, bytes : Bytes) : UInt64 + abstract def encode(int : Int64, bytes : Bytes) : UInt64 + abstract def encode(int : UInt64, bytes : Bytes) : UInt64 + abstract def encode(int : Int128, bytes : Bytes) : UInt64 + abstract def encode(int : UInt128, bytes : Bytes) : UInt64 abstract def decode(int : Int8.class, io : IO) abstract def decode(int : UInt8.class, io : IO) @@ -77,11 +77,11 @@ module IO::ByteFormat abstract def decode(int : Int128.class, bytes : Bytes) abstract def decode(int : UInt128.class, bytes : Bytes) - def encode(float : Float32, io : IO) + def encode(float : Float32, io : IO) : UInt64 encode(float.unsafe_as(Int32), io) end - def encode(float : Float32, bytes : Bytes) + def encode(float : Float32, bytes : Bytes) : UInt64 encode(float.unsafe_as(Int32), bytes) end @@ -93,11 +93,11 @@ module IO::ByteFormat decode(Int32, bytes).unsafe_as(Float32) end - def encode(float : Float64, io : IO) + def encode(float : Float64, io : IO) : UInt64 encode(float.unsafe_as(Int64), io) end - def encode(float : Float64, bytes : Bytes) + def encode(float : Float64, bytes : Bytes) : UInt64 encode(float.unsafe_as(Int64), bytes) end @@ -125,16 +125,18 @@ module IO::ByteFormat {% for type, i in %w(Int8 UInt8 Int16 UInt16 Int32 UInt32 Int64 UInt64 Int128 UInt128) %} {% bytesize = 2 ** (i // 2) %} - def self.encode(int : {{type.id}}, io : IO) + def self.encode(int : {{type.id}}, io : IO) : UInt64 buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self io.write(buffer.to_slice) + UInt64.new({{bytesize}}) end - def self.encode(int : {{type.id}}, bytes : Bytes) + def self.encode(int : {{type.id}}, bytes : Bytes) : UInt64 buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self buffer.to_slice.copy_to(bytes) + UInt64.new({{bytesize}}) end def self.decode(type : {{type.id}}.class, io : IO) diff --git a/src/io/delimited.cr b/src/io/delimited.cr index 2142391502a0..70fc30f98a38 100644 --- a/src/io/delimited.cr +++ b/src/io/delimited.cr @@ -107,7 +107,7 @@ class IO::Delimited < IO read_bytes end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 raise IO::Error.new "Can't write to IO::Delimited" end diff --git a/src/io/encoding.cr b/src/io/encoding.cr index 174029fbb31c..49a5c086d67a 100644 --- a/src/io/encoding.cr +++ b/src/io/encoding.cr @@ -27,7 +27,8 @@ class IO @closed = false end - def write(io, slice : Bytes) + def write(io, slice : Bytes) : UInt64 + bytes_written = 0u64 inbuf_ptr = slice.to_unsafe inbytesleft = LibC::SizeT.new(slice.size) outbuf = uninitialized UInt8[1024] @@ -38,8 +39,9 @@ class IO if err == Crystal::Iconv::ERROR @iconv.handle_invalid(pointerof(inbuf_ptr), pointerof(inbytesleft)) end - io.write(outbuf.to_slice[0, outbuf.size - outbytesleft]) + bytes_written &+= io.write(outbuf.to_slice[0, outbuf.size - outbytesleft]) end + bytes_written end def close diff --git a/src/io/hexdump.cr b/src/io/hexdump.cr index 3df681f62ef8..35afd9b7727b 100644 --- a/src/io/hexdump.cr +++ b/src/io/hexdump.cr @@ -32,8 +32,8 @@ class IO::Hexdump < IO end end - def write(buf : Bytes) : Nil - return if buf.empty? + def write(buf : Bytes) : UInt64 + return 0u64 if buf.empty? @io.write(buf).tap do @output.puts buf.hexdump if @write diff --git a/src/io/memory.cr b/src/io/memory.cr index 4a2f0d51852d..a72839c2cd2e 100644 --- a/src/io/memory.cr +++ b/src/io/memory.cr @@ -82,13 +82,13 @@ class IO::Memory < IO # See `IO#write(slice)`. Raises if this `IO::Memory` is non-writeable, # or if it's non-resizeable and a resize is needed. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_writeable check_open count = slice.size - return if count == 0 + return 0u64 if count == 0 new_bytesize = @pos + count if new_bytesize > @capacity @@ -105,7 +105,7 @@ class IO::Memory < IO @pos += count @bytesize = @pos if @pos > @bytesize - nil + slice.size.to_u64 end # See `IO#write_byte`. Raises if this `IO::Memory` is non-writeable, @@ -194,7 +194,8 @@ class IO::Memory < IO end # :nodoc: - def skip(bytes_count) + def skip(bytes_count : Int) : UInt64 + bytes_count = bytes_count.to_u64 check_open available = @bytesize - @pos @@ -203,13 +204,16 @@ class IO::Memory < IO else raise IO::EOFError.new end + bytes_count end # :nodoc: - def skip_to_end + def skip_to_end : UInt64 check_open + skipped = @bytesize - @pos @pos = @bytesize + skipped.to_u64 end # :nodoc: diff --git a/src/io/multi_writer.cr b/src/io/multi_writer.cr index c06bc8649ed9..35e7682f961c 100644 --- a/src/io/multi_writer.cr +++ b/src/io/multi_writer.cr @@ -29,12 +29,14 @@ class IO::MultiWriter < IO @writers = writers.map(&.as(IO)).to_a end - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_open - return if slice.empty? + return 0u64 if slice.empty? @writers.each { |writer| writer.write(slice) } + + slice.size.to_u64 end def read(slice : Bytes) diff --git a/src/io/sized.cr b/src/io/sized.cr index 253df834a743..ae9b88cdb9fe 100644 --- a/src/io/sized.cr +++ b/src/io/sized.cr @@ -61,7 +61,8 @@ class IO::Sized < IO peek end - def skip(bytes_count) : Nil + def skip(bytes_count : Int) : UInt64 + bytes_count = bytes_count.to_u64 check_open if bytes_count <= @read_remaining @@ -70,6 +71,8 @@ class IO::Sized < IO else raise IO::EOFError.new end + + bytes_count end def write(slice : Bytes) : NoReturn diff --git a/src/io/stapled.cr b/src/io/stapled.cr index 24e9067d33e6..52c1f7fb69f1 100644 --- a/src/io/stapled.cr +++ b/src/io/stapled.cr @@ -51,12 +51,19 @@ class IO::Stapled < IO end # Skips `reader`. - def skip(bytes_count : Int) : Nil + def skip(bytes_count : Int) : UInt64 check_open @reader.skip(bytes_count) end + # Skips `reader`. + def skip_to_end : UInt64 + check_open + + @reader.skip_to_end + end + # Writes a byte to `writer`. def write_byte(byte : UInt8) : Nil check_open @@ -65,10 +72,10 @@ class IO::Stapled < IO end # Writes a slice to `writer`. - def write(slice : Bytes) : Nil + def write(slice : Bytes) : UInt64 check_open - return if slice.empty? + return 0u64 if slice.empty? @writer.write(slice) end diff --git a/src/openssl/digest/digest_io.cr b/src/openssl/digest/digest_io.cr index 4093725fefcc..2a3b002ad97a 100644 --- a/src/openssl/digest/digest_io.cr +++ b/src/openssl/digest/digest_io.cr @@ -42,8 +42,8 @@ module OpenSSL read_bytes end - def write(slice : Bytes) : Nil - return if slice.empty? + def write(slice : Bytes) : UInt64 + return 0u64 if slice.empty? if @mode.write? digest_algorithm.update(slice) diff --git a/src/openssl/ssl/socket.cr b/src/openssl/ssl/socket.cr index 2869d7988c30..cbab93538402 100644 --- a/src/openssl/ssl/socket.cr +++ b/src/openssl/ssl/socket.cr @@ -144,7 +144,6 @@ abstract class OpenSSL::SSL::Socket < IO unless bytes > 0 raise OpenSSL::SSL::Error.new(@ssl, bytes, "SSL_write") end - nil end def unbuffered_flush diff --git a/src/string/builder.cr b/src/string/builder.cr index bd6f94b8f15c..c117a45fec4f 100644 --- a/src/string/builder.cr +++ b/src/string/builder.cr @@ -38,8 +38,8 @@ class String::Builder < IO raise "Not implemented" end - def write(slice : Bytes) : Nil - return if slice.empty? + def write(slice : Bytes) : UInt64 + return 0u64 if slice.empty? count = slice.size new_bytesize = real_bytesize + count @@ -50,7 +50,7 @@ class String::Builder < IO slice.copy_to(@buffer + real_bytesize, count) @bytesize += count - nil + slice.size.to_u64 end def write_byte(byte : UInt8) From 445b993dec1a2680ee35a8000bac18e09f0cb833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 18 May 2020 21:00:21 +0200 Subject: [PATCH 044/263] [Docs] Fix source_url_pattern with canonical repo name (#9305) GitHub and Gitlab won't resolve source URLs when the repo name has a .git extension. This extension works for other use cases (like git URLs). --- spec/compiler/crystal/tools/doc/project_info_spec.cr | 9 ++++++--- src/compiler/crystal/tools/doc/project_info.cr | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/project_info_spec.cr b/spec/compiler/crystal/tools/doc/project_info_spec.cr index 439639f0b9de..f255d8ae72de 100644 --- a/spec/compiler/crystal/tools/doc/project_info_spec.cr +++ b/spec/compiler/crystal/tools/doc/project_info_spec.cr @@ -258,7 +258,7 @@ describe Crystal::Doc::ProjectInfo do ProjectInfo.find_source_url_pattern("http://example.com/foo/bar").should be_nil ProjectInfo.find_source_url_pattern("git@github.com:foo/bar/").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" - ProjectInfo.find_source_url_pattern("git@github.com:foo/bar.git").should eq "https://github.com/foo/bar.git/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("git@github.com:foo/bar.git").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("git@github.com:foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("http://github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" @@ -266,18 +266,21 @@ describe Crystal::Doc::ProjectInfo do ProjectInfo.find_source_url_pattern("http://www.github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("https://www.github.com/foo/bar").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" - ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.git").should eq "https://github.com/foo/bar.git/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.git").should eq "https://github.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.cr").should eq "https://github.com/foo/bar.cr/blob/%{refname}/%{path}#L%{line}" - ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.cr.git").should eq "https://github.com/foo/bar.cr.git/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("https://github.com/foo/bar.cr.git").should eq "https://github.com/foo/bar.cr/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("git@gitlab.com:foo/bar").should eq "https://gitlab.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("http://gitlab.com/foo/bar").should eq "https://gitlab.com/foo/bar/blob/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://gitlab.com/foo/bar.git").should eq "https://gitlab.com/foo/bar/blob/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("git@bitbucket.com:foo/bar").should eq "https://bitbucket.com/foo/bar/src/%{refname}/%{path}#%{filename}-%{line}" ProjectInfo.find_source_url_pattern("http://bitbucket.com/foo/bar").should eq "https://bitbucket.com/foo/bar/src/%{refname}/%{path}#%{filename}-%{line}" + ProjectInfo.find_source_url_pattern("http://bitbucket.com/foo/bar.git").should eq "https://bitbucket.com/foo/bar/src/%{refname}/%{path}#%{filename}-%{line}" ProjectInfo.find_source_url_pattern("git@git.sr.ht:~foo/bar").should eq "https://git.sr.ht/~foo/bar/tree/%{refname}/%{path}#L%{line}" ProjectInfo.find_source_url_pattern("http://git.sr.ht/~foo/bar").should eq "https://git.sr.ht/~foo/bar/tree/%{refname}/%{path}#L%{line}" + ProjectInfo.find_source_url_pattern("http://git.sr.ht/~foo/bar.git").should eq "https://git.sr.ht/~foo/bar.git/tree/%{refname}/%{path}#L%{line}" end describe "#source_url" do diff --git a/src/compiler/crystal/tools/doc/project_info.cr b/src/compiler/crystal/tools/doc/project_info.cr index 0ebf986e1209..01ec280e0076 100644 --- a/src/compiler/crystal/tools/doc/project_info.cr +++ b/src/compiler/crystal/tools/doc/project_info.cr @@ -92,12 +92,20 @@ module Crystal::Doc case host when "github.com", "www.github.com" + # GitHub only resolves URLs with the canonical repo name without .git extension. + path = path.rchop(".git") "https://github.com/#{path}/blob/%{refname}/%{path}#L%{line}" when "gitlab.com", "www.gitlab.com" + # Gitlab only resolves URLs with the canonical repo name without .git extension. + path = path.rchop(".git") "https://gitlab.com/#{path}/blob/%{refname}/%{path}#L%{line}" when "bitbucket.com", "www.bitbucket.com" + # Bitbucket does resolve URLs the URL with .git extension, but without it + # the canonical form and should be preferred. + path = path.rchop(".git") "https://bitbucket.com/#{path}/src/%{refname}/%{path}#%{filename}-%{line}" when "git.sr.ht" + # On git.sr.ht ~foo/bar and ~foo/bar.git seem to mean different repos. "https://git.sr.ht/#{path}/tree/%{refname}/%{path}#L%{line}" else # Unknown remote host, can't determine source url pattern From 7b9069f1dfac1d1c59b44e13916cee6fc17312d0 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 18 May 2020 21:10:09 +0200 Subject: [PATCH 045/263] Implement Process(env:) on Windows and add more Env specs (#9310) * Implement Process(env:) on Windows and add more Env specs Co-authored-by: Jan Zajic * Add comment about "empty" keys in env * Add comments Co-authored-by: Jan Zajic --- spec/std/env_spec.cr | 23 ++++ spec/std/process_spec.cr | 100 ++++++++++++++++-- src/crystal/system/win32/env.cr | 20 +++- src/crystal/system/win32/process.cr | 31 ++++-- .../c/processthreadsapi.cr | 2 + 5 files changed, 157 insertions(+), 19 deletions(-) diff --git a/spec/std/env_spec.cr b/spec/std/env_spec.cr index bd7e062c0d69..7465aea27128 100644 --- a/spec/std/env_spec.cr +++ b/spec/std/env_spec.cr @@ -20,6 +20,23 @@ describe "ENV" do ENV.delete("FOO") end + {% if flag?(:win32) %} + it "sets and gets case-insensitive" do + (ENV["FOO"] = "1").should eq("1") + ENV["Foo"].should eq("1") + ENV["foo"]?.should eq("1") + ensure + ENV.delete("FOO") + end + {% else %} + it "sets and gets case-sensitive" do + ENV["FOO"] = "1" + ENV["foo"]?.should be_nil + ensure + ENV.delete("FOO") + end + {% end %} + it "sets to nil (same as delete)" do ENV["FOO"] = "1" ENV["FOO"]?.should_not be_nil @@ -58,6 +75,12 @@ describe "ENV" do ENV.delete("BAR") end + it "does not have an empty key" do + # Setting an empty key is invalid on both POSIX and Windows. So reporting an empty key + # would always be a bug. And there *was* a bug - see win32/ Crystal::System::Env.each + ENV.keys.should_not contain("") + end + it "does .values" do [1, 2].each { |i| ENV.values.should_not contain("SOMEVALUE_#{i}") } ENV["FOO"] = "SOMEVALUE_1" diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 1fb65ffcc09d..7389fa8a4a78 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -33,6 +33,15 @@ private def stdin_to_stdout_command {% end %} end +private def print_env_command + {% if flag?(:win32) %} + # cmd adds these by itself, clear them out before printing. + shell_command("set COMSPEC=& set PATHEXT=& set PROMPT=& set") + {% else %} + {"env", [] of String} + {% end %} +end + private def standing_command {% if flag?(:win32) %} {"cmd.exe"} @@ -189,28 +198,99 @@ describe Process do end describe "environ" do - pending_win32 "clears the environment" do - value = Process.run("env", clear_env: true) do |proc| + it "clears the environment" do + value = Process.run(*print_env_command, clear_env: true) do |proc| proc.output.gets_to_end end value.should eq("") end - pending_win32 "sets an environment variable" do - env = {"FOO" => "bar"} - value = Process.run("env", clear_env: true, env: env) do |proc| + it "clears and sets an environment variable" do + value = Process.run(*print_env_command, clear_env: true, env: {"FOO" => "bar"}) do |proc| proc.output.gets_to_end end - value.should eq("FOO=bar\n") + value.should eq("FOO=bar#{newline}") end - pending_win32 "deletes an environment variable" do - env = {"HOME" => nil} - value = Process.run("env | egrep '^HOME='", env: env, shell: true) do |proc| + it "sets an environment variable" do + value = Process.run(*print_env_command, env: {"FOO" => "bar"}) do |proc| proc.output.gets_to_end end - value.should eq("") + value.should match /(*ANYCRLF)^FOO=bar$/m + end + + it "sets an empty environment variable" do + value = Process.run(*print_env_command, env: {"FOO" => ""}) do |proc| + proc.output.gets_to_end + end + value.should match /(*ANYCRLF)^FOO=$/m + end + + it "deletes existing environment variable" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command, env: {"FOO" => nil}) do |proc| + proc.output.gets_to_end + end + value.should_not match /(*ANYCRLF)^FOO=/m + ensure + ENV.delete("FOO") + end + + {% if flag?(:win32) %} + it "deletes existing environment variable case-insensitive" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command, env: {"foo" => nil}) do |proc| + proc.output.gets_to_end + end + value.should_not match /(*ANYCRLF)^FOO=/mi + ensure + ENV.delete("FOO") + end + {% end %} + + it "preserves existing environment variable" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command) do |proc| + proc.output.gets_to_end + end + value.should match /(*ANYCRLF)^FOO=bar$/m + ensure + ENV.delete("FOO") + end + + it "preserves and sets an environment variable" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command, env: {"FOO2" => "bar2"}) do |proc| + proc.output.gets_to_end + end + value.should match /(*ANYCRLF)^FOO=bar$/m + value.should match /(*ANYCRLF)^FOO2=bar2$/m + ensure + ENV.delete("FOO") + end + + it "overrides existing environment variable" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command, env: {"FOO" => "different"}) do |proc| + proc.output.gets_to_end + end + value.should match /(*ANYCRLF)^FOO=different$/m + ensure + ENV.delete("FOO") end + + {% if flag?(:win32) %} + it "overrides existing environment variable case-insensitive" do + ENV["FOO"] = "bar" + value = Process.run(*print_env_command, env: {"fOo" => "different"}) do |proc| + proc.output.gets_to_end + end + value.should_not match /(*ANYCRLF)^FOO=/m + value.should match /(*ANYCRLF)^fOo=different$/m + ensure + ENV.delete("FOO") + end + {% end %} end describe "signal" do diff --git a/src/crystal/system/win32/env.cr b/src/crystal/system/win32/env.cr index b908dfef1b4e..baad80a01234 100644 --- a/src/crystal/system/win32/env.cr +++ b/src/crystal/system/win32/env.cr @@ -65,13 +65,27 @@ module Crystal::System::Env begin while !pointer.value.zero? string, pointer = String.from_utf16(pointer) - key_value = string.split('=', 2) - key = key_value[0] - value = key_value[1]? || "" + key, _, value = string.partition('=') + # The actual env variables are preceded by these weird lines in the output: + # "=::=::\", "=C:=c:\foo\bar", "=ExitCode=00000000" -- skip them. + next if key.empty? yield key, value end ensure LibC.FreeEnvironmentStringsW(orig_pointer) end end + + # Used internally to create an input for `CreateProcess` `lpEnvironment`. + def self.make_env_block(env : Enumerable({String, String})) + String.build do |io| + env.each do |(key, value)| + if key.includes?('=') || key.empty? + raise ArgumentError.new("Invalid env key #{key.inspect}") + end + io << key.check_no_null_byte("key") << '=' << value.check_no_null_byte("value") << '\0' + end + io << '\0' + end.to_utf16.to_unsafe + end end diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 0ee56c760cd6..27853dee7024 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -111,10 +111,6 @@ struct Crystal::System::Process end def self.spawn(command_args, env, clear_env, input, output, error, chdir) - if env || clear_env - raise NotImplementedError.new("Process.new with env or clear_env options") - end - startup_info = LibC::STARTUPINFOW.new startup_info.cb = sizeof(LibC::STARTUPINFOW) startup_info.dwFlags = LibC::STARTF_USESTDHANDLES @@ -126,8 +122,8 @@ struct Crystal::System::Process process_info = LibC::PROCESS_INFORMATION.new if LibC.CreateProcessW( - nil, command_args.check_no_null_byte.to_utf16, nil, nil, true, 0, - nil, chdir.try &.check_no_null_byte.to_utf16, + nil, command_args.check_no_null_byte.to_utf16, nil, nil, true, LibC::CREATE_UNICODE_ENVIRONMENT, + make_env_block(env, clear_env), chdir.try &.check_no_null_byte.to_utf16, pointerof(startup_info), pointerof(process_info) ) == 0 raise RuntimeError.from_winerror("Error executing process") @@ -190,6 +186,29 @@ struct Crystal::System::Process def self.chroot(path) raise NotImplementedError.new("Process.chroot") end + + protected def self.make_env_block(env, clear_env : Bool) : UInt16* + # If neither clearing nor adding anything, use the default behavior of inheriting everything. + return Pointer(UInt16).null if !env && !clear_env + + # Emulate case-insensitive behavior using a Hash like {"KEY" => {"kEy", "value"}, ...} + final_env = {} of String => {String, String} + unless clear_env + Crystal::System::Env.each do |key, val| + final_env[key.upcase] = {key, val} + end + end + env.try &.each do |(key, val)| + if val + # Note: in the case of overriding, the last "case-spelling" of the key wins. + final_env[key.upcase] = {key, val} + else + final_env.delete key.upcase + end + end + # The "values" we're passing are actually key-value pairs. + Crystal::System::Env.make_env_block(final_env.each_value) + end end private def close_handle(handle) diff --git a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr index 476c54af3028..6f5ed78f865d 100644 --- a/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/processthreadsapi.cr @@ -1,6 +1,8 @@ require "./basetsd" lib LibC + CREATE_UNICODE_ENVIRONMENT = 0x00000400 + struct PROCESS_INFORMATION hProcess : HANDLE hThread : HANDLE From 5188e1027896a5e1472ae668770049b52fa15a42 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 18 May 2020 16:13:29 -0300 Subject: [PATCH 046/263] Support different number of fraction digits for RFC3339 time format (#9283) * Support different number of fraction digits for RFC3339 time format. Added `fraction_digits` parameter to `Time#to_rfc3339` and `Time#to_rfc3339(IO)`. * `Time::Format::Formatter#second_fraction?` also raises with negative values * Make the `fraction_digits` a keyword argument in `Time#to_rfc3339` * RFC3339 format without seconds by default * Change the default of `second_fraction?` back to full precision --- spec/std/time/format_spec.cr | 18 ++++++++++++++++++ spec/std/yaml/serialization_spec.cr | 3 ++- src/log/format.cr | 2 +- src/time.cr | 15 +++++++++++---- src/time/format/custom/rfc_3339.cr | 6 +++--- src/time/format/formatter.cr | 12 ++++++++---- 6 files changed, 43 insertions(+), 13 deletions(-) diff --git a/spec/std/time/format_spec.cr b/spec/std/time/format_spec.cr index 44926a37f907..9820df0c63b7 100644 --- a/spec/std/time/format_spec.cr +++ b/spec/std/time/format_spec.cr @@ -153,6 +153,24 @@ describe Time::Format do Time.parse_rfc2822(time.to_rfc2822).should eq time end + it "formats rfc3339 with different fraction digits" do + time = Time.utc(2016, 2, 15, 8, 23, 45, nanosecond: 123456789) + time.to_rfc3339.should eq "2016-02-15T08:23:45Z" + time.to_rfc3339(fraction_digits: 0).should eq "2016-02-15T08:23:45Z" + time.to_rfc3339(fraction_digits: 3).should eq "2016-02-15T08:23:45.123Z" + time.to_rfc3339(fraction_digits: 6).should eq "2016-02-15T08:23:45.123456Z" + time.to_rfc3339(fraction_digits: 9).should eq "2016-02-15T08:23:45.123456789Z" + expect_raises(ArgumentError, "Invalid fraction digits: 5") { time.to_rfc3339(fraction_digits: 5) } + expect_raises(ArgumentError, "Invalid fraction digits: -1") { time.to_rfc3339(fraction_digits: -1) } + + time = Time.utc(2016, 2, 15, 8, 23, 45) + time.to_rfc3339.should eq "2016-02-15T08:23:45Z" + time.to_rfc3339(fraction_digits: 0).should eq "2016-02-15T08:23:45Z" + time.to_rfc3339(fraction_digits: 3).should eq "2016-02-15T08:23:45.000Z" + time.to_rfc3339(fraction_digits: 6).should eq "2016-02-15T08:23:45.000000Z" + time.to_rfc3339(fraction_digits: 9).should eq "2016-02-15T08:23:45.000000000Z" + end + it "parses empty" do t = Time.parse("", "", Time::Location.local) t.year.should eq(1) diff --git a/spec/std/yaml/serialization_spec.cr b/spec/std/yaml/serialization_spec.cr index 6a09eb59cd9c..9fd542308f60 100644 --- a/spec/std/yaml/serialization_spec.cr +++ b/spec/std/yaml/serialization_spec.cr @@ -1,3 +1,4 @@ +require "../spec_helper" require "spec" require "yaml" {% unless flag?(:win32) %} @@ -349,7 +350,7 @@ describe "YAML serialization" do it "does for utc time" do time = Time.utc(2010, 11, 12, 1, 2, 3) - assert_yaml_document_end(time.to_yaml, "--- 2010-11-12 01:02:03\n") + assert_yaml_document_end(time.to_yaml, "--- 2010-11-12 01:02:03.000000000\n") end it "does for time at date" do diff --git a/src/log/format.cr b/src/log/format.cr index 57ffe7346ebd..e7be76543464 100644 --- a/src/log/format.cr +++ b/src/log/format.cr @@ -63,7 +63,7 @@ class Log # Write the entry timestamp in RFC3339 format def timestamp - @entry.timestamp.to_rfc3339(@io) + @entry.timestamp.to_rfc3339(@io, fraction_digits: 6) end # Write a fixed string diff --git a/src/time.cr b/src/time.cr index c4264019fb21..1f878880b9be 100644 --- a/src/time.cr +++ b/src/time.cr @@ -1105,14 +1105,21 @@ struct Time # ISO 8601 allows some freedom over the syntax and RFC 3339 exercises that # freedom to rigidly define a fixed format intended for use in internet # protocols and standards. - def to_rfc3339 - Format::RFC_3339.format(to_utc) + # + # Number of seconds decimals can be selected with *fraction_digits*. + # Values accepted are 0 (the default, no decimals), 3 (milliseconds), 6 (microseconds) or 9 (nanoseconds). + def to_rfc3339(*, fraction_digits : Int = 0) + Format::RFC_3339.format(to_utc, fraction_digits) end # Format this time using the format specified by [RFC 3339](https://tools.ietf.org/html/rfc3339) ([ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf) profile). # into the given *io*. - def to_rfc3339(io : IO) - Format::RFC_3339.format(to_utc, io) + # + # + # Number of seconds decimals can be selected with *fraction_digits*. + # Values accepted are 0 (the default, no decimals), 3 (milliseconds), 6 (microseconds) or 9 (nanoseconds). + def to_rfc3339(io : IO, *, fraction_digits : Int = 0) + Format::RFC_3339.format(to_utc, io, fraction_digits) end # Parse time format specified by [RFC 3339](https://tools.ietf.org/html/rfc3339) ([ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf) profile). diff --git a/src/time/format/custom/rfc_3339.cr b/src/time/format/custom/rfc_3339.cr index 94a68ac9f09e..7c16669aab9a 100644 --- a/src/time/format/custom/rfc_3339.cr +++ b/src/time/format/custom/rfc_3339.cr @@ -9,14 +9,14 @@ struct Time::Format end # Formats a `Time` into the given *io*. - def self.format(time : Time, io : IO, fraction_digits = nil) + def self.format(time : Time, io : IO, fraction_digits = 0) formatter = Formatter.new(time, io) formatter.rfc_3339(fraction_digits: fraction_digits) io end # Formats a `Time` into a `String`. - def self.format(time : Time, fraction_digits = nil) + def self.format(time : Time, fraction_digits = 0) String.build do |io| format(time, io, fraction_digits: fraction_digits) end @@ -24,7 +24,7 @@ struct Time::Format end module Pattern - def rfc_3339(fraction_digits = nil) + def rfc_3339(fraction_digits = 0) year_month_day char 'T', 't', ' ' twenty_four_hour_time_with_seconds diff --git a/src/time/format/formatter.cr b/src/time/format/formatter.cr index ca52495dfa4e..cc259ca53081 100644 --- a/src/time/format/formatter.cr +++ b/src/time/format/formatter.cr @@ -147,10 +147,14 @@ struct Time::Format nanoseconds end - def second_fraction?(fraction_digits = nil) - unless time.nanosecond == 0 || fraction_digits == 0 - char '.' - second_fraction + def second_fraction?(fraction_digits : Int = 9) + case fraction_digits + when 0 + when 3 then char '.'; milliseconds + when 6 then char '.'; microseconds + when 9 then char '.'; nanoseconds + else + raise ArgumentError.new("Invalid fraction digits: #{fraction_digits}") end end From 40be93b586323a9ca7be17386f78b57a4a9d451e Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 18 May 2020 16:16:20 -0300 Subject: [PATCH 047/263] Followup of #9134. Missing swap of arguments (#9303) --- src/http/formdata/builder.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/formdata/builder.cr b/src/http/formdata/builder.cr index f8d9893ab913..5947dc06724b 100644 --- a/src/http/formdata/builder.cr +++ b/src/http/formdata/builder.cr @@ -91,19 +91,19 @@ module HTTP::FormData if creation_time = metadata.creation_time io << %(; creation-date=") - creation_time.to_s("%a, %d %b %Y %H:%M:%S %z", io) + creation_time.to_s(io, "%a, %d %b %Y %H:%M:%S %z") io << '"' end if modification_time = metadata.modification_time io << %(; modification-date=") - modification_time.to_s("%a, %d %b %Y %H:%M:%S %z", io) + modification_time.to_s(io, "%a, %d %b %Y %H:%M:%S %z") io << '"' end if read_time = metadata.read_time io << %(; read-date=") - read_time.to_s("%a, %d %b %Y %H:%M:%S %z", io) + read_time.to_s(io, "%a, %d %b %Y %H:%M:%S %z") io << '"' end From ed2bb7bc3e245b3c3215753143c451d1665e8d95 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 18 May 2020 20:19:54 -0300 Subject: [PATCH 048/263] Log::Metadata optimizations (#9295) * Refactor: simplify initializers and drop merge in favor of extend We are no longer allowing _merging_ two context, a context can be extended via NamedTuple/Hash. This will allow later implementation refactor to chain context in a linked list fashion. Initializers are simplified and keys are converted to String along the way * Refactor: Avoid duplication due to as_h? * Split root Metadata with Metadata::Value Drop immutability of Metadata::Value Keep JSON support for both Remove validations of Hash like Metadata since the root is always hash. Use Symbols as keys of Metadata, while using String as keys in Metadata::Value * Refactor: Allocate entries inline in Metadata * Add specs and minor optimizations * Refactor: defrag entries before iteration * Add Metadata#[] accessors * Make thread-safe Metadata#defrag * Tweak Log::Metadata#to_s --- spec/std/log/context_spec.cr | 15 +-- spec/std/log/format_spec.cr | 8 +- spec/std/log/io_backend_spec.cr | 2 +- spec/std/log/log_spec.cr | 17 +-- spec/std/log/metadata_spec.cr | 132 +++++++++++++------ src/log/entry.cr | 1 - src/log/format.cr | 4 +- src/log/json.cr | 17 ++- src/log/main.cr | 8 +- src/log/metadata.cr | 224 +++++++++++++++++++++++++------- 10 files changed, 302 insertions(+), 126 deletions(-) diff --git a/spec/std/log/context_spec.cr b/spec/std/log/context_spec.cr index ccc66b48503d..1b217e8fdb03 100644 --- a/spec/std/log/context_spec.cr +++ b/spec/std/log/context_spec.cr @@ -2,7 +2,7 @@ require "spec" require "log" private def m(value) - Log::Metadata.new(value) + Log::Metadata.build(value) end describe "Log.context" do @@ -14,12 +14,6 @@ describe "Log.context" do Log.context.clear end - it "validates hash" do - expect_raises(ArgumentError, "Expected hash context, not Int32") do - Log.context = m(1) - end - end - it "can be set and cleared" do Log.context.metadata.should eq(Log::Metadata.new) @@ -90,13 +84,6 @@ describe "Log.context" do Log.context.metadata.should eq(m({a: 1, b: 2})) end - it "is assignable from a hash with string keys" do - Log.context.set a: 1 - extra = {"b" => 2} - Log.context.set extra - Log.context.metadata.should eq(m({a: 1, b: 2})) - end - it "is assignable from a named tuple" do Log.context.set a: 1 extra = {b: 2} diff --git a/spec/std/log/format_spec.cr b/spec/std/log/format_spec.cr index c8b9ead1f5db..feef1beb3874 100644 --- a/spec/std/log/format_spec.cr +++ b/spec/std/log/format_spec.cr @@ -24,17 +24,17 @@ class Log end io = IO::Memory.new ShortFormat.format(entry, io) - io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- {"a" => 1, "b" => 2}$/) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- a: 1, b: 2$/) end it "shows context and entry data" do entry = Log.with_context do Log.context.set a: 1, b: 2 - Entry.new("source", :info, "message", Log::Metadata.new({c: 3, d: 4}), nil) + Entry.new("source", :info, "message", Log::Metadata.build({c: 3, d: 4}), nil) end io = IO::Memory.new ShortFormat.format(entry, io) - io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- {"c" => 3, "d" => 4} -- {"a" => 1, "b" => 2}$/) + io.to_s.should match(/^[\d\-.:TZ]+\s* INFO - source: message -- c: 3, d: 4 -- a: 1, b: 2$/) end it "appends the exception" do @@ -79,7 +79,7 @@ class Log TestFormatter.format(Entry.new("source", :error, "Oh, no", Log::Metadata.empty, exception), io) io.rewind - io.gets.should eq(" INFO [source] test message ({\"a\" => 1, \"b\" => 2})") + io.gets.should eq(" INFO [source] test message (a: 1, b: 2)") io.gets.should eq(" INFO test message") io.gets.should eq(" ERROR [source] test Oh, no") io.gets_to_end.should eq(exception.inspect_with_backtrace) diff --git a/spec/std/log/io_backend_spec.cr b/spec/std/log/io_backend_spec.cr index 56a1a4dca46b..8c03f8e1bc17 100644 --- a/spec/std/log/io_backend_spec.cr +++ b/spec/std/log/io_backend_spec.cr @@ -43,7 +43,7 @@ describe Log::IOBackend do logger.info { "info:show" } end - r.gets.should match(/info:show -- {"foo" => "bar"}/) + r.gets.should match(/info:show -- foo: "bar"/) end end diff --git a/spec/std/log/log_spec.cr b/spec/std/log/log_spec.cr index aeb8f294d01d..69259bc6576f 100644 --- a/spec/std/log/log_spec.cr +++ b/spec/std/log/log_spec.cr @@ -6,7 +6,7 @@ private def s(value : Log::Severity) end private def m(value) - Log::Metadata.new(value) + Log::Metadata.build(value) end describe Log do @@ -114,7 +114,7 @@ describe Log do log.info { "info message" } - backend.entries.first.context.should eq(Log::Metadata.new({a: 1})) + backend.entries.first.context.should eq(Log::Metadata.build({a: 1})) end it "context can be changed within the block, yet it's not restored" do @@ -125,8 +125,8 @@ describe Log do log.info { Log.context.set(b: 2); "info message" } - backend.entries.first.context.should eq(Log::Metadata.new({a: 1, b: 2})) - Log.context.metadata.should eq(Log::Metadata.new({a: 1, b: 2})) + backend.entries.first.context.should eq(Log::Metadata.build({a: 1, b: 2})) + Log.context.metadata.should eq(Log::Metadata.build({a: 1, b: 2})) end describe "emitter dsl" do @@ -257,14 +257,5 @@ describe Log do entry.data.should eq(m({a: 1})) entry.exception.should be_nil end - - it "validates hash" do - expect_raises(ArgumentError, "Expected hash data, not Int32") do - backend = Log::MemoryBackend.new - log = Log.new("a", backend, :notice) - - log.notice &.emit(m(1)) - end - end end end diff --git a/spec/std/log/metadata_spec.cr b/spec/std/log/metadata_spec.cr index 4674c9ecef99..5319462c0756 100644 --- a/spec/std/log/metadata_spec.cr +++ b/spec/std/log/metadata_spec.cr @@ -3,67 +3,121 @@ require "log" require "log/json" private def m(value) - Log::Metadata.new(value) + Log::Metadata.build(value) end -describe Log::Metadata do - it "initialize" do - m({a: 1}).should eq(m({"a" => m(1)})) - m({a: 1, b: ["str", true], num: 1i64}).should eq(m({"a" => m(1), "b" => m([m("str"), m(true)]), "num" => m(1i64)})) - m({a: 1f32, b: 1f64}).should eq(m({"a" => m(1f32), "b" => m(1f64)})) - t = Time.local - m({time: t}).should eq(m({"time" => m(t)})) - Log::Metadata.new.should eq(m(NamedTuple.new)) - end +private def v(value) + Log::Metadata::Value.new(value) +end +describe Log::Metadata do it "empty" do Log::Metadata.empty.should eq(Log::Metadata.new) Log::Metadata.empty.object_id.should_not eq(Log::Metadata.new.object_id) Log::Metadata.empty.object_id.should eq(Log::Metadata.empty.object_id) end - it "immutability" do - context = m({a: 1}) - other = context.as_h - other["a"] = m(2) + it "empty?" do + Log::Metadata.empty.should be_empty + m({} of Symbol => String).should be_empty + Log::Metadata.new.should be_empty + m({} of Symbol => String).extend({} of Symbol => String).should be_empty - other.should eq({"a" => m(2)}) - context.should eq(m({a: 1})) + m({a: 1}).should_not be_empty + m({} of Symbol => String).extend({a: 1}).should_not be_empty + m({a: 1}).extend({} of Symbol => String).should_not be_empty end - it "nested immutability" do - context = m({a: {b: 1}}) - other = context.as_h - other["a"].raw.as(Hash)["b"] = m(2) + it "extend" do + m({a: 1}).extend({b: 2}).should eq(m({a: 1, b: 2})) + m({a: 1, b: 3}).extend({b: 2}).should eq(m({a: 1, b: 2})) + m({a: 1, b: 3}).extend({b: nil}).should eq(m({a: 1, b: nil})) + end - other.should eq({"a" => m({"b" => 2})}) - context.should eq({"a" => m({"b" => 1})}) + it "extend against empty values without creating a new instance" do + c1 = m({a: 1, b: 3}) + c1.extend(NamedTuple.new).should be(c1) + c1.extend(Hash(Symbol, String).new).should be(c1) end - it "merge" do - m({a: 1}).merge(m({b: 2})).should eq(m({a: 1, b: 2})) - m({a: 1, b: 3}).merge(m({b: 2})).should eq(m({a: 1, b: 2})) - m({a: 1, b: 3}).merge(m({b: nil})).should eq(m({a: 1, b: nil})) + it "==" do + m({} of Symbol => String).should eq(m({} of Symbol => String)) + m({a: 1}).should eq(m({a: 1})) + m({a: 1, b: 2}).should eq(m({b: 2, a: 1})) + + m({a: 1}).should_not eq(m({a: 2})) + m({a: 1}).should_not eq(m({b: 1})) + + m({a: 1}).extend({b: 2}).should eq(m({b: 2}).extend({a: 1})) + m({a: 1, b: 1}).extend({b: 2}).should eq(m({b: 2}).extend({a: 1})) + m({a: 1, b: 2}).extend({b: 1}).should eq(m({a: 1, b: 1})) end - it "merge against Log::Metadata.empty without creating a new instance" do - c1 = m({a: 1, b: 3}) - c1.merge(Log::Metadata.empty).should be(c1) - Log::Metadata.empty.merge(c1).should be(c1) + it "json" do + m({a: 1}).to_json.should eq(%({"a":1})) + m({a: 1, b: 1}).extend({b: 2}).to_json.should eq(%({"b":2,"a":1})) + end + + it "defrags" do + parent = m({a: 1, b: 2}).extend({a: 2}) + md = parent.extend({a: 3}) + + md.@size.should eq(1) + md.@max_total_size.should eq(4) + md.@overriden_size.should eq(1) + md.@parent.should be(parent) + + md.should eq(m({a: 3, b: 2})) + + md.@size.should eq(2) + md.@max_total_size.should eq(2) + md.@overriden_size.should eq(1) + md.@parent.should be_nil + end + + it "[]" do + md = m({a: 1, b: 2}).extend({a: 3}) + + md[:a].should eq(3) + md[:b].should eq(2) + expect_raises(KeyError) { md[:c] } + end + + it "[]?" do + md = m({a: 1, b: 2}).extend({a: 3}) + + md[:a]?.should eq(3) + md[:b]?.should eq(2) + md[:c]?.should be_nil + end +end + +describe Log::Metadata::Value do + it "initialize" do + v({a: 1}).should eq(v({"a" => v(1)})) + v({a: 1, b: ["str", true], num: 1i64}).should eq(v({"a" => v(1), "b" => v([v("str"), v(true)]), "num" => v(1i64)})) + v({a: 1f32, b: 1f64}).should eq(v({"a" => v(1f32), "b" => v(1f64)})) + t = Time.local + v({time: t}).should eq(v({"time" => v(t)})) + v({} of String => String).should eq(v(NamedTuple.new)) end it "accessors" do - m(nil).as_nil.should be_nil + v(nil).as_nil.should be_nil + + v(1).as_i.should eq(1) - m(1).as_i.should eq(1) + v("a").as_s.should eq("a") + v(1).as_s?.should be_nil - m("a").as_s.should eq("a") - m(1).as_s?.should be_nil + v(true).as_bool.should eq(true) + v(false).as_bool.should eq(false) + v(true).as_bool?.should eq(true) + v(false).as_bool?.should eq(false) + v(nil).as_bool?.should be_nil + end - m(true).as_bool.should eq(true) - m(false).as_bool.should eq(false) - m(true).as_bool?.should eq(true) - m(false).as_bool?.should eq(false) - m(nil).as_bool?.should be_nil + it "json" do + v({a: 1}).to_json.should eq(%({"a":1})) end end diff --git a/src/log/entry.cr b/src/log/entry.cr index 185d2b62e0f2..29b91770cef3 100644 --- a/src/log/entry.cr +++ b/src/log/entry.cr @@ -43,6 +43,5 @@ struct Log::Entry getter exception : Exception? def initialize(@source : String, @severity : Severity, @message : String, @data : Log::Metadata, @exception : Exception?) - raise ArgumentError.new "Expected hash data, not #{data.raw.class}" unless data.as_h? end end diff --git a/src/log/format.cr b/src/log/format.cr index e7be76543464..baf56f6b165d 100644 --- a/src/log/format.cr +++ b/src/log/format.cr @@ -104,7 +104,7 @@ class Log # Parameters `before` and `after` can be provided to be written around # the value. def data(*, before = nil, after = nil) - if @entry.data.size > 0 + unless @entry.data.empty? @io << before << @entry.data << after end end @@ -115,7 +115,7 @@ class Log # Parameters `before` and `after` can be provided to be written around # the value. def context(*, before = nil, after = nil) - if @entry.context.size > 0 + unless @entry.context.empty? @io << before << @entry.context << after end end diff --git a/src/log/json.cr b/src/log/json.cr index 2e6f43710767..1d1bb26780ed 100644 --- a/src/log/json.cr +++ b/src/log/json.cr @@ -9,6 +9,21 @@ class Log::Metadata # log_entry.context.to_json # => "{\"user_id\":1}" # ``` def to_json(builder : JSON::Builder) : Nil - @raw.to_json builder + builder.object do + each do |(key, value)| + builder.field key.to_json_object_key do + value.to_json(builder) + end + end + end + end + + struct Value + # Returns `Log::Metadata::Value` as JSON value. + # + # NOTE: `require "log/json"` is required to opt-in to this feature. + def to_json(builder : JSON::Builder) : Nil + @raw.to_json builder + end end end diff --git a/src/log/main.cr b/src/log/main.cr index fa5ae4883cb0..9d7d8f695994 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -136,17 +136,17 @@ class Log # Log.info { %q(message with {"a" => 1, "b" => 2, "c" => 3 } context) } # ``` def set(**kwargs) - extend_fiber_context(Fiber.current, Log::Metadata.build(kwargs)) + extend_fiber_context(Fiber.current, kwargs) end # :ditto: def set(values) - extend_fiber_context(Fiber.current, Log::Metadata.build(values)) + extend_fiber_context(Fiber.current, values) end - private def extend_fiber_context(fiber : Fiber, values : Metadata) + private def extend_fiber_context(fiber : Fiber, values) context = fiber.logging_context - fiber.logging_context = @metadata = context.merge(values) + fiber.logging_context = @metadata = context.extend(values) end end diff --git a/src/log/metadata.cr b/src/log/metadata.cr index 8d7c26f645f2..b79d6dcbefec 100644 --- a/src/log/metadata.cr +++ b/src/log/metadata.cr @@ -4,83 +4,214 @@ # # NOTE: If you'd like to format the context as JSON, remember to `require "log/json"`. class Log::Metadata - Crystal.datum types: {nil: Nil, bool: Bool, i: Int32, i64: Int64, f: Float32, f64: Float64, s: String, time: Time}, hash_key_type: String, immutable: true, target_type: Log::Metadata + struct Value; end + + include Enumerable({Symbol, Log::Metadata::Value}) + alias Entry = {key: Symbol, value: Value} # Returns an empty `Log::Metadata`. # # NOTE: Since `Log::Metadata` is immutable, it's safe to share this instance. class_getter empty : Log::Metadata = Log::Metadata.new - # Creates an empty `Log::Metadata`. - def initialize - @raw = Hash(String, Metadata).new + @parent : Metadata? + # The maximum size this metadata would need. + # Initially is the parent.max_total_size + entries.size . + # When the metadata is defragmented max_total_size will be updated with size + protected getter max_total_size : Int32 + @max_total_size = uninitialized Int32 + # How many entries are potentially overriden from parent (ie: initial entries.size) + @overriden_size = uninitialized Int32 + # How many entries are stored from @first. + # Initially are @overriden_size, the one explictly overriden in entries argument. + # When the metadata is defragmented @size will be increased up to + # the actual number of entries resulting from merging the parent + @size = uninitialized Int32 + # @first needs to be the last ivar of Metadata. The entries are allocated together with self + @first = uninitialized Entry + + def self.new(parent : Metadata? = nil, entries : NamedTuple | Hash = NamedTuple.new) + data_size = instance_sizeof(self) + sizeof(Entry) * {entries.size + (parent.try(&.max_total_size) || 0) - 1, 0}.max + data = GC.malloc(data_size).as(self) + data.setup(parent, entries) + data end - # Creates `Log::Metadata` from the given *tuple*. - def initialize(tuple : NamedTuple) - @raw = raw = Hash(String, Metadata).new - tuple.each do |key, value| - raw[key.to_s] = to_metadata(value) + protected def setup(@parent : Metadata?, entries : NamedTuple | Hash) + @size = @overriden_size = entries.size + @max_total_size = @size + (@parent.try(&.max_total_size) || 0) + ptr_entries = pointerof(@first) + + if entries.is_a?(NamedTuple) + entries.each_with_index do |key, value, i| + ptr_entries[i] = {key: key, value: Value.to_metadata_value(value)} + end + else + entries.each_with_index do |(key, value), i| + ptr_entries[i] = {key: key, value: Value.to_metadata_value(value)} + end end end - # Creates `Log::Metadata` from the given *hash*. - def initialize(hash : Hash(String, V)) forall V - @raw = raw = Hash(String, Metadata).new - hash.each do |key, value| - raw[key] = to_metadata(value) + # Returns a `Metadata` with the information of the argument. + # Used to handle `Log::Context#set` and `Log#Emitter.emit` overloads. + def self.build(value : NamedTuple | Hash) + return @@empty if value.empty? + Metadata.new(nil, value) + end + + # :ditto: + def self.build(value : Metadata) + value + end + + # Returns a `Log::Metadata` with all the entries of *self* + # and *other*. If a key is defined in both, the values in *other* are used. + def extend(other : NamedTuple | Hash) : Metadata + return Metadata.build(other) if self.empty? + return self if other.empty? + + Metadata.new(self, other) + end + + def empty? + parent = @parent + + @size == 0 && (parent.nil? || parent.empty?) + end + + # Removes the reference to *parent*. Flattening the entries from it into *self*. + # *self* was originally allocated with enough entries to perform this action. + # + # If multiple threads execute defrag concurrently, the entries + # will be recomputed, but the result should be the same. + # + # * @parent.nil? signals if the defrag is needed/done + # * The values of @overriden_size, pointerof(@first) are never changed + # * @parent is set at the very end of the method + protected def defrag + parent = @parent + return if parent.nil? + + total_size = @overriden_size + ptr_entries = pointerof(@first) + next_free_entry = ptr_entries + @overriden_size + + parent.each do |(key, value)| + overriden = false + @overriden_size.times do |i| + if ptr_entries[i][:key] == key + overriden = true + break + end + end + + unless overriden + next_free_entry.value = {key: key, value: value} + next_free_entry += 1 + total_size += 1 + end end + + @size = total_size + @max_total_size = total_size + @parent = nil end - # Creates `Log::Metadata` from the given *hash*. - def initialize(hash : Hash(Symbol, V)) forall V - @raw = raw = Hash(String, Metadata).new - hash.each do |key, value| - raw[key.to_s] = to_metadata(value) + def each(& : {Symbol, Value} ->) + defrag + ptr_entries = pointerof(@first) + + @size.times do |i| + entry = ptr_entries[i] + yield({entry[:key], entry[:value]}) end end - # :nodoc: - def initialize(ary : Array) - @raw = ary.map { |e| to_metadata(e) } + def [](key : Symbol) : Value + fetch(key) { raise KeyError.new "Missing metadata key: #{key.inspect}" } end - # Returns a new `Log::Metadata` with the keys and values of this context and *other* combined. - # A value in *other* takes precedence over the one in this context. - def merge(other : Metadata) - return other if self.object_id == @@empty.object_id - return self if other.object_id == @@empty.object_id - Metadata.new(self.as_h.merge(other.as_h).clone) + def []?(key : Symbol) : Value? + fetch(key) { nil } end - private def to_metadata(value) - value.is_a?(Metadata) ? value : Metadata.new(value) + def fetch(key) + entry = find_entry(key) + entry ? entry[:value] : yield key end - # Returns a `Metadata` with the information of the argument. - # Used to handle `Log::Context#set` and `Log#Emitter.emit` overloads. - def self.build(value : Nil) - Metadata.empty + protected def find_entry(key) : Entry? + # checking the @parent before @size ensures that if other + # thread is doing defrag, the results will be consistent + # without locking. + + parent = @parent + + ptr_entries = pointerof(@first) + @size.times do |i| + return ptr_entries[i] if ptr_entries[i][:key] == key + end + + return parent.find_entry(key) if parent + + nil end - # :ditto: - def self.build(value : NamedTuple) - Metadata.new(value) + def ==(other : Metadata) + self_kv = self.to_a + other_kv = other.to_a + + return false if self_kv.size != other_kv.size + + # sort kv tuples by key + self_kv.sort_by!(&.[0]) + other_kv.sort_by!(&.[0]) + + self_kv.each_with_index do |(key, value), i| + return false unless key == other_kv[i][0] && value == other_kv[i][1] + end + + true end - # :ditto: - def self.build(value : Hash(String, V)) forall V - Metadata.new(value) + # :nodoc: + def ==(other) + false end - # :ditto: - def self.build(value : Hash(Symbol, V)) forall V - Metadata.new(value) + def to_s(io : IO) : Nil + found_one = false + each do |(key, value)| + io << ", " if found_one + io << key + io << ": " + value.inspect(io) + found_one = true + end end - # :ditto: - def self.build(value : Metadata) - value + struct Value + Crystal.datum types: {nil: Nil, bool: Bool, i: Int32, i64: Int64, f: Float32, f64: Float64, s: String, time: Time}, hash_key_type: String, immutable: false, target_type: Log::Metadata::Value + + # Creates `Log::Metadata` from the given *values*. + # All keys are converted to `String` + def initialize(hash : NamedTuple | Hash) + @raw = raw = Hash(String, Value).new + hash.each do |key, value| + raw[key.to_s] = Value.to_metadata_value(value) + end + end + + # :nodoc: + def initialize(ary : Array) + @raw = ary.map { |e| Value.to_metadata_value(e) } + end + + # :nodoc: + def self.to_metadata_value(value) + value.is_a?(Value) ? value : Value.new(value) + end end end @@ -90,7 +221,6 @@ class Fiber # :nodoc: def logging_context=(value : Log::Metadata) - raise ArgumentError.new "Expected hash context, not #{value.raw.class}" unless value.as_h? @logging_context = value end end From 6c5ce5262e2cfd22d22d780779eb5e5c9ba74ef6 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 19 May 2020 12:38:35 +0200 Subject: [PATCH 049/263] Followup of #9134. Missing swap of arguments (#9318) --- src/crystal/system/win32/process.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 27853dee7024..537e053fed16 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -152,7 +152,7 @@ struct Crystal::System::Process end private def self.args_to_string(args, io : IO) - args.join(' ', io) do |arg| + args.join(io, ' ') do |arg| quotes = arg.empty? || arg.includes?(' ') || arg.includes?('\t') io << '"' if quotes From c10dbc845aa7c997e750e07e3ca6d29327cca32d Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 19 May 2020 22:19:07 +0900 Subject: [PATCH 050/263] refactor: remove heredoc related code from StringLiteral formatter (#9231) Because all heredoc are StringInterpolation even if it has no interpolation. --- src/compiler/crystal/tools/formatter.cr | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 0e1abd5bf71e..6ca82cac6454 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -154,7 +154,7 @@ module Crystal def end_visit_any(node) case node - when StringLiteral, StringInterpolation + when StringInterpolation # Nothing else @last_is_heredoc = false @@ -447,8 +447,6 @@ module Crystal end def visit(node : StringLiteral) - @last_is_heredoc = false - column = @column if @token.type == :__FILE__ || @token.type == :__DIR__ @@ -459,12 +457,6 @@ module Crystal check :DELIMITER_START is_regex = @token.delimiter_state.kind == :regex - is_heredoc = @token.delimiter_state.kind == :heredoc - @last_is_heredoc = is_heredoc - - indent_difference = @token.column_number - (@column + 1) - heredoc_line = @line - heredoc_end = @line write @token.raw next_string_token @@ -488,7 +480,6 @@ module Crystal write "}" next_string_token when :DELIMITER_END - heredoc_end = @line break else raise "Bug: unexpected token: #{@token.type}" @@ -498,16 +489,6 @@ module Crystal write @token.raw format_regex_modifiers if is_regex - if is_heredoc - if indent_difference > 0 - @heredoc_fixes << HeredocFix.new(heredoc_line, @line, indent_difference) - end - (heredoc_line...heredoc_end).each do |line| - @no_rstrip_lines.add line - end - write_line - end - if space_slash_newline? old_indent = @indent @indent = column if @string_continuation == 0 From 28821fc3a1dfa220acc77b4919c2842cb84609ff Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 19 May 2020 10:20:07 -0300 Subject: [PATCH 051/263] Keep Log::Severity::Warning as deprecated definition (#9316) --- src/log/entry.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/log/entry.cr b/src/log/entry.cr index 29b91770cef3..817706b4d310 100644 --- a/src/log/entry.cr +++ b/src/log/entry.cr @@ -10,6 +10,8 @@ enum Log::Severity Notice # Used for conditions that can potentially cause application oddities, but that can be automatically recovered. Warn + # DEPRECATED: Use `Warn`. + Warning = Warn # Used for any error that is fatal to the operation, but not to the service or application. Error # Used for any error that is forcing a shutdown of the service or application From ea80f4c81bf23eab4118796d98b1f6405e8de377 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Tue, 19 May 2020 17:10:06 -0300 Subject: [PATCH 052/263] Always include `lib` directory in the CRYSTAL_PATH (#9315) --- .../crystal_path/crystal_path_spec.cr | 15 ++++++++++++++ spec/std/log/env_config_spec.cr | 18 +---------------- spec/support/env.cr | 16 +++++++++++++++ src/compiler/crystal/crystal_path.cr | 20 +++++++++++++------ 4 files changed, 46 insertions(+), 23 deletions(-) create mode 100644 spec/support/env.cr diff --git a/spec/compiler/crystal_path/crystal_path_spec.cr b/spec/compiler/crystal_path/crystal_path_spec.cr index 8cbe0ab8376d..6f2398f2743e 100644 --- a/spec/compiler/crystal_path/crystal_path_spec.cr +++ b/spec/compiler/crystal_path/crystal_path_spec.cr @@ -1,4 +1,5 @@ require "../../spec_helper" +require "../../support/env" private def assert_finds(search, results, relative_to = nil, path = __DIR__, file = __FILE__, line = __LINE__) it "finds #{search.inspect}", file, line do @@ -114,4 +115,18 @@ describe Crystal::CrystalPath do ex.message.not_nil!.should_not contain "If you're trying to require a shard" end + + it "includes 'lib' by default" do + with_env("CRYSTAL_PATH": nil) do + crystal_path = Crystal::CrystalPath.new + crystal_path.entries[0].should eq("lib") + end + end + + it "overrides path with environment variable" do + with_env("CRYSTAL_PATH": "foo:bar") do + crystal_path = Crystal::CrystalPath.new + crystal_path.entries.should eq(%w(foo bar)) + end + end end diff --git a/spec/std/log/env_config_spec.cr b/spec/std/log/env_config_spec.cr index a98167fea23a..7bfdfb0f239a 100644 --- a/spec/std/log/env_config_spec.cr +++ b/spec/std/log/env_config_spec.cr @@ -1,28 +1,12 @@ require "spec" require "log" require "log/spec" +require "../../support/env" private def s(value : Log::Severity) value end -private def with_env(**values) - old_values = {} of String => String? - begin - values.each do |key, value| - key = key.to_s - old_values[key] = ENV[key]? - ENV[key] = value - end - - yield - ensure - old_values.each do |key, old_value| - ENV[key] = old_value - end - end -end - describe "Log.setup_from_env" do after_all do # Setup logging in specs (again) since these specs perform Log.setup diff --git a/spec/support/env.cr b/spec/support/env.cr new file mode 100644 index 000000000000..9e99f903c126 --- /dev/null +++ b/spec/support/env.cr @@ -0,0 +1,16 @@ +def with_env(**values) + old_values = {} of String => String? + begin + values.each do |key, value| + key = key.to_s + old_values[key] = ENV[key]? + ENV[key] = value + end + + yield + ensure + old_values.each do |key, old_value| + ENV[key] = old_value + end + end +end diff --git a/src/compiler/crystal/crystal_path.cr b/src/compiler/crystal/crystal_path.cr index 619cf1a4d343..9e3190618370 100644 --- a/src/compiler/crystal/crystal_path.cr +++ b/src/compiler/crystal/crystal_path.cr @@ -6,24 +6,32 @@ module Crystal class Error < LocationlessException end + private DEFAULT_LIB_PATH = "lib" + def self.default_path - ENV["CRYSTAL_PATH"]? || Crystal::Config.path + ENV["CRYSTAL_PATH"]? || begin + if Crystal::Config.path.split(Process::PATH_DELIMITER).includes?(DEFAULT_LIB_PATH) + Crystal::Config.path + else + {DEFAULT_LIB_PATH, Crystal::Config.path}.join(Process::PATH_DELIMITER) + end + end end - @crystal_path : Array(String) + property entries : Array(String) def initialize(path = CrystalPath.default_path, codegen_target = Config.host_target) - @crystal_path = path.split(Process::PATH_DELIMITER).reject &.empty? + @entries = path.split(Process::PATH_DELIMITER).reject &.empty? add_target_path(codegen_target) end private def add_target_path(codegen_target) target = "#{codegen_target.architecture}-#{codegen_target.os_name}" - @crystal_path.each do |path| + @entries.each do |path| path = File.join(path, "lib_c", target) if Dir.exists?(path) - @crystal_path << path unless @crystal_path.includes?(path) + @entries << path unless @entries.includes?(path) return end end @@ -144,7 +152,7 @@ module Crystal end private def find_in_crystal_path(filename) - @crystal_path.each do |path| + @entries.each do |path| required = find_in_path_relative_to_dir(filename, path) return required if required end From 2f90b6f50557e78948336310b97480dbefe9dd8a Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 19 May 2020 20:39:15 -0300 Subject: [PATCH 053/263] Range: use `while true` when possible in `each` and `reverse_each` --- spec/std/range_spec.cr | 20 ++++++++++++++++++++ src/range.cr | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/spec/std/range_spec.cr b/spec/std/range_spec.cr index 75fae97a2665..ffa28a209db4 100644 --- a/spec/std/range_spec.cr +++ b/spec/std/range_spec.cr @@ -28,6 +28,18 @@ struct RangeSpecIntWrapper end end +private def range_endless_each + (2..).each do |x| + return x + end +end + +private def range_beginless_reverse_each + (..2).reverse_each do |x| + return x + end +end + describe "Range" do it "initialized with new method" do Range.new(1, 10).should eq(1..10) @@ -215,6 +227,14 @@ describe "Range" do range.each { } end end + + it "doesn't have Nil as a type for endless each" do + typeof(range_endless_each).should eq(Int32) + end + + it "doesn't have Nil as a type for beginless each" do + typeof(range_beginless_reverse_each).should eq(Int32) + end end describe "reverse_each" do diff --git a/src/range.cr b/src/range.cr index 5f39162dfdef..b4497b17ae92 100644 --- a/src/range.cr +++ b/src/range.cr @@ -112,12 +112,22 @@ struct Range(B, E) raise ArgumentError.new("Can't each beginless range") end - end_value = @end - while end_value.nil? || current < end_value - yield current - current = current.succ - end - yield current if !@exclusive && current == end_value + # TODO: This typeof and the macro interpolations are a workaround until #9324 is fixed. + typeof(yield current) + + {% if E == Nil %} + while true + {{ "yield current".id }} + current = current.succ + end + {% else %} + end_value = @end + while end_value.nil? || current < end_value + {{ "yield current".id }} + current = current.succ + end + {{ "yield current".id }} if !@exclusive && current == end_value + {% end %} end # Returns an `Iterator` over the elements of this range. @@ -159,10 +169,19 @@ struct Range(B, E) yield end_value if !@exclusive && (begin_value.nil? || !(end_value < begin_value)) current = end_value - while begin_value.nil? || begin_value < current - current = current.pred - yield current - end + # TODO: The macro interpolations are a workaround until #9324 is fixed. + + {% if B == Nil %} + while true + current = current.pred + {{ "yield current".id }} + end + {% else %} + while begin_value.nil? || begin_value < current + current = current.pred + {{ "yield current".id }} + end + {% end %} end # Returns a reverse `Iterator` over the elements of this range. From c68b49f0452d83abec679b8392a598966200b717 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 21 May 2020 09:22:52 -0300 Subject: [PATCH 054/263] Compiler: fix yield computation inside macro code (#9324) --- spec/compiler/parser/parser_spec.cr | 6 ++++ spec/compiler/semantic/abstract_def_spec.cr | 38 +++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 2 +- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 01487730ac10..5111e83b1502 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1995,6 +1995,12 @@ module Crystal name_location.line_number.should eq(1) name_location.column_number.should eq(12) end + + it "doesn't override yield with macro yield" do + parser = Parser.new("def foo; yield 1; {% begin %} yield 1 {% end %}; end") + a_def = parser.parse.as(Def) + a_def.yields.should eq(1) + end end end end diff --git a/spec/compiler/semantic/abstract_def_spec.cr b/spec/compiler/semantic/abstract_def_spec.cr index 1f95e94da679..f28851e72210 100644 --- a/spec/compiler/semantic/abstract_def_spec.cr +++ b/spec/compiler/semantic/abstract_def_spec.cr @@ -636,4 +636,42 @@ describe "Semantic: abstract def" do end )) end + + it "can implement even if yield comes later in macro code" do + semantic(%( + module Moo + abstract def each(& : Int32 -> _) + end + + class Foo + include Moo + + def each + yield 1 + + {% if true %} + yield 2 + {% end %} + end + end + )) + end + + it "can implement by block signature even if yield comes later in macro code" do + semantic(%( + module Moo + abstract def each(& : Int32 -> _) + end + + class Foo + include Moo + + def each(& : Int32 -> _) + {% if true %} + yield 2 + {% end %} + end + end + )) + end end diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 1795cbc07ba9..6abba79cdc86 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3042,7 +3042,7 @@ module Crystal next_macro_token macro_state, skip_whitespace macro_state = @token.macro_state if macro_state.yields - @yields = 0 + @yields ||= 0 end skip_whitespace = false From c13fd9c05d6d8dbcf785439c30d63e8383dd42f4 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 21 May 2020 10:35:13 -0300 Subject: [PATCH 055/263] Docs: correctly attach docs before annotations to following types (#9332) --- spec/compiler/semantic/doc_spec.cr | 88 +++++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 23 ++--- 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/spec/compiler/semantic/doc_spec.cr b/spec/compiler/semantic/doc_spec.cr index d8006f0af7fc..928bc2ac47cd 100644 --- a/spec/compiler/semantic/doc_spec.cr +++ b/spec/compiler/semantic/doc_spec.cr @@ -399,4 +399,92 @@ describe "Semantic: doc" do type = program.types["MyClass1"] type.doc.should eq("Some description") end + + context "doc before annotation" do + it "attached to struct/class" do + result = semantic %( + # Some description + @[Packed] + struct Foo + end + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to module" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + module Foo + end + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to enum" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + enum Foo + One + end + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to constant" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + Foo = 1 + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to alias" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + alias Foo = Int32 + ), wants_doc: true + program = result.program + type = program.types["Foo"] + type.doc.should eq("Some description") + end + + it "attached to def" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + def foo + end + ), wants_doc: true + program = result.program + a_def = program.lookup_defs("foo").first + a_def.doc.should eq("Some description") + end + end end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 7fe825cce656..94105e058021 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -192,7 +192,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor type.add_annotation(annotation_type, ann) end - attach_doc type, node + attach_doc type, node, annotations pushing_type(type) do run_hooks(hook_type(superclass), type, :inherited, node) if created_new_type @@ -235,7 +235,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor node.resolved_type = type - attach_doc type, node + attach_doc type, node, annotations process_annotations(annotations) do |annotation_type, ann| type.add_annotation(annotation_type, ann) @@ -262,7 +262,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor scope.types[name] = type end - attach_doc type, node + attach_doc type, node, annotations: nil false end @@ -270,6 +270,8 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor def visit(node : Alias) check_outside_exp node, "declare alias" + annotations = read_annotations + scope, name, existing_type = lookup_type_def(node) if existing_type @@ -281,7 +283,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor end alias_type = AliasType.new(@program, scope, name, node.value) - attach_doc alias_type, node + attach_doc alias_type, node, annotations scope.types[name] = alias_type alias_type.private = true if node.visibility.private? @@ -530,8 +532,6 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor annotations = read_annotations - annotations_doc = annotations_doc(annotations) - scope, name, enum_type = lookup_type_def(node) if enum_type @@ -561,9 +561,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor end node.resolved_type = enum_type - attach_doc enum_type, node - - enum_type.doc ||= annotations_doc(annotations) + attach_doc enum_type, node, annotations pushing_type(enum_type) do counter = enum_type.flags? ? 1 : 0 @@ -741,6 +739,8 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor check_outside_exp node, "declare constant" @exp_nest += 1 + annotations = read_annotations + scope, name = lookup_type_def_name(target) if current_type.is_a?(Program) scope = program.check_private(target) || scope @@ -755,7 +755,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor const.private = true if target.visibility.private? check_ditto node, node.location - attach_doc const, node + attach_doc const, node, annotations scope.types[name] = const @@ -1047,9 +1047,10 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor call_convention end - def attach_doc(type, node) + def attach_doc(type, node, annotations) if @program.wants_doc? type.doc ||= node.doc + type.doc ||= annotations_doc(annotations) if annotations end if node_location = node.location From b3ebb1cde9c08eb15d3401775b7d5b8811335d7f Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Sat, 23 May 2020 11:13:32 -0300 Subject: [PATCH 056/263] Allow annotations and ditto in macro (#9341) Enable doc generator tool to keep same treatment for macros as for method annotations. Refactor AST & Types annotation related methods to Annotatable module. --- spec/compiler/semantic/doc_spec.cr | 30 ++++++++++++ src/compiler/crystal/annotatable.cr | 23 ++++++++++ src/compiler/crystal/semantic/ast.cr | 46 +++---------------- .../crystal/semantic/top_level_visitor.cr | 9 +++- src/compiler/crystal/tools/doc/macro.cr | 3 +- src/compiler/crystal/types.cr | 19 +------- 6 files changed, 70 insertions(+), 60 deletions(-) create mode 100644 src/compiler/crystal/annotatable.cr diff --git a/spec/compiler/semantic/doc_spec.cr b/spec/compiler/semantic/doc_spec.cr index 928bc2ac47cd..9c985ffef953 100644 --- a/spec/compiler/semantic/doc_spec.cr +++ b/spec/compiler/semantic/doc_spec.cr @@ -148,6 +148,21 @@ describe "Semantic: doc" do bar.doc.should eq("Hello") end + it "stores doc for macro when using ditto" do + result = semantic %( + # Hello + macro bar + end + + # :ditto: + macro bar2 + end + ), wants_doc: true + program = result.program + bar2 = program.lookup_macros("bar2").as(Array(Macro)).first + bar2.doc.should eq("Hello") + end + {% for def_type in %w[def macro].map &.id %} it "overwrites doc for {{def_type}} when redefining" do result = semantic %( @@ -486,5 +501,20 @@ describe "Semantic: doc" do a_def = program.lookup_defs("foo").first a_def.doc.should eq("Some description") end + + it "attached to macro" do + result = semantic %( + annotation Ann + end + + # Some description + @[Ann] + macro foo + end + ), wants_doc: true + program = result.program + type = program.lookup_macros("foo").as(Array(Macro)).first + type.doc.should eq("Some description") + end end end diff --git a/src/compiler/crystal/annotatable.cr b/src/compiler/crystal/annotatable.cr new file mode 100644 index 000000000000..0459a912c2b3 --- /dev/null +++ b/src/compiler/crystal/annotatable.cr @@ -0,0 +1,23 @@ +module Crystal + module Annotatable + # Annotations on this instance + property annotations : Hash(AnnotationType, Array(Annotation))? + + # Adds an annotation with the given type and value + def add_annotation(annotation_type : AnnotationType, value : Annotation) + annotations = @annotations ||= {} of AnnotationType => Array(Annotation) + annotations[annotation_type] ||= [] of Annotation + annotations[annotation_type] << value + end + + # Returns the last defined annotation with the given type, if any, or `nil` otherwise + def annotation(annotation_type) : Annotation? + @annotations.try &.[annotation_type]?.try &.last? + end + + # Returns all annotations with the given type, if any, or `nil` otherwise + def annotations(annotation_type) : Array(Annotation)? + @annotations.try &.[annotation_type]? + end + end +end diff --git a/src/compiler/crystal/semantic/ast.cr b/src/compiler/crystal/semantic/ast.cr index 19c84a3a4485..e2ba5c40dec5 100644 --- a/src/compiler/crystal/semantic/ast.cr +++ b/src/compiler/crystal/semantic/ast.cr @@ -119,6 +119,8 @@ module Crystal end class Def + include Annotatable + property! owner : Type property! original_owner : Type property vars : MetaVars? @@ -147,9 +149,6 @@ module Crystal # Is this a `new` method that was expanded from an initialize? property? new = false - # Annotations on this def - property annotations : Hash(AnnotationType, Array(Annotation))? - @macro_owner : Type? def macro_owner=(@macro_owner) @@ -179,23 +178,6 @@ module Crystal end end - # Adds an annotation with the given type and value - def add_annotation(annotation_type : AnnotationType, value : Annotation) - annotations = @annotations ||= {} of AnnotationType => Array(Annotation) - annotations[annotation_type] ||= [] of Annotation - annotations[annotation_type] << value - end - - # Returns the last defined annotation with the given type, if any, or `nil` otherwise - def annotation(annotation_type) : Annotation? - @annotations.try &.[annotation_type]?.try &.last? - end - - # Returns all annotations with the given type, if any, or `nil` otherwise - def annotations(annotation_type) : Array(Annotation)? - @annotations.try &.[annotation_type]? - end - # Returns the minimum and maximum number of arguments that must # be passed to this method. def min_max_args_sizes @@ -250,6 +232,8 @@ module Crystal end class Macro + include Annotatable + # Yields `arg, arg_index, object, object_index` corresponding # to arguments matching the given objects, taking into account this # macro's splat index. @@ -494,6 +478,8 @@ module Crystal # A variable belonging to a type: a global, # class or instance variable (globals belong to the program). class MetaTypeVar < Var + include Annotatable + property nil_reason : NilReason? # The owner of this variable, useful for showing good @@ -514,9 +500,6 @@ module Crystal # Is this variable "unsafe" (no need to check if it was initialized)? property? uninitialized = false - # Annotations of this instance var - property annotations : Hash(AnnotationType, Array(Annotation))? - def kind case name[0] when '@' @@ -533,23 +516,6 @@ module Crystal def global? kind == :global end - - # Adds an annotation with the given type and value - def add_annotation(annotation_type : AnnotationType, value : Annotation) - annotations = @annotations ||= {} of AnnotationType => Array(Annotation) - annotations[annotation_type] ||= [] of Annotation - annotations[annotation_type] << value - end - - # Returns the last defined annotation with the given type, if any, or `nil` otherwise - def annotation(annotation_type) : Annotation? - @annotations.try &.[annotation_type]?.try &.last? - end - - # Returns all annotations with the given type, if any, or `nil` otherwise - def annotations(annotation_type) : Array(Annotation)? - @annotations.try &.[annotation_type]? - end end class ClassVar diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 94105e058021..4cd9660c7f80 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -296,6 +296,13 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor def visit(node : Macro) check_outside_exp node, "declare macro" + annotations = read_annotations + process_annotations(annotations) do |annotation_type, ann| + node.add_annotation(annotation_type, ann) + end + node.doc ||= annotations_doc(annotations) + check_ditto node, node.location + node.set_type @program.nil if node.name == "finished" @@ -1058,7 +1065,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor end end - def check_ditto(node : Def | Assign | FunDef | Const, location : Location?) : Nil + def check_ditto(node : Def | Assign | FunDef | Const | Macro, location : Location?) : Nil return if !@program.wants_doc? stripped_doc = node.doc.try &.strip if stripped_doc == ":ditto:" diff --git a/src/compiler/crystal/tools/doc/macro.cr b/src/compiler/crystal/tools/doc/macro.cr index f84eafd90db6..e7072d6e611d 100644 --- a/src/compiler/crystal/tools/doc/macro.cr +++ b/src/compiler/crystal/tools/doc/macro.cr @@ -145,7 +145,6 @@ class Crystal::Doc::Macro end def annotations(annotation_type) - # macros does not support annotations - nil + @macro.annotations(annotation_type) end end diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 3d3038f86133..4c985b32bdab 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -3,6 +3,8 @@ require "./syntax/ast" module Crystal # Abstract base class of all types abstract class Type + include Annotatable + # Returns the program where this type belongs. getter program @@ -710,23 +712,6 @@ module Crystal end end - # Adds an annotation with the given type and value - def add_annotation(annotation_type : AnnotationType, value : Annotation) - annotations = @annotations ||= {} of AnnotationType => Array(Annotation) - annotations[annotation_type] ||= [] of Annotation - annotations[annotation_type] << value - end - - # Returns the last defined annotation with the given type, if any, or `nil` otherwise - def annotation(annotation_type) : Annotation? - @annotations.try &.[annotation_type]?.try &.last? - end - - # Returns all annotations with the given type, if any, or `nil` otherwise - def annotations(annotation_type) : Array(Annotation)? - @annotations.try &.[annotation_type]? - end - def get_instance_var_initializer(name) nil end From d91a6386f8626bebba146614773d5cb6bc3c33f7 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 25 May 2020 14:18:38 +0200 Subject: [PATCH 057/263] Win CI: add libxml2 and enable more specs (#9346) * Win CI: Add libxml2 and enable XML specs * Win CI: Enable specs that were already working --- .github/workflows/win.yml | 14 ++++++++ spec/std/json/any_spec.cr | 6 ++-- spec/std/json/serializable_spec.cr | 50 +++++++++++++-------------- spec/std/spec/junit_formatter_spec.cr | 14 ++++---- spec/std/string_spec.cr | 2 +- spec/win32_std_spec.cr | 10 +++--- 6 files changed, 52 insertions(+), 44 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index e0e950ad7c51..c518e1885a1c 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -99,6 +99,19 @@ jobs: run: | cmake . -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded cmake --build . --config Release + - name: Download libxml2 + if: steps.cache-libs.outputs.cache-hit != 'true' + uses: actions/checkout@v2 + with: + repository: GNOME/libxml2 + ref: a230b728f1289dd24c1666856ac4fb55579c6dfb # master @ 2020-05-04 + path: libxml2 + - name: Build libxml2 + if: steps.cache-libs.outputs.cache-hit != 'true' + working-directory: ./libxml2 + run: | + cmake . -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_TESTS=OFF -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + cmake --build . --config Release - name: Gather libraries if: steps.cache-libs.outputs.cache-hit != 'true' run: | @@ -107,6 +120,7 @@ jobs: mv bdwgc/Release/gc.lib libs/ mv zlib/Release/zlibstatic.lib libs/z.lib mv libyaml/Release/yaml.lib libs/ + mv libxml2/Release/libxml2s.lib libs/xml2.lib - name: Cache LLVM id: cache-llvm diff --git a/spec/std/json/any_spec.cr b/spec/std/json/any_spec.cr index 9178f44a5403..9758bbad0022 100644 --- a/spec/std/json/any_spec.cr +++ b/spec/std/json/any_spec.cr @@ -1,8 +1,6 @@ require "../spec_helper" require "json" -{% unless flag?(:win32) %} - require "yaml" -{% end %} +require "yaml" describe JSON::Any do describe "casts" do @@ -172,7 +170,7 @@ describe JSON::Any do any2.as_a[0].as_a.should_not be(any.as_a[0].as_a) end - pending_win32 "#to_yaml" do + it "#to_yaml" do any = JSON.parse <<-JSON { "foo": "bar", diff --git a/spec/std/json/serializable_spec.cr b/spec/std/json/serializable_spec.cr index 65ad32f1cbff..9349c16672f5 100644 --- a/spec/std/json/serializable_spec.cr +++ b/spec/std/json/serializable_spec.cr @@ -1,7 +1,7 @@ require "../spec_helper" require "json" +require "yaml" {% unless flag?(:win32) %} - require "yaml" require "big" require "big/json" {% end %} @@ -301,38 +301,36 @@ class JSONAttrModuleTest2 < JSONAttrModuleTest end end -{% unless flag?(:win32) %} - struct JSONAttrPersonWithYAML - include JSON::Serializable - include YAML::Serializable +struct JSONAttrPersonWithYAML + include JSON::Serializable + include YAML::Serializable - property name : String - property age : Int32? + property name : String + property age : Int32? - def initialize(@name : String, @age : Int32? = nil) - end + def initialize(@name : String, @age : Int32? = nil) end +end - struct JSONAttrPersonWithYAMLInitializeHook - include JSON::Serializable - include YAML::Serializable +struct JSONAttrPersonWithYAMLInitializeHook + include JSON::Serializable + include YAML::Serializable - property name : String - property age : Int32? + property name : String + property age : Int32? - def initialize(@name : String, @age : Int32? = nil) - after_initialize - end + def initialize(@name : String, @age : Int32? = nil) + after_initialize + end - @[JSON::Field(ignore: true)] - @[YAML::Field(ignore: true)] - property msg : String? + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + property msg : String? - def after_initialize - @msg = "Hello " + name - end + def after_initialize + @msg = "Hello " + name end -{% end %} +end abstract class JSONShape include JSON::Serializable @@ -816,7 +814,7 @@ describe "JSON mapping" do it { JSONAttrModuleTest2.from_json(%({"bar": 30, "moo": 40})).to_tuple.should eq({40, 15, 30}) } end - pending_win32 "works together with yaml" do + it "works together with yaml" do person = JSONAttrPersonWithYAML.new("Vasya", 30) person.to_json.should eq "{\"name\":\"Vasya\",\"age\":30}" person.to_yaml.should eq "---\nname: Vasya\nage: 30\n" @@ -825,7 +823,7 @@ describe "JSON mapping" do JSONAttrPersonWithYAML.from_yaml(person.to_yaml).should eq person end - pending_win32 "yaml and json with after_initialize hook" do + it "yaml and json with after_initialize hook" do person = JSONAttrPersonWithYAMLInitializeHook.new("Vasya", 30) person.msg.should eq "Hello Vasya" diff --git a/spec/std/spec/junit_formatter_spec.cr b/spec/std/spec/junit_formatter_spec.cr index 734255438d12..d2f5cd9fcc75 100644 --- a/spec/std/spec/junit_formatter_spec.cr +++ b/spec/std/spec/junit_formatter_spec.cr @@ -1,7 +1,5 @@ require "../spec_helper" -{% unless flag?(:win32) %} - require "xml" -{% end %} +require "xml" class Spec::JUnitFormatter property started_at @@ -103,7 +101,7 @@ describe "JUnit Formatter" do output.should eq(expected) end - pending_win32 "encodes class names from the relative file path" do + it "encodes class names from the relative file path" do output = build_report do |f| f.report Spec::Result.new(:success, "foo", __FILE__, __LINE__, nil, nil) end @@ -112,7 +110,7 @@ describe "JUnit Formatter" do classname.should eq("spec.std.spec.junit_formatter_spec") end - pending_win32 "outputs timestamp according to RFC 3339" do + it "outputs timestamp according to RFC 3339" do now = Time.utc output = build_report(timestamp: now) do |f| @@ -123,7 +121,7 @@ describe "JUnit Formatter" do classname.should eq(now.to_rfc3339) end - pending_win32 "escapes spec names" do + it "escapes spec names" do output = build_report do |f| f.report Spec::Result.new(:success, %(complicated " '&ame), __FILE__, __LINE__, nil, nil) f.report Spec::Result.new(:success, %(ctrl characters follow - \r\n), __FILE__, __LINE__, nil, nil) @@ -136,7 +134,7 @@ describe "JUnit Formatter" do name.should eq(%(ctrl characters follow - \\r\\n)) end - pending_win32 "report failure stacktrace if present" do + it "report failure stacktrace if present" do cause = exception_with_backtrace("Something happened") output = build_report do |f| @@ -151,7 +149,7 @@ describe "JUnit Formatter" do backtrace.should eq(cause.backtrace.join('\n')) end - pending_win32 "report error stacktrace if present" do + it "report error stacktrace if present" do cause = exception_with_backtrace("Something happened") output = build_report do |f| diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 44af6621348b..121ab3c9d96c 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -2447,7 +2447,7 @@ describe "String" do end end - pending_win32 "applies formatting to %<...> placeholder" do + it "applies formatting to %<...> placeholder" do res = "change %.2f" % {"this" => 23.456} res.should eq "change 23.46" diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index f82d89155c29..dbf3f13c8a08 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -222,11 +222,11 @@ require "./std/uri/punycode_spec.cr" require "./std/uri_spec.cr" require "./std/uuid_spec.cr" require "./std/weak_ref_spec.cr" -# require "./std/xml/builder_spec.cr" (failed linking) -# require "./std/xml/html_spec.cr" (failed linking) -# require "./std/xml/reader_spec.cr" (failed linking) -# require "./std/xml/xml_spec.cr" (failed linking) -# require "./std/xml/xpath_spec.cr" (failed linking) +require "./std/xml/builder_spec.cr" +require "./std/xml/html_spec.cr" +require "./std/xml/reader_spec.cr" +require "./std/xml/xml_spec.cr" +require "./std/xml/xpath_spec.cr" require "./std/yaml/any_spec.cr" require "./std/yaml/builder_spec.cr" require "./std/yaml/mapping_spec.cr" From 15e6e28ad9155c12b4be6f03a94aff5c0d85ddb0 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Wed, 27 May 2020 03:01:38 +0900 Subject: [PATCH 058/263] Parser: fix to parse `{[] of Foo, self.foo}` correctly (#9329) * Parser: fix to parse `{[] of Foo, self.foo}` correctly * Add missing delimiter `=` --- spec/compiler/parser/parser_spec.cr | 8 +++ src/compiler/crystal/syntax/parser.cr | 94 ++++++++++++++++++--------- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 5111e83b1502..b79f32aab62c 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1764,6 +1764,14 @@ module Crystal assert_syntax_error %(def foo("bar");end), "expected argument internal name" + it_parses "{[] of Foo, Bar::Baz.new}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(Path.new(%w[Bar Baz]), "new")] of ASTNode) + it_parses "{[] of Foo, ::Bar::Baz.new}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(Path.new(%w[Bar Baz], global: true), "new")] of ASTNode) + it_parses "{[] of Foo, Bar::Baz + 2}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(Path.new(%w[Bar Baz]), "+", [2.int32] of ASTNode)] of ASTNode) + it_parses "{[] of Foo, Bar::Baz * 2}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(Path.new(%w[Bar Baz]), "*", [2.int32] of ASTNode)] of ASTNode) + it_parses "{[] of Foo, Bar::Baz ** 2}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(Path.new(%w[Bar Baz]), "**", [2.int32] of ASTNode)] of ASTNode) + it_parses "{[] of Foo, ::foo}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new(nil, "foo", global: true)] of ASTNode) + it_parses "{[] of Foo, self.foo}", TupleLiteral.new([ArrayLiteral.new([] of ASTNode, "Foo".path), Call.new("self".var, "foo")] of ASTNode) + describe "end locations" do assert_end_location "nil" assert_end_location "false" diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 6abba79cdc86..2cb5843d8cc5 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -4665,7 +4665,7 @@ module Crystal # To determine to consume comma, looking-ahead is needed. # Consider `[ [] of Int32, Foo.new ]`, we want to parse it as `[ ([] of Int32), Foo.new ]` of course. # If the parser consumes comma afrer Int32 quickly, it may cause parsing error. - unless @token.type == :"->" || (@token.type == :"," && type_start?) + unless @token.type == :"->" || (@token.type == :"," && type_start?(consume_newlines: true)) if type.is_a?(Splat) raise "invalid type splat", type.location.not_nil! end @@ -4677,7 +4677,7 @@ module Crystal loop do next_token_skip_space_or_newline input_types << parse_type_splat { parse_union_type } - break unless @token.type == :"," && type_start? + break unless @token.type == :"," && type_start?(consume_newlines: true) end end @@ -5016,50 +5016,82 @@ module Crystal end # Looks ahead next tokens to check whether they indicate type. - def type_start?(consume_newlines = true) + def type_start?(*, consume_newlines) old_pos, old_line, old_column = current_pos, @line_number, @column_number @temp_token.copy_from(@token) - if consume_newlines - next_token_skip_space_or_newline - else - next_token_skip_space + begin + if consume_newlines + next_token_skip_space_or_newline + else + next_token_skip_space + end + + type_start? + ensure + @token.copy_from(@temp_token) + self.current_pos, @line_number, @column_number = old_pos, old_line, old_column end + end + def type_start? while @token.type == :"(" || @token.type == :"{" next_token_skip_space_or_newline end # TODO: the below conditions are not complete, and there are many false-positive or true-negative examples. - # For example, `[ [] of Int32, Foo::Bar.new ]` should be parsed to `[ ([] of Int32), Foo::Bar.new ]`, - # however, the current implementation mistakes `Foo::Bar` as type name, so parsing is failed. - begin - case @token.type - when :IDENT - case @token.value - when :typeof, :self, "self?" - true - else - false - end - when :CONST - return false if named_tuple_start? - next_token_skip_space - return true unless @token.type == :"." - next_token_skip_space_or_newline - @token.keyword?(:class) - when :"::" - next_token - @token.type == :CONST - when :UNDERSCORE, :"->" + case @token.type + when :IDENT + case @token.value + when :typeof true + when :self, "self?" + next_token_skip_space + delimiter_or_type_suffix? else false end - ensure - @token.copy_from(@temp_token) - self.current_pos, @line_number, @column_number = old_pos, old_line, old_column + when :CONST + return false if named_tuple_start? + type_path_start? + when :"::" + next_token + type_path_start? + when :UNDERSCORE, :"->" + true + when :"*" + next_token_skip_space_or_newline + type_start? + else + false + end + end + + def type_path_start? + while @token.type == :CONST + next_token + break unless @token.type == :"::" + next_token_skip_space_or_newline + end + + skip_space + delimiter_or_type_suffix? + end + + def delimiter_or_type_suffix? + case @token.type + when :"." + next_token_skip_space_or_newline + @token.keyword?(:class) + when :"?", :"*", :"**" + # They are conflicted with operators, so more look-ahead is needed. + next_token_skip_space + delimiter_or_type_suffix? + when :"->", :"|", :",", :NEWLINE, :EOF, :"=", :";", :"(", :")", :"[", :"]" + true + else + false end end From d9e9d2b95d756f913cabcbae2b52797f3ccc4ab4 Mon Sep 17 00:00:00 2001 From: luna Date: Tue, 26 May 2020 15:04:44 -0300 Subject: [PATCH 059/263] LibSSL: Add NO_TLS_V1_3 option (#9350) --- src/openssl/lib_ssl.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index 455e83c41ab5..a7ad45ea6da7 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -96,6 +96,7 @@ lib LibSSL NO_SSL_V3 = 0x02000000 NO_TLS_V1 = 0x04000000 + NO_TLS_V1_3 = 0x20000000 NO_TLS_V1_2 = 0x08000000 NO_TLS_V1_1 = 0x10000000 From f3f0b29c5a2b09b91ab72fc545f2981195eb3dc1 Mon Sep 17 00:00:00 2001 From: Michael Neumann Date: Tue, 26 May 2020 20:07:44 +0200 Subject: [PATCH 060/263] Fix Enumerable#{zip,zip?} when self is an Iterator (#9328) (#9330) * Fix Enumerable#{zip,zip?} when self is an Iterator (#9328) Both methods call `size` and as such consume the Iterator (`self`). Then they call Enumerable.zip which tries to again iterate over the now empty Iterator resulting in an empty array. Running std_spec "before" removing the call to `size` results in the following failures: Failures: 1) Enumerable zip works for Iterators as receiver Failure/Error: SpecCountUpIterator.new(3).zip(1..3, 2..4).should eq([{0, 1, 2}, {1, 2, 3}, {2, 3, 4}]) Expected: [{0, 1, 2}, {1, 2, 3}, {2, 3, 4}] got: [] # spec/std/enumerable_spec.cr:1057 2) Enumerable zip? works for Iterators as receiver Failure/Error: SpecCountUpIterator.new(3).zip?(1..2, 2..4).should eq([{0, 1, 2}, {1, 2, 3}, {2, nil, 4}]) Expected: [{0, 1, 2}, {1, 2, 3}, {2, nil, 4}] got: [] # spec/std/enumerable_spec.cr:1063 Fixes: #9328 * Add back optimization of Enumerable#{zip,zip?} when self is Indexable --- spec/std/enumerable_spec.cr | 23 +++++++++++++++++++++++ src/enumerable.cr | 2 ++ 2 files changed, 25 insertions(+) diff --git a/spec/std/enumerable_spec.cr b/spec/std/enumerable_spec.cr index b1becdf0334e..8cc3290968c3 100644 --- a/spec/std/enumerable_spec.cr +++ b/spec/std/enumerable_spec.cr @@ -17,6 +17,17 @@ private class SpecEmptyEnumerable end end +private class SpecCountUpIterator + include Iterator(Int32) + + def initialize(@size : Int32, @count = 0) + end + + def next + (@count += 1) <= @size ? (@count - 1) : stop + end +end + describe "Enumerable" do describe "all? with block" do it "returns true" do @@ -1040,4 +1051,16 @@ describe "Enumerable" do (1..3).to_h { |i| {i, i ** 2} }.should eq({1 => 1, 2 => 4, 3 => 9}) end end + + describe "zip" do + it "works for Iterators as receiver" do + SpecCountUpIterator.new(3).zip(1..3, 2..4).should eq([{0, 1, 2}, {1, 2, 3}, {2, 3, 4}]) + end + end + + describe "zip?" do + it "works for Iterators as receiver" do + SpecCountUpIterator.new(3).zip?(1..2, 2..4).should eq([{0, 1, 2}, {1, 2, 3}, {2, nil, 4}]) + end + end end diff --git a/src/enumerable.cr b/src/enumerable.cr index eb407b00cdf8..135ccde206e2 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -1634,6 +1634,7 @@ module Enumerable(T) # a.zip(b, c) # => [{1, 4, 8}, {2, 5, 7}, {3, 6, 6}] # ``` def zip(*others : Indexable | Iterable | Iterator) + size = self.is_a?(Indexable) ? self.size : 0 pairs = Array(typeof(zip(*others) { |e| break e }.not_nil!)).new(size) zip(*others) { |e| pairs << e } pairs @@ -1704,6 +1705,7 @@ module Enumerable(T) # a.zip?(b, c) # => [{1, 4, 8}, {2, 5, 7}, {3, nil, nil}] # ``` def zip?(*others : Indexable | Iterable | Iterator) + size = self.is_a?(Indexable) ? self.size : 0 pairs = Array(typeof(zip?(*others) { |e| break e }.not_nil!)).new(size) zip?(*others) { |e| pairs << e } pairs From 6e621f6b0255070dc846f1aa93323b1628b23136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 26 May 2020 20:08:47 +0200 Subject: [PATCH 061/263] Use SOURCE_DATE_EPOCH only to determine compiler date (#9088) * Use SOURCE_DATE_EPOCH only to determine compiler date * Add fallback for Makefile mtime --- Makefile | 4 +++- src/compiler/crystal/config.cr | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0222da6f4ad5..c2be15297353 100644 --- a/Makefile +++ b/Makefile @@ -31,10 +31,12 @@ override FLAGS += $(if $(release),--release )$(if $(stats),--stats )$(if $(progr SPEC_WARNINGS_OFF := --exclude-warnings spec/std --exclude-warnings spec/compiler SPEC_FLAGS := $(if $(verbose),-v )$(if $(junit_output),--junit_output $(junit_output) ) CRYSTAL_CONFIG_BUILD_COMMIT := $(shell git rev-parse --short HEAD 2> /dev/null) +SOURCE_DATE_EPOCH := $(shell (git show -s --format=%ct HEAD || stat -c "%Y" Makefile || stat -f "%m" Makefile) 2> /dev/null) EXPORTS := \ $(if $(release),,CRYSTAL_CONFIG_PATH="$(PWD)/src") \ CRYSTAL_CONFIG_LIBRARY_PATH="$(shell crystal env CRYSTAL_LIBRARY_PATH)" \ - CRYSTAL_CONFIG_BUILD_COMMIT="$(CRYSTAL_CONFIG_BUILD_COMMIT)" + CRYSTAL_CONFIG_BUILD_COMMIT="$(CRYSTAL_CONFIG_BUILD_COMMIT)" \ + SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" SHELL = sh LLVM_CONFIG := $(shell src/llvm/ext/find-llvm-config) LLVM_EXT_DIR = src/llvm/ext diff --git a/src/compiler/crystal/config.cr b/src/compiler/crystal/config.cr index 6d88fcfa9c0f..eb77b9e8886f 100644 --- a/src/compiler/crystal/config.cr +++ b/src/compiler/crystal/config.cr @@ -32,8 +32,12 @@ module Crystal end def self.date - time = {{ (env("SOURCE_DATE_EPOCH") || `date +%s`).to_i }} - Time.unix(time).to_s("%Y-%m-%d") + source_date_epoch = {{ (t = env("SOURCE_DATE_EPOCH")) && !t.empty? ? t.to_i : nil }} + if source_date_epoch + Time.unix(source_date_epoch).to_s("%Y-%m-%d") + else + "" + end end @@host_target : Crystal::Codegen::Target? From dbf2a21e76c45b369fb69a948065fc550fae989b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 26 May 2020 16:35:04 -0300 Subject: [PATCH 062/263] Compiler: warn on deprecated macros (#9343) * Improve assert_error reporting * Refactor annotation validation to apply to every target * Add Macro#owner property for error reporting * Add compiler warning on deprecated macro expansions * Rename original_location to expanded_location * Rename user_location to macro_location --- spec/compiler/codegen/warnings_spec.cr | 2 +- spec/compiler/semantic/macro_spec.cr | 2 +- spec/compiler/semantic/warnings_spec.cr | 176 ++++++++++++++++++ spec/spec_helper.cr | 14 +- src/compiler/crystal/codegen/codegen.cr | 4 +- src/compiler/crystal/codegen/debug.cr | 6 +- src/compiler/crystal/macros/macros.cr | 2 + src/compiler/crystal/macros/methods.cr | 8 +- src/compiler/crystal/semantic/ast.cr | 2 + .../crystal/semantic/semantic_visitor.cr | 19 +- .../crystal/semantic/top_level_visitor.cr | 10 - src/compiler/crystal/semantic/warnings.cr | 40 ++++ src/compiler/crystal/syntax/ast.cr | 2 +- src/compiler/crystal/syntax/location.cr | 21 ++- src/compiler/crystal/types.cr | 2 + 15 files changed, 276 insertions(+), 34 deletions(-) create mode 100644 spec/compiler/semantic/warnings_spec.cr diff --git a/spec/compiler/codegen/warnings_spec.cr b/spec/compiler/codegen/warnings_spec.cr index 65d26833d3e4..18fcc137b1ad 100644 --- a/spec/compiler/codegen/warnings_spec.cr +++ b/spec/compiler/codegen/warnings_spec.cr @@ -255,7 +255,7 @@ describe "Code gen: warnings" do "Error: wrong number of deprecated annotation arguments (given 2, expected 1)" end - it "errors if missing link arguments" do + it "errors if invalid named arguments" do assert_error %( @[Deprecated(invalid: "Do not use me")] def foo diff --git a/spec/compiler/semantic/macro_spec.cr b/spec/compiler/semantic/macro_spec.cr index 3d595e37e8ba..aba8a8aff2dd 100644 --- a/spec/compiler/semantic/macro_spec.cr +++ b/spec/compiler/semantic/macro_spec.cr @@ -1461,6 +1461,6 @@ describe "Semantic: macro" do ), inject_primitives: false) method = result.program.types["Foo"].lookup_first_def("bar", false).not_nil! - method.location.not_nil!.original_location.not_nil!.line_number.should eq(10) + method.location.not_nil!.expanded_location.not_nil!.line_number.should eq(10) end end diff --git a/spec/compiler/semantic/warnings_spec.cr b/spec/compiler/semantic/warnings_spec.cr new file mode 100644 index 000000000000..79e91c665d6f --- /dev/null +++ b/spec/compiler/semantic/warnings_spec.cr @@ -0,0 +1,176 @@ +require "../spec_helper" + +describe "Semantic: warnings" do + it "detects top-level deprecated marcos" do + assert_warning %( + @[Deprecated("Do not use me")] + macro foo + end + + foo + ), "warning in line 6\nWarning: Deprecated top-level foo. Do not use me", + inject_primitives: false + end + + it "deprecation reason is optional" do + assert_warning %( + @[Deprecated] + macro foo + end + + foo + ), "warning in line 6\nWarning: Deprecated top-level foo.", + inject_primitives: false + end + + it "detects deprecated class macros" do + assert_warning %( + class Foo + @[Deprecated("Do not use me")] + macro m + end + end + + Foo.m + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", + inject_primitives: false + end + + it "detects deprecated generic class macros" do + assert_warning %( + class Foo(T) + @[Deprecated("Do not use me")] + macro m + end + end + + Foo.m + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", + inject_primitives: false + end + + it "detects deprecated module macros" do + assert_warning %( + module Foo + @[Deprecated("Do not use me")] + macro m + end + end + + Foo.m + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", + inject_primitives: false + end + + it "detects deprecated macros with named arguments" do + assert_warning %( + @[Deprecated] + macro foo(*, a) + end + + foo(a: 2) + ), "warning in line 6\nWarning: Deprecated top-level foo.", + inject_primitives: false + end + + it "informs warnings once per call site location (a)" do + warning_failures = warnings_result %( + class Foo + @[Deprecated("Do not use me")] + macro m + end + + macro b + Foo.m + end + end + + Foo.b + Foo.b + ), inject_primitives: false + + warning_failures.size.should eq(1) + end + + it "informs warnings once per call site location (b)" do + warning_failures = warnings_result %( + class Foo + @[Deprecated("Do not use me")] + macro m + end + end + + Foo.m + Foo.m + ), inject_primitives: false + + warning_failures.size.should eq(2) + end + + it "ignore deprecation excluded locations" do + with_tempfile("check_warnings_excludes") do |path| + FileUtils.mkdir_p File.join(path, "lib") + + # NOTE tempfile might be created in symlinked folder + # which affects how to match current dir /var/folders/... + # with the real path /private/var/folders/... + path = File.real_path(path) + + main_filename = File.join(path, "main.cr") + output_filename = File.join(path, "main") + + Dir.cd(path) do + File.write main_filename, %( + require "./lib/foo" + + bar + foo + ) + File.write File.join(path, "lib", "foo.cr"), %( + @[Deprecated("Do not use me")] + macro foo + end + + macro bar + foo + end + ) + + compiler = create_spec_compiler + compiler.warnings = Warnings::All + compiler.warnings_exclude << Crystal.normalize_path "lib" + compiler.prelude = "empty" + result = compiler.compile Compiler::Source.new(main_filename, File.read(main_filename)), output_filename + + result.program.warning_failures.size.should eq(1) + end + end + end + + it "errors if invalid argument type" do + assert_error %( + @[Deprecated(42)] + macro foo + end + ), + "Error: first argument must be a String" + end + + it "errors if too many arguments" do + assert_error %( + @[Deprecated("Do not use me", "extra arg")] + macro foo + end + ), + "Error: wrong number of deprecated annotation arguments (given 2, expected 1)" + end + + it "errors if invalid named argument" do + assert_error %( + @[Deprecated(invalid: "Do not use me")] + macro foo + end + ), + "Error: too many named arguments (given 1, expected maximum 0)" + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index ed0862a1cb4c..c35ef4e87a87 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -104,10 +104,10 @@ def assert_after_cleanup(before, after) result.node.to_s.strip.should eq(after.strip) end -def assert_error(str, message, inject_primitives = true) +def assert_error(str, message, inject_primitives = true, file = __FILE__, line = __LINE__) str = inject_primitives(str) if inject_primitives nodes = parse str - expect_raises TypeException, message do + expect_raises TypeException, message, file, line do semantic nodes end end @@ -132,15 +132,15 @@ def warnings_result(code, inject_primitives = true) result.program.warning_failures end -def assert_warning(code, message, inject_primitives = true) +def assert_warning(code, message, inject_primitives = true, file = __FILE__, line = __LINE__) warning_failures = warnings_result(code, inject_primitives) - warning_failures.size.should eq(1) - warning_failures[0].should start_with(message) + warning_failures.size.should eq(1), file, line + warning_failures[0].should start_with(message), file, line end -def assert_no_warnings(code, inject_primitives = true) +def assert_no_warnings(code, inject_primitives = true, file = __FILE__, line = __LINE__) warning_failures = warnings_result(code, inject_primitives) - warning_failures.size.should eq(0) + warning_failures.size.should eq(0), file, line end def assert_macro(macro_args, macro_body, call_args, expected, expected_pragmas = nil, flags = nil) diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index a0456e9b7925..09eef8806dc0 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -563,7 +563,7 @@ module Crystal end def fun_literal_name(node : ProcLiteral) - location = node.location.try &.original_location + location = node.location.try &.expanded_location if location && (type = node.type?) proc_name = true filename = location.filename.as(String) @@ -1305,7 +1305,7 @@ module Crystal ] of ASTNode if location = node.location - pieces << StringLiteral.new(", at #{location.original_location}:#{location.line_number}").at(node) + pieces << StringLiteral.new(", at #{location.expanded_location}:#{location.line_number}").at(node) end ex = Call.new(Path.global("TypeCastError").at(node), "new", StringInterpolation.new(pieces).at(node)).at(node) diff --git a/src/compiler/crystal/codegen/debug.cr b/src/compiler/crystal/codegen/debug.cr index d3c978d04635..408aaf13069c 100644 --- a/src/compiler/crystal/codegen/debug.cr +++ b/src/compiler/crystal/codegen/debug.cr @@ -318,7 +318,7 @@ module Crystal end private def declare_local(type, alloca, location, basic_block : LLVM::BasicBlock? = nil) - location = location.try &.original_location + location = location.try &.expanded_location return false unless location file, dir = file_and_dir(location.filename) @@ -435,7 +435,7 @@ module Crystal end def set_current_debug_location(location) - location = location.try &.original_location + location = location.try &.expanded_location return unless location @current_debug_location = location @@ -464,7 +464,7 @@ module Crystal end def emit_def_debug_metadata(target_def) - location = target_def.location.try &.original_location + location = target_def.location.try &.expanded_location return unless location file, dir = file_and_dir(location.filename) diff --git a/src/compiler/crystal/macros/macros.cr b/src/compiler/crystal/macros/macros.cr index bd08725be600..c93c4392ec51 100644 --- a/src/compiler/crystal/macros/macros.cr +++ b/src/compiler/crystal/macros/macros.cr @@ -16,6 +16,8 @@ class Crystal::Program property compiled_macros_cache = {} of String => CompiledMacroRun def expand_macro(a_macro : Macro, call : Call, scope : Type, path_lookup : Type? = nil, a_def : Def? = nil) + check_call_to_deprecated_macro a_macro, call + interpreter = MacroInterpreter.new self, scope, path_lookup || scope, a_macro, call, a_def, in_macro: true a_macro.body.accept interpreter {interpreter.to_s, interpreter.macro_expansion_pragmas} diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 98d8a08163e9..e11e045c76ef 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -352,22 +352,22 @@ module Crystal end when "line_number" interpret_argless_method("line_number", args) do - line_number = location.try &.original_location.try &.line_number + line_number = location.try &.expanded_location.try &.line_number line_number ? NumberLiteral.new(line_number) : NilLiteral.new end when "column_number" interpret_argless_method("column_number", args) do - column_number = location.try &.original_location.try &.column_number + column_number = location.try &.expanded_location.try &.column_number column_number ? NumberLiteral.new(column_number) : NilLiteral.new end when "end_line_number" interpret_argless_method("end_line_number", args) do - line_number = end_location.try &.original_location.try &.line_number + line_number = end_location.try &.expanded_location.try &.line_number line_number ? NumberLiteral.new(line_number) : NilLiteral.new end when "end_column_number" interpret_argless_method("end_column_number", args) do - column_number = end_location.try &.original_location.try &.column_number + column_number = end_location.try &.expanded_location.try &.column_number column_number ? NumberLiteral.new(column_number) : NilLiteral.new end when "==" diff --git a/src/compiler/crystal/semantic/ast.cr b/src/compiler/crystal/semantic/ast.cr index e2ba5c40dec5..80c12bb70c8d 100644 --- a/src/compiler/crystal/semantic/ast.cr +++ b/src/compiler/crystal/semantic/ast.cr @@ -234,6 +234,8 @@ module Crystal class Macro include Annotatable + property! owner : Type + # Yields `arg, arg_index, object, object_index` corresponding # to arguments matching the given objects, taking into account this # macro's splat index. diff --git a/src/compiler/crystal/semantic/semantic_visitor.cr b/src/compiler/crystal/semantic/semantic_visitor.cr index a165a1659352..0a92fea1d212 100644 --- a/src/compiler/crystal/semantic/semantic_visitor.cr +++ b/src/compiler/crystal/semantic/semantic_visitor.cr @@ -429,7 +429,9 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor def process_annotations(annotations) annotations.try &.each do |ann| - yield lookup_annotation(ann), ann + annotation_type = lookup_annotation(ann) + validate_annotation(annotation_type, ann) + yield annotation_type, ann end end @@ -456,6 +458,21 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor type end + def validate_annotation(annotation_type, ann) + case annotation_type + when @program.deprecated_annotation + # Check whether a DeprecatedAnnotation can be built. + # There is no need to store it, but enforcing + # arguments makes sense here. + DeprecatedAnnotation.from(ann) + when @program.experimental_annotation + # ditto DeprecatedAnnotation + ExperimentalAnnotation.from(ann) + else + # go on + end + end + def check_class_var_annotations thread_local = false process_annotations(@annotations) do |annotation_type, ann| diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 4cd9660c7f80..c79a7485386b 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -1100,16 +1100,6 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor node.returns_twice = true when @program.raises_annotation node.raises = true - when @program.deprecated_annotation - # Check whether a DeprecatedAnnotation can be built. - # There is no need to store it, but enforcing - # arguments makes sense here. - DeprecatedAnnotation.from(ann) - yield annotation_type, ann - when @program.experimental_annotation - # ditto DeprecatedAnnotation - ExperimentalAnnotation.from(ann) - yield annotation_type, ann else yield annotation_type, ann end diff --git a/src/compiler/crystal/semantic/warnings.cr b/src/compiler/crystal/semantic/warnings.cr index 74642087ab5d..c7ff6b393a35 100644 --- a/src/compiler/crystal/semantic/warnings.cr +++ b/src/compiler/crystal/semantic/warnings.cr @@ -21,5 +21,45 @@ module Crystal self.warning_failures << message end + + @deprecated_macros_detected = Set(String).new + + def check_call_to_deprecated_macro(a_macro : Macro, call : Call) + return unless self.warnings.all? + + if (ann = a_macro.annotation(self.deprecated_annotation)) && + (deprecated_annotation = DeprecatedAnnotation.from(ann)) + call_location = call.location.try(&.macro_location) || call.location + + return if self.ignore_warning_due_to_location?(call_location) + short_reference = a_macro.short_reference + warning_key = call_location.try { |l| "#{short_reference} #{l}" } + + # skip warning if the call site was already informed + # if there is no location information just inform it. + return if !warning_key || @deprecated_macros_detected.includes?(warning_key) + @deprecated_macros_detected.add(warning_key) if warning_key + + message = deprecated_annotation.message + message = message ? " #{message}" : "" + + full_message = call.warning "Deprecated #{short_reference}.#{message}" + + self.warning_failures << full_message + end + end + end + + class Macro + def short_reference + case owner + when Program + "top-level #{name}" + when MetaclassType + "#{owner.instance_type.to_s(generic_args: false)}.#{name}" + else + "#{owner}.#{name}" + end + end end end diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index 45c9582c5301..38107b6f992e 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -2188,7 +2188,7 @@ module Crystal end def self.expand_line(location) - (location.try(&.original_location) || location).try(&.line_number) || 0 + (location.try(&.expanded_location) || location).try(&.line_number) || 0 end def self.expand_file_node(location) diff --git a/src/compiler/crystal/syntax/location.cr b/src/compiler/crystal/syntax/location.cr index 86ef22717bd4..00e8ca62cb64 100644 --- a/src/compiler/crystal/syntax/location.cr +++ b/src/compiler/crystal/syntax/location.cr @@ -18,20 +18,33 @@ class Crystal::Location # Returns the Location whose filename is a String, not a VirtualFile, # traversing virtual file expanded locations. - def original_location + def expanded_location case filename = @filename when String self when VirtualFile - filename.expanded_location.try &.original_location + filename.expanded_location.try &.expanded_location else nil end end - # Returns the filename of the `original_location` + # Returns the Location whose filename is a String, not a VirtualFile, + # traversing virtual file expanded locations leading to the original user source code + def macro_location + case filename = @filename + when String + self + when VirtualFile + filename.macro.location.try(&.macro_location) + else + nil + end + end + + # Returns the filename of the `expanded_location` def original_filename - original_location.try &.filename.as?(String) + expanded_location.try &.filename.as?(String) end def between?(min, max) diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 4c985b32bdab..625fb0f3ba19 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -895,6 +895,8 @@ module Crystal end def add_macro(a_macro) + a_macro.owner = self + case a_macro.name when "inherited" return add_hook :inherited, a_macro From 499d08d471aab7ac95b4ba3fa8187fe932658756 Mon Sep 17 00:00:00 2001 From: Stephanie Wilde-Hobbs Date: Wed, 27 May 2020 00:50:59 +0100 Subject: [PATCH 063/263] Simplify Link annotation handling (#8972) * Deprecate per-library static linking hints Static linking per-library is not supported well on any platform we support, and the feature is rarely used. Deprecate it so we can simplify the linking process. * Deprecate positional arguments in Link annotations * Simplify Link annotation handling * Tweak Link annotation docs * Update src/annotations.cr Co-authored-by: Brian J. Cardiff Co-authored-by: Brian J. Cardiff --- spec/compiler/semantic/lib_spec.cr | 20 ++- src/annotations.cr | 16 +-- src/compiler/crystal/codegen/link.cr | 122 ++++++++---------- src/compiler/crystal/compiler.cr | 1 - .../crystal/semantic/top_level_visitor.cr | 31 ++--- src/gc/boehm.cr | 2 +- 6 files changed, 95 insertions(+), 97 deletions(-) diff --git a/spec/compiler/semantic/lib_spec.cr b/spec/compiler/semantic/lib_spec.cr index b9d3131093c9..b91936fa1f4b 100644 --- a/spec/compiler/semantic/lib_spec.cr +++ b/spec/compiler/semantic/lib_spec.cr @@ -345,7 +345,7 @@ describe "Semantic: lib" do lib LibFoo end ), - "unknown link argument: 'boo' (valid arguments are 'lib', 'ldflags', 'static' and 'framework')" + "unknown link argument: 'boo' (valid arguments are 'lib', 'ldflags', 'static', 'pkg_config' and 'framework')" end it "errors if lib already specified with positional argument" do @@ -376,6 +376,24 @@ describe "Semantic: lib" do )) { int32 } end + it "warns if @[Link(static: true)] is specified" do + assert_warning %( + @[Link("foo", static: true)] + lib Foo + end + ), + "warning in line 3\nWarning: specifying static linking for individual libraries is deprecated" + end + + it "warns if Link annotations use positional arguments" do + assert_warning %( + @[Link("foo", "bar")] + lib Foo + end + ), + "warning in line 3\nWarning: using non-named arguments for Link annotations is deprecated" + end + it "allows invoking lib call without obj inside lib" do assert_type(%( lib LibFoo diff --git a/src/annotations.cr b/src/annotations.cr index db393cb6b772..c906f61b9b79 100644 --- a/src/annotations.cr +++ b/src/annotations.cr @@ -27,7 +27,7 @@ end annotation Flags end -# A `lib` can be marked with `@[Link(lib : String, ldflags : String, static : Bool, framework : String)]` +# A `lib` can be marked with `@[Link(lib : String, *, ldflags : String, framework : String, pkg_config : String)]` # to declare the library that should be linked when compiling the program. # # At least one of the *lib*, *ldflags*, *framework* arguments needs to be specified. @@ -38,17 +38,17 @@ end # 1. will lookup `pcre` using `pkg-config`, if not found # 2. will pass `-lpcre` to the linker. # -# `@[Link("pcre", static: true)]` will favor static libraries over shared libraries. -# 1. will lookup `libpcre.a` in `CRYSTAL_LIBRARY_PATH`, if not found -# 2. will lookup `pcre` using `pkg-config --static`, if not found, -# 3. will lookup `libpcre.a` in `/usr/lib`, `/usr/local/lib` +# `@[Link("pcre", pkg_config: "libpcre")]` will lookup for a shared library. +# 1. will lookup `libpcre` using `pkg-config`, if not found +# 2. will lookup `pcre` using `pkg-config`, if not found +# 3. will pass `-lpcre` to the linker. # # `@[Link(framework: "Cocoa")]` will pass `-framework Cocoa` to the linker. # # When an `-l` option is passed to the linker, it will lookup the libraries in -# paths passed with the `-L` option. `CRYSTAL_LIBRARY_PATH`, `/usr/lib`, -# and `/usr/local/lib` are added by default. Custom paths can be passed -# using `ldflags`: `@[Link(ldflags: "-Lvendor/bin")]`. +# paths passed with the `-L` option. Any paths in `CRYSTAL_LIBRARY_PATH` are +# added by default. Custom paths can be passed using `ldflags`: +# `@[Link(ldflags: "-Lvendor/bin")]`. annotation Link end diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 70ed81bdab2a..6bef92573968 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -1,10 +1,11 @@ module Crystal struct LinkAnnotation getter lib : String? + getter pkg_config : String? getter ldflags : String? getter framework : String? - def initialize(@lib = nil, @ldflags = nil, @static = false, @framework = nil) + def initialize(@lib = nil, @pkg_config = @lib, @ldflags = nil, @static = false, @framework = nil) end def static? @@ -22,6 +23,7 @@ module Crystal lib_name = nil lib_ldflags = nil lib_static = false + lib_pkg_config = nil lib_framework = nil count = 0 @@ -66,12 +68,15 @@ module Crystal named_arg.raise "'framework' link argument already specified" if count > 3 named_arg.raise "'framework' link argument must be a String" unless value.is_a?(StringLiteral) lib_framework = value.value + when "pkg_config" + named_arg.raise "'pkg_config' link argument must be a String" unless value.is_a?(StringLiteral) + lib_pkg_config = value.value else - named_arg.raise "unknown link argument: '#{named_arg.name}' (valid arguments are 'lib', 'ldflags', 'static' and 'framework')" + named_arg.raise "unknown link argument: '#{named_arg.name}' (valid arguments are 'lib', 'ldflags', 'static', 'pkg_config' and 'framework')" end end - new(lib_name, lib_ldflags, lib_static, lib_framework) + new(lib_name, lib_pkg_config, lib_ldflags, lib_static, lib_framework) end end @@ -109,83 +114,68 @@ module Crystal end private def lib_flags_posix - library_path = ENV["LIBRARY_PATH"]?.try(&.split(':', remove_empty: true)) || - ["/usr/lib", "/usr/local/lib"] - has_pkg_config = nil + flags = [] of String + static_build = has_flag?("static") - String.build do |flags| - link_annotations.reverse_each do |ann| - if ldflags = ann.ldflags - flags << ' ' << ldflags - end + # Instruct the linker to link statically if the user asks + flags << "-static" if static_build - if libname = ann.lib - if has_pkg_config.nil? - has_pkg_config = Process.run("pkg-config", ["-h"]).success? - end - - static = has_flag?("static") || ann.static? - - if static && (static_lib = find_static_lib(libname, CrystalLibraryPath.paths)) - flags << ' ' << static_lib - elsif has_pkg_config && (libflags = pkg_config_flags(libname, static, library_path)) - flags << ' ' << libflags - elsif static && (static_lib = find_static_lib(libname, library_path)) - flags << ' ' << static_lib - else - flags << " -l" << libname - end - end + # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially + # searches user-given library paths. + CrystalLibraryPath.paths.each do |path| + flags << "'-L#{path}'" + end - if framework = ann.framework - flags << " -framework " << framework - end + link_annotations.reverse_each do |ann| + if ldflags = ann.ldflags + flags << ldflags end - # Append the CRYSTAL_LIBRARY_PATH values as -L flags. - CrystalLibraryPath.paths.each do |path| - flags << " -L#{path}" + # First, check pkg-config for the pkg-config module name if provided, then + # check pkg-config with the lib name, then fall back to -lname + if (pkg_config_name = ann.pkg_config) && (flag = pkg_config(pkg_config_name, static_build)) + flags << flag + elsif (lib_name = ann.lib) && (flag = pkg_config(lib_name, static_build)) + flags << flag + elsif (lib_name = ann.lib) + flags << "-l#{lib_name}" end - # Append the default paths as -L flags in case the linker doesn't know - # about them (eg: FreeBSD won't search /usr/local/lib by default): - library_path.each do |path| - flags << " -L#{path}" + + if framework = ann.framework + flags << "-framework" << framework end end - end - def link_annotations - annotations = [] of LinkAnnotation - add_link_annotations @types, annotations - annotations + flags.join(" ") end - private def pkg_config_flags(libname, static, library_path) - if system("pkg-config #{libname}") - if static - flags = [] of String - `pkg-config #{libname} --libs --static`.split.each do |cfg| - if cfg.starts_with?("-L") - library_path << cfg[2..-1] - elsif cfg.starts_with?("-l") - flags << (find_static_lib(cfg[2..-1], library_path) || cfg) - else - flags << cfg - end - end - flags.join ' ' - else - `pkg-config #{libname} --libs`.chomp - end + PKG_CONFIG_PATH = Process.find_executable("pkg-config") + + # Returns the result of running `pkg-config mod` but returns nil if + # pkg-config is not installed, or the module does not exist. + private def pkg_config(mod, static = false) : String? + return unless pkg_config_path = PKG_CONFIG_PATH + return unless Process.run(pkg_config_path, {mod}).success? + + args = ["--libs"] + args << "--static" if static + args << mod + + process = Process.new(pkg_config_path, args, input: :close, output: :pipe, error: :inherit) + flags = process.output.gets_to_end.chomp + status = process.wait + if status.success? + flags + else + nil end end - private def find_static_lib(libname, library_path) - library_path.each do |libdir| - static_lib = "#{libdir}/lib#{libname}.a" - return static_lib if File.exists?(static_lib) - end - nil + # Returns every @[Link] annotation in the program parsed as `LinkAnnotation` + def link_annotations + annotations = [] of LinkAnnotation + add_link_annotations @types, annotations + annotations end private def add_link_annotations(types, annotations) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index dbea26202368..fc0fee3568d2 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -359,7 +359,6 @@ module Crystal link_flags = @link_flags || "" link_flags += " -rdynamic" - link_flags += " -static" if static? { %(#{cc} "${@}" -o '#{output_filename}' #{link_flags} #{program.lib_flags}), object_names } end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index c79a7485386b..deb9c4c861d3 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -464,7 +464,17 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor process_annotations(annotations) do |annotation_type, ann| case annotation_type when @program.link_annotation - type.add_link_annotation(LinkAnnotation.from(ann)) + link_annotation = LinkAnnotation.from(ann) + + if link_annotation.static? + @program.report_warning(ann, "specifying static linking for individual libraries is deprecated") + end + + if ann.args.size > 1 + @program.report_warning(ann, "using non-named arguments for Link annotations is deprecated") + end + + type.add_link_annotation(link_annotation) when @program.call_convention_annotation type.call_convention = parse_call_convention(ann, type.call_convention) else @@ -967,25 +977,6 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor false end - def process_lib_annotations - link_annotations = nil - call_convention = nil - - process_annotations do |annotation_type, ann| - case annotation_type - when @program.link - link_annotations ||= [] of LinkAnnotation - link_annotations << LinkAnnotation.from(ann) - when @program.call_convention - call_convention = parse_call_convention(ann, call_convention) - end - end - - @annotations = nil - - {link_annotations, call_convention} - end - def include_in(current_type, node, kind) node_name = node.name diff --git a/src/gc/boehm.cr b/src/gc/boehm.cr index b72ec95bd44a..1da7a4af7199 100644 --- a/src/gc/boehm.cr +++ b/src/gc/boehm.cr @@ -9,7 +9,7 @@ {% if flag?(:freebsd) || flag?(:dragonfly) %} @[Link("gc-threaded")] {% else %} - @[Link("gc", static: true)] + @[Link("gc")] {% end %} lib LibGC From 9ba66f990de04065645755b642bf33fa411d3575 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Thu, 28 May 2020 08:44:57 -0300 Subject: [PATCH 064/263] Make autocasting work in default values against unions --- spec/compiler/semantic/automatic_cast.cr | 54 +++++++++++++++++++ .../class_vars_initializer_visitor.cr | 2 +- .../instance_vars_initializer_visitor.cr | 2 +- src/compiler/crystal/semantic/main_visitor.cr | 24 +++++---- src/compiler/crystal/types.cr | 2 +- 5 files changed, 71 insertions(+), 13 deletions(-) diff --git a/spec/compiler/semantic/automatic_cast.cr b/spec/compiler/semantic/automatic_cast.cr index d1010287e66c..375379992b21 100644 --- a/spec/compiler/semantic/automatic_cast.cr +++ b/spec/compiler/semantic/automatic_cast.cr @@ -492,4 +492,58 @@ describe "Semantic: automatic cast" do fill(0, 0) )) { float64 } end + + it "can autocast to union in default value" do + assert_type(%( + def fill(x : Int64 | String = 1) + x + end + + fill() + )) { int64 } + end + + it "can autocast to alias in default value" do + assert_type(%( + alias X = Int64 | String + + def fill(x : X = 1) + x + end + + fill() + )) { int64 } + end + + it "can autocast to union in default value (symbol and int)" do + assert_type(%( + enum Color + Red + end + + def fill(x : Int64 | Color = :red) + x + end + + fill() + )) { types["Color"] } + end + + it "can autocast to union in default value (multiple enums)" do + assert_type(%( + enum Color + Red + end + + enum AnotherColor + Blue + end + + def fill(x : Color | AnotherColor = :blue) + x + end + + fill() + )) { types["AnotherColor"] } + end end diff --git a/src/compiler/crystal/semantic/class_vars_initializer_visitor.cr b/src/compiler/crystal/semantic/class_vars_initializer_visitor.cr index 700666a9183b..c00a318fc81e 100644 --- a/src/compiler/crystal/semantic/class_vars_initializer_visitor.cr +++ b/src/compiler/crystal/semantic/class_vars_initializer_visitor.cr @@ -73,7 +73,7 @@ module Crystal (class_var_type = class_var.type?) cloned_node = node.clone cloned_node.accept MainVisitor.new(self) - if casted_value = MainVisitor.check_automatic_cast(cloned_node, class_var_type) + if casted_value = MainVisitor.check_automatic_cast(@program, cloned_node, class_var_type) node = initializer.node = casted_value end end diff --git a/src/compiler/crystal/semantic/instance_vars_initializer_visitor.cr b/src/compiler/crystal/semantic/instance_vars_initializer_visitor.cr index 91e37e6aa16a..b67c9f3f0d80 100644 --- a/src/compiler/crystal/semantic/instance_vars_initializer_visitor.cr +++ b/src/compiler/crystal/semantic/instance_vars_initializer_visitor.cr @@ -98,7 +98,7 @@ class Crystal::InstanceVarsInitializerVisitor < Crystal::SemanticVisitor (scope_initializer = scope_initializers[index]) cloned_value = value.clone cloned_value.accept MainVisitor.new(program) - if casted_value = MainVisitor.check_automatic_cast(cloned_value, scope.lookup_instance_var(i.target.name).type) + if casted_value = MainVisitor.check_automatic_cast(@program, cloned_value, scope.lookup_instance_var(i.target.name).type) scope_initializer.value = casted_value next end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index 238a353dfe6e..c6b4ec4009c4 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -977,23 +977,27 @@ module Crystal # See if we can automatically cast the value if the types don't exactly match def check_automatic_cast(value, var_type, assign = nil) - MainVisitor.check_automatic_cast(value, var_type, assign) + MainVisitor.check_automatic_cast(@program, value, var_type, assign) end - def self.check_automatic_cast(value, var_type, assign = nil) - if value.is_a?(NumberLiteral) && value.type != var_type && (var_type.is_a?(IntegerType) || var_type.is_a?(FloatType)) - if value.can_be_autocast_to?(var_type) - value.type = var_type - value.kind = var_type.kind + def self.check_automatic_cast(program, value, var_type, assign = nil) + if value.is_a?(NumberLiteral) && value.type != var_type + literal_type = NumberLiteralType.new(program, value) + restricted = literal_type.restrict(var_type, MatchContext.new(value.type, value.type)) + if restricted.is_a?(IntegerType) || restricted.is_a?(FloatType) + value.type = restricted + value.kind = restricted.kind assign.value = value if assign return value end - elsif value.is_a?(SymbolLiteral) && var_type.is_a?(EnumType) - member = var_type.find_member(value.value) - if member + elsif value.is_a?(SymbolLiteral) && value.type != var_type + literal_type = SymbolLiteralType.new(program, value) + restricted = literal_type.restrict(var_type, MatchContext.new(value.type, value.type)) + if restricted.is_a?(EnumType) + member = restricted.find_member(value.value).not_nil! path = Path.new(member.name) path.target_const = member - path.type = var_type + path.type = restricted value = path assign.value = value if assign return value diff --git a/src/compiler/crystal/types.cr b/src/compiler/crystal/types.cr index 625fb0f3ba19..affb0c69a546 100644 --- a/src/compiler/crystal/types.cr +++ b/src/compiler/crystal/types.cr @@ -1618,7 +1618,7 @@ module Crystal # Check if automatic cast can be done if instance_var.type != value.type && (value.is_a?(NumberLiteral) || value.is_a?(SymbolLiteral)) - if casted_value = MainVisitor.check_automatic_cast(value, instance_var.type) + if casted_value = MainVisitor.check_automatic_cast(@program, value, instance_var.type) value = casted_value end end From aa893b75e7ce8e50bddf658f7fe16c572199ea5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 28 May 2020 16:29:02 +0200 Subject: [PATCH 065/263] Deprecate JSON.mapping and YAML.mapping (#9272) --- src/json.cr | 12 +++--------- src/json/mapping.cr | 2 ++ src/json/to_json.cr | 45 +++++++++++++++++++++++++-------------------- src/uuid/json.cr | 4 +++- src/yaml.cr | 4 ++-- src/yaml/mapping.cr | 2 ++ src/yaml/to_yaml.cr | 9 +++++---- 7 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/json.cr b/src/json.cr index b963373b190d..713eafb03ea1 100644 --- a/src/json.cr +++ b/src/json.cr @@ -28,7 +28,7 @@ # Most types in the standard library implement these methods. For user-defined types # you can define a `self.new(pull : JSON::PullParser)` for parsing and # `to_json(builder : JSON::Builder)` for serializing. The following sections -# show convenient ways to do this using either `JSON.mapping` or `JSON::Serializable`. +# show convenient ways to do this using `JSON::Serializable`. # # NOTE: JSON object keys are always strings but they can still be parsed # and deserialized to other types. To deserialize, define a @@ -46,12 +46,6 @@ # {1.5 => 2}.to_json # => "{\"1.5\":2}" # ``` # -# ### Parsing and generating with `JSON.mapping` -# -# Use `JSON.mapping` to define how an object is mapped to JSON, making it -# the recommended easy, type-safe and efficient option for parsing and generating -# JSON. Refer to that module's documentation to learn about it. -# # ### Parsing with `JSON.parse` # # `JSON.parse` will return an `Any`, which is a convenient wrapper around all possible JSON types, @@ -83,7 +77,7 @@ # end # ``` # -# Parsing with `JSON.parse` is useful for dealing with a dynamic JSON structure but is slower than using `JSON.mapping`. +# Parsing with `JSON.parse` is useful for dealing with a dynamic JSON structure. # # ### Generating with `JSON.build` # @@ -112,7 +106,7 @@ # # `to_json`, `to_json(IO)` and `to_json(JSON::Builder)` methods are provided # for primitive types, but you need to define `to_json(JSON::Builder)` -# for custom objects, either manually or using `JSON.mapping`. +# for custom objects, either manually or using `JSON::Serializable`. module JSON # Generic JSON error. class Error < Exception diff --git a/src/json/mapping.cr b/src/json/mapping.cr index b486050cd581..a33a0529c088 100644 --- a/src/json/mapping.cr +++ b/src/json/mapping.cr @@ -65,6 +65,7 @@ module JSON # If *strict* is `true`, unknown properties in the JSON # document will raise a parse exception. The default is `false`, so unknown properties # are silently ignored. + @[Deprecated("use JSON::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/json_mapping.cr)")] macro mapping(_properties_, strict = false) {% for key, value in _properties_ %} {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} @@ -227,6 +228,7 @@ module JSON # This is a convenience method to allow invoking `JSON.mapping` # with named arguments instead of with a hash/named-tuple literal. + @[Deprecated("use JSON::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/json_mapping.cr)")] macro mapping(**_properties_) ::JSON.mapping({{_properties_}}) end diff --git a/src/json/to_json.cr b/src/json/to_json.cr index 75e9334cd2a0..ebb80566f146 100644 --- a/src/json/to_json.cr +++ b/src/json/to_json.cr @@ -179,16 +179,17 @@ struct Time end end -# Converter to be used with `JSON.mapping` +# Converter to be used with `JSON::Serializable` # to serialize the `Array(T)` elements with the custom converter. # # ``` # require "json" # # class TimestampArray -# JSON.mapping({ -# dates: {type: Array(Time), converter: JSON::ArrayConverter(Time::EpochConverter)}, -# }) +# include JSON::Serializable +# +# @[JSON::Field(converter: JSON::ArrayConverter(Time::EpochConverter))] +# property dates : Array(Time) # end # # timestamp = TimestampArray.from_json(%({"dates":[1459859781,1567628762]})) @@ -205,16 +206,17 @@ module JSON::ArrayConverter(Converter) end end -# Converter to be used with `JSON.mapping` +# Converter to be used with `JSON::Serializable` # to serialize the `Hash(K, V)` values elements with the custom converter. # # ``` # require "json" # # class TimestampHash -# JSON.mapping({ -# birthdays: {type: Hash(String, Time), converter: JSON::HashValueConverter(Time::EpochConverter)}, -# }) +# include JSON::Serializable +# +# @[JSON::Field(converter: JSON::HashValueConverter(Time::EpochConverter))] +# birthdays : Hash(String, Time) # end # # timestamp = TimestampHash.from_json(%({"birthdays":{"foo":1459859781,"bar":1567628762}})) @@ -233,7 +235,7 @@ module JSON::HashValueConverter(Converter) end end -# Converter to be used with `JSON.mapping` and `YAML.mapping` +# Converter to be used with `JSON::Serializable` and `YAML::Serializable` # to serialize a `Time` instance as the number of seconds # since the unix epoch. See `Time#to_unix`. # @@ -241,9 +243,10 @@ end # require "json" # # class Person -# JSON.mapping({ -# birth_date: {type: Time, converter: Time::EpochConverter}, -# }) +# include JSON::Serializable +# +# @[JSON::Field(converter: Time::EpochConverter)] +# birth_date : Time # end # # person = Person.from_json(%({"birth_date": 1459859781})) @@ -256,7 +259,7 @@ module Time::EpochConverter end end -# Converter to be used with `JSON.mapping` and `YAML.mapping` +# Converter to be used with `JSON::Serializable` and `YAML::Serializable` # to serialize a `Time` instance as the number of milliseconds # since the unix epoch. See `Time#to_unix_ms`. # @@ -264,9 +267,10 @@ end # require "json" # # class Timestamp -# JSON.mapping({ -# value: {type: Time, converter: Time::EpochMillisConverter}, -# }) +# include JSON::Serializable +# +# @[JSON::Field(converter: Time::EpochMillisConverter)] +# value : Time # end # # timestamp = Timestamp.from_json(%({"value": 1459860483856})) @@ -279,7 +283,7 @@ module Time::EpochMillisConverter end end -# Converter to be used with `JSON.mapping` to read the raw +# Converter to be used with `JSON::Serializable` to read the raw # value of a JSON object property as a `String`. # # It can be useful to read ints and floats without losing precision, @@ -290,9 +294,10 @@ end # require "json" # # class Raw -# JSON.mapping({ -# value: {type: String, converter: String::RawConverter}, -# }) +# include JSON::Serializable +# +# @[JSON::Field(converter: String::RawConverter)] +# value : String # end # # raw = Raw.from_json(%({"value": 123456789876543212345678987654321})) diff --git a/src/uuid/json.cr b/src/uuid/json.cr index 062787985db8..edc38182c0db 100644 --- a/src/uuid/json.cr +++ b/src/uuid/json.cr @@ -12,7 +12,9 @@ struct UUID # require "uuid/json" # # class Example - # JSON.mapping id: UUID + # include JSON::Serializable + # + # property id : UUID # end # # example = Example.from_json(%({"id": "ba714f86-cac6-42c7-8956-bcf5105e1b81"})) diff --git a/src/yaml.cr b/src/yaml.cr index b238b5b52a17..cc7f0251b364 100644 --- a/src/yaml.cr +++ b/src/yaml.cr @@ -50,7 +50,7 @@ require "base64" # anchored values (see `YAML::PullParser` for an explanation of this). # # Crystal primitive types, `Time`, `Bytes` and `Union` implement -# this method. `YAML.mapping` can be used to implement this method +# this method. `YAML::Serializable` can be used to implement this method # for user types. # # ### Dumping with `YAML.dump` or `#to_yaml` @@ -64,7 +64,7 @@ require "base64" # `to_yaml(builder : YAML::Nodes::Builder`). # # Crystal primitive types, `Time` and `Bytes` implement -# this method. `YAML.mapping` can be used to implement this method +# this method. `YAML::Serializable` can be used to implement this method # for user types. # # ``` diff --git a/src/yaml/mapping.cr b/src/yaml/mapping.cr index 74665ffb997f..ea85141223e1 100644 --- a/src/yaml/mapping.cr +++ b/src/yaml/mapping.cr @@ -67,6 +67,7 @@ module YAML # it and initializes this type's instance variables. # # This macro also declares instance variables of the types given in the mapping. + @[Deprecated("use YAML::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/yaml_mapping.cr)")] macro mapping(_properties_, strict = false) {% for key, value in _properties_ %} {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} @@ -217,6 +218,7 @@ module YAML # This is a convenience method to allow invoking `YAML.mapping` # with named arguments instead of with a hash/named-tuple literal. + @[Deprecated("use YAML::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/yaml_mapping.cr)")] macro mapping(**_properties_) ::YAML.mapping({{_properties_}}) end diff --git a/src/yaml/to_yaml.cr b/src/yaml/to_yaml.cr index adbd75c0b86e..db488be72bb7 100644 --- a/src/yaml/to_yaml.cr +++ b/src/yaml/to_yaml.cr @@ -136,16 +136,17 @@ module Time::EpochMillisConverter end end -# Converter to be used with `YAML.mapping` +# Converter to be used with `YAML::Serializable` # to serialize the `Array(T)` elements with the custom converter. # # ``` # require "yaml" # # class Timestamp -# YAML.mapping({ -# values: {type: Array(Time), converter: YAML::ArrayConverter(Time::EpochConverter)}, -# }) +# include YAML::Serializable +# +# @[YAML::Field(converter: YAML::ArrayConverter(Time::EpochConverter))] +# values : Array(Time) # end # # timestamp = Timestamp.from_yaml(%({"values":[1459859781,1567628762]})) From 05e5f958ffbea5387ff93b6fc1df544b5929bacb Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Thu, 28 May 2020 14:13:35 -0300 Subject: [PATCH 066/263] Process raises `IO::Error` (or subclasses) (#9340) * Process raises `IO::Error` (or subclasses) * Disable failing specs in Windows * Rollback spec change to make it pass in Windows * Fix `Process.exec` when the process cannot be executed * Fix spec on Windows --- spec/std/process_spec.cr | 29 +++++++++++++++-- src/crystal/system/unix/process.cr | 49 +++++++++++++++++++++++------ src/crystal/system/win32/process.cr | 8 ++++- src/process.cr | 2 +- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 7389fa8a4a78..3f82c98190ef 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -69,9 +69,28 @@ describe Process do process.wait.exit_code.should eq(1) end + it "raises if command doesn't exist" do + expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do + Process.new("foobarbaz") + end + end + + pending_win32 "raises if command is not executable" do + with_tempfile("crystal-spec-run") do |path| + File.touch path + expect_raises(File::AccessDeniedError, "Error executing process: '#{path.inspect_unquoted}'") do + Process.new(path) + end + end + end + it "raises if command could not be executed" do - expect_raises(RuntimeError, "Error executing process:") do - Process.new("foobarbaz", ["foo"]) + with_tempfile("crystal-spec-run") do |path| + File.touch path + command = File.join(path, "foo") + expect_raises(IO::Error, "Error executing process: '#{command.inspect_unquoted}'") do + Process.new(command) + end end end @@ -338,6 +357,12 @@ describe Process do File.exists?(path).should be_true end end + + it "gets error from exec" do + expect_raises(File::NotFoundError, "Error executing process: 'foobarbaz'") do + Process.exec("foobarbaz") + end + end {% end %} pending_win32 "checks for existence" do diff --git a/src/crystal/system/unix/process.cr b/src/crystal/system/unix/process.cr index 3c4828e455d8..dc62f2db7bd9 100644 --- a/src/crystal/system/unix/process.cr +++ b/src/crystal/system/unix/process.cr @@ -2,6 +2,7 @@ require "c/signal" require "c/stdlib" require "c/sys/resource" require "c/unistd" +require "file/error" struct Crystal::System::Process getter pid : LibC::PidT @@ -123,8 +124,11 @@ struct Crystal::System::Process begin reader_pipe.close writer_pipe.close_on_exec = true - self.replace(command_args, env, clear_env, input, output, error, chdir) + self.try_replace(command_args, env, clear_env, input, output, error, chdir) + writer_pipe.write_byte(1) + writer_pipe.write_bytes(Errno.value.to_i) rescue ex + writer_pipe.write_byte(0) writer_pipe.write_bytes(ex.message.try(&.bytesize) || 0) writer_pipe << ex.message writer_pipe.close @@ -134,16 +138,28 @@ struct Crystal::System::Process end writer_pipe.close - bytes = uninitialized UInt8[4] - if reader_pipe.read(bytes.to_slice) == 4 - message_size = IO::ByteFormat::SystemEndian.decode(Int32, bytes.to_slice) - if message_size > 0 - message = String.build(message_size) { |io| IO.copy(reader_pipe, io, message_size) } + begin + case reader_pipe.read_byte + when nil + # Pipe was closed, no error + when 0 + # Error message coming + message_size = reader_pipe.read_bytes(Int32) + if message_size > 0 + message = String.build(message_size) { |io| IO.copy(reader_pipe, io, message_size) } + end + reader_pipe.close + raise RuntimeError.new("Error executing process: '#{command_args[0]}': #{message}") + when 1 + # Errno coming + errno = Errno.new(reader_pipe.read_bytes(Int32)) + self.raise_exception_from_errno(command_args[0], errno) + else + raise RuntimeError.new("BUG: Invalid error response received from subprocess") end + ensure reader_pipe.close - raise RuntimeError.new("Error executing process: #{message}") end - reader_pipe.close pid end @@ -173,7 +189,7 @@ struct Crystal::System::Process end end - def self.replace(command_args, env, clear_env, input, output, error, chdir) : NoReturn + private def self.try_replace(command_args, env, clear_env, input, output, error, chdir) reopen_io(input, ORIGINAL_STDIN) reopen_io(output, ORIGINAL_STDOUT) reopen_io(error, ORIGINAL_STDERR) @@ -194,7 +210,20 @@ struct Crystal::System::Process argv << Pointer(UInt8).null LibC.execvp(command, argv) - raise RuntimeError.from_errno + end + + def self.replace(command_args, env, clear_env, input, output, error, chdir) + try_replace(command_args, env, clear_env, input, output, error, chdir) + raise_exception_from_errno(command_args[0]) + end + + private def self.raise_exception_from_errno(command, errno = Errno.value) + case errno + when Errno::EACCES, Errno::ENOENT + raise ::File::Error.from_errno("Error executing process", errno, file: command) + else + raise IO::Error.from_errno("Error executing process: '#{command}'", errno) + end end private def self.reopen_io(src_io : IO::FileDescriptor, dst_io : IO::FileDescriptor) diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 537e053fed16..44c240d15086 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -126,7 +126,13 @@ struct Crystal::System::Process make_env_block(env, clear_env), chdir.try &.check_no_null_byte.to_utf16, pointerof(startup_info), pointerof(process_info) ) == 0 - raise RuntimeError.from_winerror("Error executing process") + error = WinError.value + case error.to_errno + when Errno::EACCES, Errno::ENOENT + raise ::File::Error.from_winerror("Error executing process", error, file: command_args) + else + raise IO::Error.from_winerror("Error executing process: '#{command_args}'", error) + end end close_handle(process_info.hThread) diff --git a/src/process.cr b/src/process.cr index e1088aea6208..3e7eb7bce939 100644 --- a/src/process.cr +++ b/src/process.cr @@ -156,7 +156,7 @@ class Process # # Available only on Unix-like operating systems. def self.exec(command : String, args = nil, env : Env = nil, clear_env : Bool = false, shell : Bool = false, - input : ExecStdio = Redirect::Inherit, output : ExecStdio = Redirect::Inherit, error : ExecStdio = Redirect::Inherit, chdir : String? = nil) + input : ExecStdio = Redirect::Inherit, output : ExecStdio = Redirect::Inherit, error : ExecStdio = Redirect::Inherit, chdir : String? = nil) : NoReturn command_args = Crystal::System::Process.prepare_args(command, args, shell) input = exec_stdio_to_fd(input, for: STDIN) From e43364f8093f961d1beb6d397fd4879e981d90e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 28 May 2020 19:14:13 +0200 Subject: [PATCH 067/263] Fix parsing AM/PM hours (#9334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix parsing AM/PM hours * fixup! Fix parsing AM/PM hours * Add specs for 0:00 am/pm * Accept midnight as 24:00 of next day * Update src/time/format/parser.cr Co-authored-by: Jonne Haß * Add spec for am/pm without %P * Fix 12-hour clock without am/pm * Make 0 am invalid * Cleanup implementation and fail hour > 12 Co-authored-by: Jonne Haß --- spec/std/time/format_spec.cr | 146 +++++++++++++++++++++++++++++++++++ spec/std/time/time_spec.cr | 16 ++++ src/time.cr | 5 +- src/time/format/parser.cr | 23 +++++- 4 files changed, 187 insertions(+), 3 deletions(-) diff --git a/spec/std/time/format_spec.cr b/spec/std/time/format_spec.cr index 9820df0c63b7..03fd3362d24d 100644 --- a/spec/std/time/format_spec.cr +++ b/spec/std/time/format_spec.cr @@ -218,6 +218,10 @@ describe Time::Format do parse_time(" 9", "%l").hour.should eq(9) parse_time("9pm", "%l%p").hour.should eq(21) parse_time("9PM", "%l%P").hour.should eq(21) + parse_time("12PM", "%l%P").hour.should eq(12) + parse_time("9am", "%l%p").hour.should eq(9) + parse_time("9AM", "%l%P").hour.should eq(9) + parse_time("12AM", "%l%P").hour.should eq(0) parse_time("09", "%M").minute.should eq(9) parse_time("09", "%S").second.should eq(9) parse_time("123", "%L").millisecond.should eq(123) @@ -267,6 +271,148 @@ describe Time::Format do parse_time("2009-W53-7", "%G-W%V-%u").should eq(Time.utc(2010, 1, 3)) end + it "parses am/pm" do + parse_time("12:00 am", "%I:%M %P").to_s("%H:%M").should eq("00:00") + parse_time("12:01 am", "%I:%M %P").to_s("%H:%M").should eq("00:01") + parse_time("01:00 am", "%I:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 am", "%I:%M %P").to_s("%H:%M").should eq("11:00") + parse_time("00:00 pm", "%I:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("00:01 pm", "%I:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("12:00 pm", "%I:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 pm", "%I:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("01:00 pm", "%I:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("11:00 pm", "%I:%M %P").to_s("%H:%M").should eq("23:00") + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("00:00 am", "%I:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("00:01 am", "%I:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00 am", "%I:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00 pm", "%I:%M %P") + end + + parse_time("12:00", "%I:%M").to_s("%H:%M").should eq("00:00") + parse_time("12:01", "%I:%M").to_s("%H:%M").should eq("00:01") + parse_time("01:00", "%I:%M").to_s("%H:%M").should eq("01:00") + parse_time("11:00", "%I:%M").to_s("%H:%M").should eq("11:00") + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("00:00", "%I:%M") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("00:01", "%I:%M") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00", "%I:%M") + end + + parse_time("12:00 am", "%l:%M %P").to_s("%H:%M").should eq("00:00") + parse_time("12:01 am", "%l:%M %P").to_s("%H:%M").should eq("00:01") + parse_time(" 1:00 am", "%l:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 am", "%l:%M %P").to_s("%H:%M").should eq("11:00") + parse_time(" 0:00 pm", "%l:%M %P").to_s("%H:%M").should eq("12:00") + parse_time(" 0:01 pm", "%l:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("12:00 pm", "%l:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 pm", "%l:%M %P").to_s("%H:%M").should eq("12:01") + parse_time(" 1:00 pm", "%l:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("11:00 pm", "%l:%M %P").to_s("%H:%M").should eq("23:00") + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time(" 0:00 am", "%l:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time(" 0:01 am", "%l:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00 am", "%l:%M %P") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00 pm", "%l:%M %P") + end + + parse_time("12:00", "%l:%M").to_s("%H:%M").should eq("00:00") + parse_time("12:01", "%l:%M").to_s("%H:%M").should eq("00:01") + parse_time("01:00", "%l:%M").to_s("%H:%M").should eq("01:00") + parse_time("11:00", "%l:%M").to_s("%H:%M").should eq("11:00") + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time(" 0:00", "%l:%M") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time(" 0:01", "%l:%M") + end + expect_raises Time::Format::Error, "Invalid hour for 12-hour clock" do + parse_time("13:00", "%l:%M") + end + end + + it "parses 24h clock" do + parse_time("00:00", "%H:%M").to_s("%H:%M").should eq("00:00") + parse_time("00:01", "%H:%M").to_s("%H:%M").should eq("00:01") + parse_time("01:00", "%H:%M").to_s("%H:%M").should eq("01:00") + parse_time("11:00", "%H:%M").to_s("%H:%M").should eq("11:00") + parse_time("12:00", "%H:%M").to_s("%H:%M").should eq("12:00") + parse_time("12:01", "%H:%M").to_s("%H:%M").should eq("12:01") + parse_time("13:00", "%H:%M").to_s("%H:%M").should eq("13:00") + parse_time("23:00", "%H:%M").to_s("%H:%M").should eq("23:00") + parse_time("24:00", "%H:%M").to_s("%H:%M").should eq("00:00") + parse_time("2020-05-21 24:00", "%F %H:%M").should eq(Time.utc(2020, 5, 22, 0, 0)) + + parse_time(" 0:00", "%k:%M").to_s("%H:%M").should eq("00:00") + parse_time(" 0:01", "%k:%M").to_s("%H:%M").should eq("00:01") + parse_time(" 1:00", "%k:%M").to_s("%H:%M").should eq("01:00") + parse_time("11:00", "%k:%M").to_s("%H:%M").should eq("11:00") + parse_time("12:00", "%k:%M").to_s("%H:%M").should eq("12:00") + parse_time("12:01", "%k:%M").to_s("%H:%M").should eq("12:01") + parse_time("13:00", "%k:%M").to_s("%H:%M").should eq("13:00") + parse_time("23:00", "%k:%M").to_s("%H:%M").should eq("23:00") + parse_time("24:00", "%k:%M").to_s("%H:%M").should eq("00:00") + parse_time("2020-05-21 24:00", "%F %k:%M").should eq(Time.utc(2020, 5, 22, 0, 0)) + end + + it "parses 24h clock with am/pm" do + parse_time("00:00 AM", "%H:%M %P").to_s("%H:%M").should eq("00:00") + parse_time("00:01 AM", "%H:%M %P").to_s("%H:%M").should eq("00:01") + parse_time("01:00 AM", "%H:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 AM", "%H:%M %P").to_s("%H:%M").should eq("11:00") + parse_time("12:00 AM", "%H:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 AM", "%H:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("13:00 AM", "%H:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("23:00 AM", "%H:%M %P").to_s("%H:%M").should eq("23:00") + parse_time("24:00 AM", "%H:%M %P").to_s("%H:%M").should eq("00:00") + + parse_time(" 0:00 AM", "%k:%M %P").to_s("%H:%M").should eq("00:00") + parse_time(" 0:01 AM", "%k:%M %P").to_s("%H:%M").should eq("00:01") + parse_time(" 1:00 AM", "%k:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 AM", "%k:%M %P").to_s("%H:%M").should eq("11:00") + parse_time("12:00 AM", "%k:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 AM", "%k:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("13:00 AM", "%k:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("23:00 AM", "%k:%M %P").to_s("%H:%M").should eq("23:00") + parse_time("24:00 AM", "%k:%M %P").to_s("%H:%M").should eq("00:00") + + parse_time("00:00 PM", "%H:%M %P").to_s("%H:%M").should eq("00:00") + parse_time("00:01 PM", "%H:%M %P").to_s("%H:%M").should eq("00:01") + parse_time("01:00 PM", "%H:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 PM", "%H:%M %P").to_s("%H:%M").should eq("11:00") + parse_time("12:00 PM", "%H:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 PM", "%H:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("13:00 PM", "%H:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("23:00 PM", "%H:%M %P").to_s("%H:%M").should eq("23:00") + parse_time("24:00 PM", "%H:%M %P").to_s("%H:%M").should eq("00:00") + + parse_time(" 0:00 PM", "%k:%M %P").to_s("%H:%M").should eq("00:00") + parse_time(" 0:01 PM", "%k:%M %P").to_s("%H:%M").should eq("00:01") + parse_time(" 1:00 PM", "%k:%M %P").to_s("%H:%M").should eq("01:00") + parse_time("11:00 PM", "%k:%M %P").to_s("%H:%M").should eq("11:00") + parse_time("12:00 PM", "%k:%M %P").to_s("%H:%M").should eq("12:00") + parse_time("12:01 PM", "%k:%M %P").to_s("%H:%M").should eq("12:01") + parse_time("13:00 PM", "%k:%M %P").to_s("%H:%M").should eq("13:00") + parse_time("23:00 PM", "%k:%M %P").to_s("%H:%M").should eq("23:00") + parse_time("24:00 PM", "%k:%M %P").to_s("%H:%M").should eq("00:00") + end + it "parses timezone" do patterns = {"%z", "%:z", "%::z"} diff --git a/spec/std/time/time_spec.cr b/spec/std/time/time_spec.cr index 57cbae04b884..99bba2bef7a0 100644 --- a/spec/std/time/time_spec.cr +++ b/spec/std/time/time_spec.cr @@ -173,6 +173,22 @@ describe Time do end end end + + it "accepts midnight 24:00" do + Time.utc(2020, 5, 21, 24, 0, 0).should eq Time.utc(2020, 5, 22, 0, 0, 0) + + expect_raises ArgumentError, "Invalid time" do + Time.utc(2020, 5, 21, 24, 0, 0, nanosecond: 1) + end + + expect_raises ArgumentError, "Invalid time" do + Time.utc(2020, 5, 21, 24, 0, 1) + end + + expect_raises ArgumentError, "Invalid time" do + Time.utc(2020, 5, 21, 24, 1, 0) + end + end end it "UNIX_EPOCH" do diff --git a/src/time.cr b/src/time.cr index 1f878880b9be..0a7cebedde8b 100644 --- a/src/time.cr +++ b/src/time.cr @@ -410,7 +410,10 @@ struct Time unless 1 <= year <= 9999 && 1 <= month <= 12 && 1 <= day <= Time.days_in_month(year, month) && - 0 <= hour <= 23 && + ( + 0 <= hour <= 23 || + (hour == 24 && minute == 0 && second == 0 && nanosecond == 0) + ) && 0 <= minute <= 59 && 0 <= second <= 59 && 0 <= nanosecond <= 999_999_999 diff --git a/src/time/format/parser.cr b/src/time/format/parser.cr index 3eb1e2576698..b038f837742f 100644 --- a/src/time/format/parser.cr +++ b/src/time/format/parser.cr @@ -33,11 +33,26 @@ struct Time::Format @second = 0 @nanosecond = 0 @pm = false + @hour_is_12 = false @nanosecond_offset = 0_i64 end def time(location : Location? = nil) - @hour += 12 if @pm + if @hour_is_12 + if @hour > 12 + raise ArgumentError.new("Invalid hour for 12-hour clock") + end + + if @pm + @hour += 12 unless @hour == 12 + else + if @hour == 0 + raise ArgumentError.new("Invalid hour for 12-hour clock") + end + + @hour = 0 if @hour == 12 + end + end if unix_seconds = @unix_seconds return Time.unix(unix_seconds) @@ -203,18 +218,22 @@ struct Time::Format end def hour_24_zero_padded + @hour_is_12 = false @hour = consume_number(2) end def hour_24_blank_padded + @hour_is_12 = false @hour = consume_number_blank_padded(2) end def hour_12_zero_padded hour_24_zero_padded + @hour_is_12 = true end def hour_12_blank_padded + @hour_is_12 = true @hour = consume_number_blank_padded(2) end @@ -269,7 +288,7 @@ struct Time::Format string = consume_string case string.downcase when "am" - # skip + @pm = false when "pm" @pm = true else From 6380fa94f9798f440fe92213d7fb6870696063d9 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 28 May 2020 14:14:47 -0300 Subject: [PATCH 068/263] Change IO#write, IO#skip, IO.copy to return Int64 (#9363) --- spec/std/http/request_spec.cr | 4 +- spec/std/http/server/response_spec.cr | 4 +- spec/std/io/io_spec.cr | 8 ++-- spec/std/io/sized_spec.cr | 4 +- spec/support/io.cr | 4 +- src/compress/deflate/writer.cr | 6 +-- src/compress/gzip/writer.cr | 6 +-- src/compress/zip/checksum_writer.cr | 4 +- src/compress/zlib/writer.cr | 6 +-- src/float.cr | 2 +- src/http/content.cr | 8 ++-- src/http/server/response.cr | 4 +- src/http/web_socket/protocol.cr | 6 +-- src/int.cr | 2 +- src/io.cr | 32 +++++++-------- src/io/buffered.cr | 18 ++++----- src/io/byte_format.cr | 58 +++++++++++++-------------- src/io/delimited.cr | 2 +- src/io/encoding.cr | 4 +- src/io/hexdump.cr | 4 +- src/io/memory.cr | 18 ++++----- src/io/multi_writer.cr | 6 +-- src/io/sized.cr | 4 +- src/io/stapled.cr | 10 ++--- src/openssl/digest/digest_io.cr | 4 +- src/string/builder.cr | 10 ++--- 26 files changed, 119 insertions(+), 119 deletions(-) diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index 5c53426a0e9a..7ae9aa34957e 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -6,8 +6,8 @@ private class EmptyIO < IO 0 end - def write(slice : Bytes) : UInt64 - slice.size.to_u64 + def write(slice : Bytes) : Int64 + slice.size.to_i64 end end diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index ff35b45527aa..b42e537d317e 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -12,11 +12,11 @@ private class ReverseResponseOutput < IO def initialize(@output : IO) end - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 slice.reverse_each do |byte| @output.write_byte(byte) end - slice.size.to_u64 + slice.size.to_i64 end def read(slice : Bytes) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 60413fcf995a..b09c096e83ea 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -46,7 +46,7 @@ private class SimpleIOMemory < IO count end - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 count = slice.size new_bytesize = bytesize + count if new_bytesize > @capacity @@ -56,7 +56,7 @@ private class SimpleIOMemory < IO slice.copy_to(@buffer + @bytesize, count) @bytesize += count - slice.size.to_u64 + slice.size.to_i64 end def to_slice @@ -99,8 +99,8 @@ private class OneByOneIO < IO 1 end - def write(slice : Bytes) : UInt64 - slice.size.to_u64 + def write(slice : Bytes) : Int64 + slice.size.to_i64 end end diff --git a/spec/std/io/sized_spec.cr b/spec/std/io/sized_spec.cr index 46cb7d625e4f..a02eafd1556f 100644 --- a/spec/std/io/sized_spec.cr +++ b/spec/std/io/sized_spec.cr @@ -5,8 +5,8 @@ private class NoPeekIO < IO 0 end - def write(bytes : Bytes) : UInt64 - 0u64 + def write(bytes : Bytes) : Int64 + 0i64 end def peek diff --git a/spec/support/io.cr b/spec/support/io.cr index 2eb785856c32..9ce2cd44c944 100644 --- a/spec/support/io.cr +++ b/spec/support/io.cr @@ -8,10 +8,10 @@ class RaiseIOError < IO raise IO::Error.new("...") end - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 @writes += 1 raise IO::Error.new("...") if @raise_on_write - slice.size.to_u64 + slice.size.to_i64 end def flush diff --git a/src/compress/deflate/writer.cr b/src/compress/deflate/writer.cr index b32a3af8b300..0df595422e04 100644 --- a/src/compress/deflate/writer.cr +++ b/src/compress/deflate/writer.cr @@ -43,16 +43,16 @@ class Compress::Deflate::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? @stream.avail_in = slice.size @stream.next_in = slice consume_output LibZ::Flush::NO_FLUSH - slice.size.to_u64 + slice.size.to_i64 end # See `IO#flush`. diff --git a/src/compress/gzip/writer.cr b/src/compress/gzip/writer.cr index 8998ee291e81..19ba228c1fb3 100644 --- a/src/compress/gzip/writer.cr +++ b/src/compress/gzip/writer.cr @@ -68,10 +68,10 @@ class Compress::Gzip::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? flate_io = write_header flate_io.write(slice) @@ -83,7 +83,7 @@ class Compress::Gzip::Writer < IO # uncompressed data size can be bigger. @isize &+= slice.size - slice.size.to_u64 + slice.size.to_i64 end # Flushes data, forcing writing the gzip header if no diff --git a/src/compress/zip/checksum_writer.cr b/src/compress/zip/checksum_writer.cr index 179bfabdcb19..194e96debe9e 100644 --- a/src/compress/zip/checksum_writer.cr +++ b/src/compress/zip/checksum_writer.cr @@ -13,8 +13,8 @@ module Compress::Zip raise IO::Error.new "Can't read from Zip::Writer entry" end - def write(slice : Bytes) : UInt64 - return 0u64 if slice.empty? + def write(slice : Bytes) : Int64 + return 0i64 if slice.empty? @count += slice.size @crc32 = Digest::CRC32.update(slice, @crc32) if @compute_crc32 diff --git a/src/compress/zlib/writer.cr b/src/compress/zlib/writer.cr index b4e4479314a9..f66605d0fbb6 100644 --- a/src/compress/zlib/writer.cr +++ b/src/compress/zlib/writer.cr @@ -44,17 +44,17 @@ class Compress::Zlib::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? write_header unless @wrote_header @flate_io.write(slice) @adler32 = Digest::Adler32.update(slice, @adler32) - slice.size.to_u64 + slice.size.to_i64 end # Flushes data, forcing writing the zlib header if no diff --git a/src/float.cr b/src/float.cr index 23eda5aeb44a..1303ee44888d 100644 --- a/src/float.cr +++ b/src/float.cr @@ -93,7 +93,7 @@ struct Float # Writes this float to the given *io* in the given *format*. # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) : UInt64 + def to_io(io : IO, format : IO::ByteFormat) : Int64 format.encode(self, io) end diff --git a/src/http/content.cr b/src/http/content.cr index 09aa6b2e0017..1719a0c4c0c0 100644 --- a/src/http/content.cr +++ b/src/http/content.cr @@ -41,7 +41,7 @@ module HTTP super end - def skip(bytes_count : Int) : UInt64 + def skip(bytes_count : Int) : Int64 ensure_send_continue super end @@ -73,7 +73,7 @@ module HTTP @io.peek end - def skip(bytes_count : Int) : UInt64 + def skip(bytes_count : Int) : Int64 ensure_send_continue @io.skip(bytes_count) end @@ -164,8 +164,8 @@ module HTTP peek end - def skip(bytes_count : Int) : UInt64 - bytes_count = bytes_count.to_u64 + def skip(bytes_count : Int) : Int64 + bytes_count = bytes_count.to_i64 ensure_send_continue if bytes_count <= @chunk_remaining diff --git a/src/http/server/response.cr b/src/http/server/response.cr index eab4a03a7907..78f273e16c53 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -80,8 +80,8 @@ class HTTP::Server end # See `IO#write(slice)`. - def write(slice : Bytes) : UInt64 - return 0u64 if slice.empty? + def write(slice : Bytes) : Int64 + return 0i64 if slice.empty? @output.write(slice) end diff --git a/src/http/web_socket/protocol.cr b/src/http/web_socket/protocol.cr index 077e99da545b..08e6c2736058 100644 --- a/src/http/web_socket/protocol.cr +++ b/src/http/web_socket/protocol.cr @@ -52,8 +52,8 @@ class HTTP::WebSocket::Protocol @pos = 0 end - def write(slice : Bytes) : UInt64 - return 0u64 if slice.empty? + def write(slice : Bytes) : Int64 + return 0i64 if slice.empty? count = Math.min(@buffer.size - @pos, slice.size) (@buffer + @pos).copy_from(slice.to_unsafe, count) @@ -67,7 +67,7 @@ class HTTP::WebSocket::Protocol write(slice + count) end - slice.size.to_u64 + slice.size.to_i64 end def read(slice : Bytes) diff --git a/src/int.cr b/src/int.cr index ec5efb4f78bc..455892562c2a 100644 --- a/src/int.cr +++ b/src/int.cr @@ -638,7 +638,7 @@ struct Int # Writes this integer to the given *io* in the given *format*. # # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) : UInt64 + def to_io(io : IO, format : IO::ByteFormat) : Int64 format.encode(self, io) end diff --git a/src/io.cr b/src/io.cr index df29eab4e666..f7b5d687bb7a 100644 --- a/src/io.cr +++ b/src/io.cr @@ -29,10 +29,10 @@ require "c/errno" # slice.size # end # -# def write(slice : Bytes) : UInt64 +# def write(slice : Bytes) : Int64 # slice.size.times { |i| @slice[i] = slice[i] } # @slice += slice.size -# slice.size.to_u64 +# slice.size.to_i64 # end # end # @@ -100,7 +100,7 @@ abstract class IO # io.write(slice) # io.to_s # => "abcd" # ``` - abstract def write(slice : Bytes) : UInt64 + abstract def write(slice : Bytes) : Int64 # Closes this `IO`. # @@ -464,7 +464,7 @@ abstract class IO end # Writes a slice of UTF-8 encoded bytes to this `IO`, using the current encoding. - def write_utf8(slice : Bytes) : UInt64 + def write_utf8(slice : Bytes) : Int64 if encoder = encoder() encoder.write(self, slice) else @@ -812,8 +812,8 @@ abstract class IO # io.gets # => "world" # io.skip(1) # raises IO::EOFError # ``` - def skip(bytes_count : Int) : UInt64 - bytes_count = bytes_count.to_u64 + def skip(bytes_count : Int) : Int64 + bytes_count = bytes_count.to_i64 remaining = bytes_count buffer = uninitialized UInt8[4096] while remaining > 0 @@ -826,8 +826,8 @@ abstract class IO # Reads and discards bytes from `self` until there # are no more bytes. - def skip_to_end : UInt64 - bytes_count = 0_u64 + def skip_to_end : Int64 + bytes_count = 0i64 buffer = uninitialized UInt8[4096] while (len = read(buffer.to_slice)) > 0 bytes_count &+= len @@ -842,7 +842,7 @@ abstract class IO # io.write_byte 97_u8 # io.to_s # => "a" # ``` - def write_byte(byte : UInt8) : UInt64 + def write_byte(byte : UInt8) : Int64 x = byte write Slice.new(pointerof(x), 1) end @@ -861,7 +861,7 @@ abstract class IO # io.rewind # io.gets(4) # => "\u{4}\u{3}\u{2}\u{1}" # ``` - def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : UInt64 + def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : Int64 object.to_io(self, format) end @@ -1117,12 +1117,12 @@ abstract class IO # # io2.to_s # => "hello" # ``` - def self.copy(src, dst) : UInt64 + def self.copy(src, dst) : Int64 buffer = uninitialized UInt8[4096] - count = 0_u64 + count = 0_i64 while (len = src.read(buffer.to_slice).to_i32) > 0 dst.write buffer.to_slice[0, len] - count += len + count &+= len end count end @@ -1137,16 +1137,16 @@ abstract class IO # # io2.to_s # => "hel" # ``` - def self.copy(src, dst, limit : Int) : UInt64 + def self.copy(src, dst, limit : Int) : Int64 raise ArgumentError.new("Negative limit") if limit < 0 - limit = limit.to_u64 + limit = limit.to_i64 buffer = uninitialized UInt8[4096] remaining = limit while (len = src.read(buffer.to_slice[0, Math.min(buffer.size, Math.max(remaining, 0))])) > 0 dst.write buffer.to_slice[0, len] - remaining -= len + remaining &-= len end limit - remaining end diff --git a/src/io/buffered.cr b/src/io/buffered.cr index f38b953f3a30..008fb466d9dc 100644 --- a/src/io/buffered.cr +++ b/src/io/buffered.cr @@ -110,8 +110,8 @@ module IO::Buffered end # :nodoc: - def skip(bytes_count : Int) : UInt64 - bytes_count = bytes_count.to_u64 + def skip(bytes_count : Int) : Int64 + bytes_count = bytes_count.to_i64 check_open if bytes_count <= @in_buffer_rem.size @@ -128,18 +128,18 @@ module IO::Buffered end # Buffered implementation of `IO#write(slice)`. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 # NOTE: It returns the bytes written without differencing whether # they are kept in the buffer or sent to the underlying IO. check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? count = slice.size if sync? unbuffered_write(slice) - return slice.size.to_u64 + return slice.size.to_i64 end if flush_on_newline? @@ -156,7 +156,7 @@ module IO::Buffered if count >= @buffer_size flush unbuffered_write slice[0, count] - return slice.size.to_u64 + return slice.size.to_i64 end if count > @buffer_size - @out_count @@ -166,11 +166,11 @@ module IO::Buffered slice.copy_to(out_buffer + @out_count, count) @out_count += count - slice.size.to_u64 + slice.size.to_i64 end # :nodoc: - def write_byte(byte : UInt8) : UInt64 + def write_byte(byte : UInt8) : Int64 check_open if sync? @@ -187,7 +187,7 @@ module IO::Buffered flush end - 1u64 + 1i64 end # Turns on/off `IO` **write** buffering. When *sync* is set to `true`, no buffering diff --git a/src/io/byte_format.cr b/src/io/byte_format.cr index e6f774106bd6..a8d2d8c5a94d 100644 --- a/src/io/byte_format.cr +++ b/src/io/byte_format.cr @@ -33,27 +33,27 @@ # io.to_slice # => Bytes[0x34, 0x12] # ``` module IO::ByteFormat - abstract def encode(int : Int8, io : IO) : UInt64 - abstract def encode(int : UInt8, io : IO) : UInt64 - abstract def encode(int : Int16, io : IO) : UInt64 - abstract def encode(int : UInt16, io : IO) : UInt64 - abstract def encode(int : Int32, io : IO) : UInt64 - abstract def encode(int : UInt32, io : IO) : UInt64 - abstract def encode(int : Int64, io : IO) : UInt64 - abstract def encode(int : UInt64, io : IO) : UInt64 - abstract def encode(int : Int128, io : IO) : UInt64 - abstract def encode(int : UInt128, io : IO) : UInt64 - - abstract def encode(int : Int8, bytes : Bytes) : UInt64 - abstract def encode(int : UInt8, bytes : Bytes) : UInt64 - abstract def encode(int : Int16, bytes : Bytes) : UInt64 - abstract def encode(int : UInt16, bytes : Bytes) : UInt64 - abstract def encode(int : Int32, bytes : Bytes) : UInt64 - abstract def encode(int : UInt32, bytes : Bytes) : UInt64 - abstract def encode(int : Int64, bytes : Bytes) : UInt64 - abstract def encode(int : UInt64, bytes : Bytes) : UInt64 - abstract def encode(int : Int128, bytes : Bytes) : UInt64 - abstract def encode(int : UInt128, bytes : Bytes) : UInt64 + abstract def encode(int : Int8, io : IO) : Int64 + abstract def encode(int : UInt8, io : IO) : Int64 + abstract def encode(int : Int16, io : IO) : Int64 + abstract def encode(int : UInt16, io : IO) : Int64 + abstract def encode(int : Int32, io : IO) : Int64 + abstract def encode(int : UInt32, io : IO) : Int64 + abstract def encode(int : Int64, io : IO) : Int64 + abstract def encode(int : UInt64, io : IO) : Int64 + abstract def encode(int : Int128, io : IO) : Int64 + abstract def encode(int : UInt128, io : IO) : Int64 + + abstract def encode(int : Int8, bytes : Bytes) : Int64 + abstract def encode(int : UInt8, bytes : Bytes) : Int64 + abstract def encode(int : Int16, bytes : Bytes) : Int64 + abstract def encode(int : UInt16, bytes : Bytes) : Int64 + abstract def encode(int : Int32, bytes : Bytes) : Int64 + abstract def encode(int : UInt32, bytes : Bytes) : Int64 + abstract def encode(int : Int64, bytes : Bytes) : Int64 + abstract def encode(int : UInt64, bytes : Bytes) : Int64 + abstract def encode(int : Int128, bytes : Bytes) : Int64 + abstract def encode(int : UInt128, bytes : Bytes) : Int64 abstract def decode(int : Int8.class, io : IO) abstract def decode(int : UInt8.class, io : IO) @@ -77,11 +77,11 @@ module IO::ByteFormat abstract def decode(int : Int128.class, bytes : Bytes) abstract def decode(int : UInt128.class, bytes : Bytes) - def encode(float : Float32, io : IO) : UInt64 + def encode(float : Float32, io : IO) : Int64 encode(float.unsafe_as(Int32), io) end - def encode(float : Float32, bytes : Bytes) : UInt64 + def encode(float : Float32, bytes : Bytes) : Int64 encode(float.unsafe_as(Int32), bytes) end @@ -93,11 +93,11 @@ module IO::ByteFormat decode(Int32, bytes).unsafe_as(Float32) end - def encode(float : Float64, io : IO) : UInt64 + def encode(float : Float64, io : IO) : Int64 encode(float.unsafe_as(Int64), io) end - def encode(float : Float64, bytes : Bytes) : UInt64 + def encode(float : Float64, bytes : Bytes) : Int64 encode(float.unsafe_as(Int64), bytes) end @@ -125,18 +125,18 @@ module IO::ByteFormat {% for type, i in %w(Int8 UInt8 Int16 UInt16 Int32 UInt32 Int64 UInt64 Int128 UInt128) %} {% bytesize = 2 ** (i // 2) %} - def self.encode(int : {{type.id}}, io : IO) : UInt64 + def self.encode(int : {{type.id}}, io : IO) : Int64 buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self io.write(buffer.to_slice) - UInt64.new({{bytesize}}) + Int64.new({{bytesize}}) end - def self.encode(int : {{type.id}}, bytes : Bytes) : UInt64 + def self.encode(int : {{type.id}}, bytes : Bytes) : Int64 buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self buffer.to_slice.copy_to(bytes) - UInt64.new({{bytesize}}) + Int64.new({{bytesize}}) end def self.decode(type : {{type.id}}.class, io : IO) diff --git a/src/io/delimited.cr b/src/io/delimited.cr index 70fc30f98a38..34d23c07066a 100644 --- a/src/io/delimited.cr +++ b/src/io/delimited.cr @@ -107,7 +107,7 @@ class IO::Delimited < IO read_bytes end - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 raise IO::Error.new "Can't write to IO::Delimited" end diff --git a/src/io/encoding.cr b/src/io/encoding.cr index 49a5c086d67a..aab1c720a0db 100644 --- a/src/io/encoding.cr +++ b/src/io/encoding.cr @@ -27,8 +27,8 @@ class IO @closed = false end - def write(io, slice : Bytes) : UInt64 - bytes_written = 0u64 + def write(io, slice : Bytes) : Int64 + bytes_written = 0i64 inbuf_ptr = slice.to_unsafe inbytesleft = LibC::SizeT.new(slice.size) outbuf = uninitialized UInt8[1024] diff --git a/src/io/hexdump.cr b/src/io/hexdump.cr index 35afd9b7727b..cec29b9a0af0 100644 --- a/src/io/hexdump.cr +++ b/src/io/hexdump.cr @@ -32,8 +32,8 @@ class IO::Hexdump < IO end end - def write(buf : Bytes) : UInt64 - return 0u64 if buf.empty? + def write(buf : Bytes) : Int64 + return 0i64 if buf.empty? @io.write(buf).tap do @output.puts buf.hexdump if @write diff --git a/src/io/memory.cr b/src/io/memory.cr index a72839c2cd2e..be97cfe1c64a 100644 --- a/src/io/memory.cr +++ b/src/io/memory.cr @@ -82,13 +82,13 @@ class IO::Memory < IO # See `IO#write(slice)`. Raises if this `IO::Memory` is non-writeable, # or if it's non-resizeable and a resize is needed. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_writeable check_open count = slice.size - return 0u64 if count == 0 + return 0i64 if count == 0 new_bytesize = @pos + count if new_bytesize > @capacity @@ -105,12 +105,12 @@ class IO::Memory < IO @pos += count @bytesize = @pos if @pos > @bytesize - slice.size.to_u64 + slice.size.to_i64 end # See `IO#write_byte`. Raises if this `IO::Memory` is non-writeable, # or if it's non-resizeable and a resize is needed. - def write_byte(byte : UInt8) + def write_byte(byte : UInt8) : Int64 check_writeable check_open @@ -129,7 +129,7 @@ class IO::Memory < IO @pos += 1 @bytesize = @pos if @pos > @bytesize - nil + 1i64 end # :nodoc: @@ -194,8 +194,8 @@ class IO::Memory < IO end # :nodoc: - def skip(bytes_count : Int) : UInt64 - bytes_count = bytes_count.to_u64 + def skip(bytes_count : Int) : Int64 + bytes_count = bytes_count.to_i64 check_open available = @bytesize - @pos @@ -208,12 +208,12 @@ class IO::Memory < IO end # :nodoc: - def skip_to_end : UInt64 + def skip_to_end : Int64 check_open skipped = @bytesize - @pos @pos = @bytesize - skipped.to_u64 + skipped.to_i64 end # :nodoc: diff --git a/src/io/multi_writer.cr b/src/io/multi_writer.cr index 35e7682f961c..7ddf7f42b4a4 100644 --- a/src/io/multi_writer.cr +++ b/src/io/multi_writer.cr @@ -29,14 +29,14 @@ class IO::MultiWriter < IO @writers = writers.map(&.as(IO)).to_a end - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? @writers.each { |writer| writer.write(slice) } - slice.size.to_u64 + slice.size.to_i64 end def read(slice : Bytes) diff --git a/src/io/sized.cr b/src/io/sized.cr index ae9b88cdb9fe..d5ae6e506cf3 100644 --- a/src/io/sized.cr +++ b/src/io/sized.cr @@ -61,8 +61,8 @@ class IO::Sized < IO peek end - def skip(bytes_count : Int) : UInt64 - bytes_count = bytes_count.to_u64 + def skip(bytes_count : Int) : Int64 + bytes_count = bytes_count.to_i64 check_open if bytes_count <= @read_remaining diff --git a/src/io/stapled.cr b/src/io/stapled.cr index 52c1f7fb69f1..1d30bdc2fe98 100644 --- a/src/io/stapled.cr +++ b/src/io/stapled.cr @@ -51,31 +51,31 @@ class IO::Stapled < IO end # Skips `reader`. - def skip(bytes_count : Int) : UInt64 + def skip(bytes_count : Int) : Int64 check_open @reader.skip(bytes_count) end # Skips `reader`. - def skip_to_end : UInt64 + def skip_to_end : Int64 check_open @reader.skip_to_end end # Writes a byte to `writer`. - def write_byte(byte : UInt8) : Nil + def write_byte(byte : UInt8) : Int64 check_open @writer.write_byte(byte) end # Writes a slice to `writer`. - def write(slice : Bytes) : UInt64 + def write(slice : Bytes) : Int64 check_open - return 0u64 if slice.empty? + return 0i64 if slice.empty? @writer.write(slice) end diff --git a/src/openssl/digest/digest_io.cr b/src/openssl/digest/digest_io.cr index 2a3b002ad97a..b04942736f1f 100644 --- a/src/openssl/digest/digest_io.cr +++ b/src/openssl/digest/digest_io.cr @@ -42,8 +42,8 @@ module OpenSSL read_bytes end - def write(slice : Bytes) : UInt64 - return 0u64 if slice.empty? + def write(slice : Bytes) : Int64 + return 0i64 if slice.empty? if @mode.write? digest_algorithm.update(slice) diff --git a/src/string/builder.cr b/src/string/builder.cr index c117a45fec4f..7ba67dd2e029 100644 --- a/src/string/builder.cr +++ b/src/string/builder.cr @@ -38,8 +38,8 @@ class String::Builder < IO raise "Not implemented" end - def write(slice : Bytes) : UInt64 - return 0u64 if slice.empty? + def write(slice : Bytes) : Int64 + return 0i64 if slice.empty? count = slice.size new_bytesize = real_bytesize + count @@ -50,10 +50,10 @@ class String::Builder < IO slice.copy_to(@buffer + real_bytesize, count) @bytesize += count - slice.size.to_u64 + slice.size.to_i64 end - def write_byte(byte : UInt8) + def write_byte(byte : UInt8) : Int64 new_bytesize = real_bytesize + 1 if new_bytesize > @capacity resize_to_capacity(Math.pw2ceil(new_bytesize)) @@ -63,7 +63,7 @@ class String::Builder < IO @bytesize += 1 - nil + 1i64 end def buffer From d60a1db55f59a8962edb24eecfb955864a66fb9c Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 28 May 2020 19:15:39 +0200 Subject: [PATCH 069/263] Add Process.quote and fix shell usages in the compiler (#9043) * Add a method to produce a command line string from args And apply it throughout the compiler for better safety. * Quote flags in link.cr --- spec/compiler/compiler_spec.cr | 4 +- spec/spec_helper.cr | 6 +- spec/std/io/file_descriptor_spec.cr | 4 +- spec/std/process_spec.cr | 38 ++++++++ src/compiler/crystal/codegen/cache_dir.cr | 2 +- src/compiler/crystal/codegen/link.cr | 8 +- src/compiler/crystal/compiler.cr | 11 ++- src/compiler/crystal/macros/methods.cr | 2 +- src/crystal/system/win32/process.cr | 31 +------ src/process/shell.cr | 105 ++++++++++++++++++++++ 10 files changed, 165 insertions(+), 46 deletions(-) create mode 100644 src/process/shell.cr diff --git a/spec/compiler/compiler_spec.cr b/spec/compiler/compiler_spec.cr index 56746736b7b1..6854302095fc 100644 --- a/spec/compiler/compiler_spec.cr +++ b/spec/compiler/compiler_spec.cr @@ -12,7 +12,7 @@ describe "Compiler" do File.exists?(path).should be_true - `#{path}`.should eq("Hello!") + `#{Process.quote(path)}`.should eq("Hello!") end end @@ -23,7 +23,7 @@ describe "Compiler" do File.exists?(path).should be_true - `#{path}`.should eq("Hello!") + `#{Process.quote(path)}`.should eq("Hello!") end end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index c35ef4e87a87..363da3952707 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -260,7 +260,7 @@ def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug:: apply_program_flags(compiler.flags) compiler.compile Compiler::Source.new("spec", code), output_filename - output = `#{output_filename}` + output = `#{Process.quote(output_filename)}` File.delete(output_filename) SpecRunOutput.new(output) @@ -277,7 +277,7 @@ def build(code) binary_file = File.tempname("build_and_run_bin") - `bin/crystal build #{encode_program_flags} #{code_file.path.inspect} -o #{binary_file.path.inspect}` + `bin/crystal build #{encode_program_flags} #{Process.quote(code_file.path.to_s)} -o #{Process.quote(binary_file.path.to_s)}` File.exists?(binary_file).should be_true yield binary_file @@ -301,7 +301,7 @@ def test_c(c_code, crystal_code) begin File.write(c_filename, c_code) - `#{Crystal::Compiler::CC} #{c_filename} -c -o #{o_filename}`.should be_truthy + `#{Crystal::Compiler::CC} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy yield run(%( require "prelude" diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index 566b8236ddc0..718a47acf5f6 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -4,8 +4,8 @@ describe IO::FileDescriptor do it "reopen STDIN with the right mode" do code = %q(puts "#{STDIN.blocking} #{STDIN.info.type}") compile_source(code) do |binpath| - `#{binpath} < #{binpath}`.chomp.should eq("true File") - `echo "" | #{binpath}`.chomp.should eq("false Pipe") + `#{Process.quote(binpath)} < #{Process.quote(binpath)}`.chomp.should eq("true File") + `echo "" | #{Process.quote(binpath)}`.chomp.should eq("false Pipe") end end end diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 3f82c98190ef..ab89a43b9314 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -425,4 +425,42 @@ describe Process do Process.find_executable("some_very_unlikely_file_to_exist").should be_nil end end + + describe "quote_posix" do + it { Process.quote_posix("").should eq "''" } + it { Process.quote_posix(" ").should eq "' '" } + it { Process.quote_posix("$hi").should eq "'$hi'" } + it { Process.quote_posix(orig = "aZ5+,-./:=@_").should eq orig } + it { Process.quote_posix(orig = "cafe").should eq orig } + it { Process.quote_posix("café").should eq "'café'" } + it { Process.quote_posix("I'll").should eq %('I'"'"'ll') } + it { Process.quote_posix("'").should eq %(''"'"'') } + it { Process.quote_posix("\\").should eq "'\\'" } + + context "join" do + it { Process.quote_posix([] of String).should eq "" } + it { Process.quote_posix(["my file.txt", "another.txt"]).should eq "'my file.txt' another.txt" } + it { Process.quote_posix(["foo ", "", " ", " bar"]).should eq "'foo ' '' ' ' ' bar'" } + it { Process.quote_posix(["foo'", "\"bar"]).should eq %('foo'"'"'' '"bar') } + end + end + + describe "quote_windows" do + it { Process.quote_windows("").should eq %("") } + it { Process.quote_windows(" ").should eq %(" ") } + it { Process.quote_windows(orig = "%hi%").should eq orig } + it { Process.quote_windows(%q(C:\"foo" project.txt)).should eq %q("C:\\\"foo\" project.txt") } + it { Process.quote_windows(%q(C:\"foo"_project.txt)).should eq %q(C:\\\"foo\"_project.txt) } + it { Process.quote_windows(%q(C:\Program Files\Foo Bar\foobar.exe)).should eq %q("C:\Program Files\Foo Bar\foobar.exe") } + it { Process.quote_windows(orig = "café").should eq orig } + it { Process.quote_windows(%(")).should eq %q(\") } + it { Process.quote_windows(%q(a\\b\ c\)).should eq %q("a\\b\ c\\") } + it { Process.quote_windows(orig = %q(a\\b\c\)).should eq orig } + + context "join" do + it { Process.quote_windows([] of String).should eq "" } + it { Process.quote_windows(["my file.txt", "another.txt"]).should eq %("my file.txt" another.txt) } + it { Process.quote_windows(["foo ", "", " ", " bar"]).should eq %("foo " "" " " " bar") } + end + end end diff --git a/src/compiler/crystal/codegen/cache_dir.cr b/src/compiler/crystal/codegen/cache_dir.cr index 72aca6ca13da..ea0184a3f1d9 100644 --- a/src/compiler/crystal/codegen/cache_dir.cr +++ b/src/compiler/crystal/codegen/cache_dir.cr @@ -119,7 +119,7 @@ module Crystal .sort_by! { |dir| File.info?(dir).try(&.modification_time) || Time.unix(0) } .reverse! .skip(10) - .each { |name| `rm -rf "#{name}"` rescue nil } + .each { |name| `rm -rf -- #{Process.quote(name)}` rescue nil } end private def gather_cache_entries(dir) diff --git a/src/compiler/crystal/codegen/link.cr b/src/compiler/crystal/codegen/link.cr index 6bef92573968..9eea3ae82fde 100644 --- a/src/compiler/crystal/codegen/link.cr +++ b/src/compiler/crystal/codegen/link.cr @@ -107,7 +107,7 @@ module Crystal end if libname = ann.lib - flags << ' ' << libname << ".lib" + flags << ' ' << Process.quote_windows("#{libname}.lib") end end end @@ -123,7 +123,7 @@ module Crystal # Add CRYSTAL_LIBRARY_PATH locations, so the linker preferentially # searches user-given library paths. CrystalLibraryPath.paths.each do |path| - flags << "'-L#{path}'" + flags << Process.quote_posix("-L#{path}") end link_annotations.reverse_each do |ann| @@ -138,11 +138,11 @@ module Crystal elsif (lib_name = ann.lib) && (flag = pkg_config(lib_name, static_build)) flags << flag elsif (lib_name = ann.lib) - flags << "-l#{lib_name}" + flags << Process.quote_posix("-l#{lib_name}") end if framework = ann.framework - flags << "-framework" << framework + flags << "-framework" << Process.quote_posix(framework) end end diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index fc0fee3568d2..c017281eb54a 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -318,7 +318,7 @@ module Crystal end private def print_command(command, args) - stdout.puts command.sub(%("${@}"), args && args.join(" ")) + stdout.puts command.sub(%("${@}"), args && Process.quote(args)) end private def linker_command(program : Program, object_names, output_filename, output_dir, expand = false) @@ -327,7 +327,10 @@ module Crystal # Execute and expand `subcommands`. lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } if expand - args = %(/nologo #{object_names.join(" ")} "/Fe#{output_filename}" #{lib_flags} #{@link_flags}) + object_arg = Process.quote_windows(object_names) + output_arg = Process.quote_windows("/Fe#{output_filename}") + + args = %(/nologo #{object_arg} #{output_arg} #{lib_flags} #{@link_flags}) cmd = "#{CL} #{args}" if cmd.to_utf16.size > 32000 @@ -339,7 +342,7 @@ module Crystal args_filename = "#{output_dir}/linker_args.txt" File.write(args_filename, args_bytes) - cmd = "#{CL} @#{args_filename}" + cmd = "#{CL} #{Process.quote_windows("@" + args_filename)}" end {cmd, nil} @@ -360,7 +363,7 @@ module Crystal link_flags = @link_flags || "" link_flags += " -rdynamic" - { %(#{cc} "${@}" -o '#{output_filename}' #{link_flags} #{program.lib_flags}), object_names } + { %(#{cc} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags}), object_names } end end diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index e11e045c76ef..3a184b72bee2 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -265,7 +265,7 @@ module Crystal if result.status.success? @last = MacroId.new(result.stdout) else - command = "#{original_filename} #{run_args.map(&.inspect).join " "}" + command = "#{Process.quote(original_filename)} #{Process.quote(run_args)}" message = IO::Memory.new message << "Error executing run (exit code: #{result.status.exit_code}): #{command}\n" diff --git a/src/crystal/system/win32/process.cr b/src/crystal/system/win32/process.cr index 44c240d15086..20a0c727837e 100644 --- a/src/crystal/system/win32/process.cr +++ b/src/crystal/system/win32/process.cr @@ -1,4 +1,5 @@ require "c/processthreadsapi" +require "process/shell" struct Crystal::System::Process getter pid : LibC::DWORD @@ -153,35 +154,7 @@ struct Crystal::System::Process else command_args = [command] command_args.concat(args) if args - String.build { |io| args_to_string(command_args, io) } - end - end - - private def self.args_to_string(args, io : IO) - args.join(io, ' ') do |arg| - quotes = arg.empty? || arg.includes?(' ') || arg.includes?('\t') - - io << '"' if quotes - - slashes = 0 - arg.each_char do |c| - case c - when '\\' - slashes += 1 - when '"' - (slashes + 1).times { io << '\\' } - slashes = 0 - else - slashes = 0 - end - - io << c - end - - if quotes - slashes.times { io << '\\' } - io << '"' - end + ::Process.quote_windows(command_args) end end diff --git a/src/process/shell.cr b/src/process/shell.cr new file mode 100644 index 000000000000..85a0050c6b02 --- /dev/null +++ b/src/process/shell.cr @@ -0,0 +1,105 @@ +class Process + # Converts a sequence of strings to one joined string with each argument shell-quoted. + # + # This is then safe to pass as part of the command when using `shell: true` or `system()`. + # + # NOTE: The actual return value is system-dependent, so it mustn't be relied on in other contexts. + # See also: `quote_posix`. + # + # ``` + # files = ["my file.txt", "another.txt"] + # `grep -E 'fo+' -- #{Process.quote(files)}` + # ``` + def self.quote(args : Enumerable(String)) : String + {% if flag?(:win32) %} + quote_windows(args) + {% else %} + quote_posix(args) + {% end %} + end + + # Shell-quotes one item, same as `quote({arg})`. + def self.quote(arg : String) : String + quote({arg}) + end + + # Converts a sequence of strings to one joined string with each argument shell-quoted. + # + # This is then safe to pass to a POSIX shell. + # + # ``` + # files = ["my file.txt", "another.txt"] + # Process.quote_posix(files) # => "'my file.txt' another.txt" + # ``` + def self.quote_posix(args : Enumerable(String)) : String + args.join(' ') do |arg| + if arg.empty? + "''" + elsif arg.matches? %r([^a-zA-Z0-9%+,\-./:=@_]) # not all characters are safe, needs quoting + "'" + arg.gsub("'", %('"'"')) + "'" # %(foo'ba#r) becomes %('foo'"'"'ba#r') + else + arg + end + end + end + + # Shell-quotes one item, same as `quote_posix({arg})`. + def self.quote_posix(arg : String) : String + quote_posix({arg}) + end + + # :nodoc: + # + # Converts a sequence of strings to one joined string with each argument shell-quoted. + # + # This is then safe to pass Windows API CreateProcess. + # + # NOTE: This is **not** safe to pass to the CMD shell. + # + # ``` + # files = ["my file.txt", "another.txt"] + # Process.quote_windows(files) # => %("my file.txt" another.txt) + # ``` + def self.quote_windows(args : Enumerable(String)) : String + String.build { |io| quote_windows(io, args) } + end + + private def self.quote_windows(io : IO, args) + args.join(' ', io) do |arg| + need_quotes = arg.empty? || arg.includes?(' ') || arg.includes?('\t') + + io << '"' if need_quotes + + slashes = 0 + arg.each_char do |c| + case c + when '\\' + slashes += 1 + when '"' + (slashes + 1).times { io << '\\' } + slashes = 0 + else + slashes = 0 + end + + io << c + end + + if need_quotes + slashes.times { io << '\\' } + io << '"' + end + end + end + + # :nodoc: + # + # Shell-quotes one item, same as `quote_windows({arg})`. + # + # ``` + # Process.quote_windows(%q(C:\"foo" project.txt)) # => %q("C:\\\"foo\" project.txt") + # ``` + def self.quote_windows(arg : String) : String + quote_windows({arg}) + end +end From b48471e5526286903fd3636d5bc943af91f7338e Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Thu, 28 May 2020 23:34:37 +0200 Subject: [PATCH 070/263] Fix typo in "deserializing" (#9368) --- src/yaml/from_yaml.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yaml/from_yaml.cr b/src/yaml/from_yaml.cr index da46d14aea24..12fa47ba64b2 100644 --- a/src/yaml/from_yaml.cr +++ b/src/yaml/from_yaml.cr @@ -242,7 +242,7 @@ def Union.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) {% end %} {% end %} - node.raise("Error deserailizing alias") + node.raise("Error deserializing alias") end {% begin %} From 0dd0c08e801551af4f72e977af011c8b83e14966 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 28 May 2020 19:25:20 -0300 Subject: [PATCH 071/263] Enumerable#join args swap. Follow up #9043 (#9369) --- src/process/shell.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/process/shell.cr b/src/process/shell.cr index 85a0050c6b02..ee69eb636f78 100644 --- a/src/process/shell.cr +++ b/src/process/shell.cr @@ -65,7 +65,7 @@ class Process end private def self.quote_windows(io : IO, args) - args.join(' ', io) do |arg| + args.join(io, ' ') do |arg| need_quotes = arg.empty? || arg.includes?(' ') || arg.includes?('\t') io << '"' if need_quotes From 290a5f9d1b7ca3a87b75989f845e82854ce9ea9f Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 28 May 2020 19:25:32 -0300 Subject: [PATCH 072/263] CI: Use --json-config-url doc option (#9370) --- bin/ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ci b/bin/ci index 9a3bfacc108e..f6870d7b5bc7 100755 --- a/bin/ci +++ b/bin/ci @@ -93,7 +93,7 @@ build() { with_build_env 'make docs threads=1' ;; *) - with_build_env 'make crystal std_spec compiler_spec docs threads=1 junit_output=.junit/spec.xml' + with_build_env 'make crystal std_spec compiler_spec docs threads=1 junit_output=.junit/spec.xml DOCS_OPTIONS="--json-config-url=/api/versions.json"' ;; esac From 9d84d377f2f6d2f27a829db9a2c5ba853fe67884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 29 May 2020 18:00:06 +0200 Subject: [PATCH 073/263] Refactor CrystalPath::Error (#9359) * Add CrystalPath::NotFoundError Introduces a dedicated error type for path lookup fails which contains information about the path being looked up. This separates creating the actual compiler error which then happens in SemanticVisitor#visit(Require). * Fix typos Thanks @sija Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Sijawusz Pur Rahnama --- .../crystal_path/crystal_path_spec.cr | 44 +++++++------------ spec/compiler/semantic/require_spec.cr | 35 +++++++++++++-- src/compiler/crystal/crystal_path.cr | 29 ++++-------- src/compiler/crystal/macros/macros.cr | 2 +- .../crystal/semantic/semantic_visitor.cr | 18 ++++++++ 5 files changed, 75 insertions(+), 53 deletions(-) diff --git a/spec/compiler/crystal_path/crystal_path_spec.cr b/spec/compiler/crystal_path/crystal_path_spec.cr index 6f2398f2743e..cdcf5ba47256 100644 --- a/spec/compiler/crystal_path/crystal_path_spec.cr +++ b/spec/compiler/crystal_path/crystal_path_spec.cr @@ -4,19 +4,23 @@ require "../../support/env" private def assert_finds(search, results, relative_to = nil, path = __DIR__, file = __FILE__, line = __LINE__) it "finds #{search.inspect}", file, line do crystal_path = Crystal::CrystalPath.new(path) - relative_to = "#{__DIR__}/#{relative_to}" if relative_to - results = results.map { |result| "#{__DIR__}/#{result}" } - matches = crystal_path.find search, relative_to: relative_to - matches.should eq(results) + results = results.map { |result| File.join(__DIR__, result) } + Dir.cd(__DIR__) do + matches = crystal_path.find search, relative_to: relative_to + matches.should eq(results), file: file, line: line + end end end -private def assert_doesnt_find(search, relative_to = nil, path = __DIR__, file = __FILE__, line = __LINE__) +private def assert_doesnt_find(search, relative_to = nil, path = __DIR__, expected_relative_to = nil, file = __FILE__, line = __LINE__) it "doesn't finds #{search.inspect}", file, line do crystal_path = Crystal::CrystalPath.new(path) - relative_to = "#{__DIR__}/#{relative_to}" if relative_to - expect_raises Exception, /can't find file/ do - crystal_path.find search, relative_to: relative_to + Dir.cd(__DIR__) do + error = expect_raises Crystal::CrystalPath::NotFoundError do + crystal_path.find search, relative_to: relative_to + end + error.relative_to.should eq(expected_relative_to), file: file, line: line + error.filename.should eq(search), file: file, line: line end end end @@ -88,33 +92,17 @@ describe Crystal::CrystalPath do assert_doesnt_find "file_two.cr" assert_doesnt_find "test_folder/file_three.cr" - assert_doesnt_find "test_folder/*", relative_to: "#{__DIR__}/test_files/file_one.cr" + assert_doesnt_find "test_folder/*", relative_to: Path[__DIR__, "test_files", "file_one.cr"].to_s, expected_relative_to: Path[__DIR__, "test_files"].to_s assert_doesnt_find "test_files/missing_file.cr" assert_doesnt_find __FILE__[1..-1], path: ":" # Don't find in CRYSTAL_PATH if the path is relative (#4742) - assert_doesnt_find "./crystal_path_spec", relative_to: "test_files/file_one.cr" - assert_doesnt_find "./crystal_path_spec.cr", relative_to: "test_files/file_one.cr" + assert_doesnt_find "./crystal_path_spec", relative_to: Path["test_files", "file_one.cr"].to_s, expected_relative_to: Path["test_files"].to_s + assert_doesnt_find "./crystal_path_spec.cr", relative_to: Path["test_files", "file_one.cr"].to_s, expected_relative_to: Path["test_files"].to_s assert_doesnt_find "../crystal_path/test_files/file_one" # Don't find relative filenames in src or shards - assert_doesnt_find "../../src/file_three", relative_to: "test_files/test_folder/test_folder.cr" - - it "prints an explanatory message for non-relative requires" do - crystal_path = Crystal::CrystalPath.new(__DIR__) - ex = expect_raises Exception, /If you're trying to require a shard/ do - crystal_path.find "non_existent", relative_to: __DIR__ - end - end - - it "doesn't print an explanatory message for relative requires" do - crystal_path = Crystal::CrystalPath.new(__DIR__) - ex = expect_raises Exception, /can't find file/ do - crystal_path.find "./non_existent", relative_to: __DIR__ - end - - ex.message.not_nil!.should_not contain "If you're trying to require a shard" - end + assert_doesnt_find "../../src/file_three", relative_to: Path["test_files", "test_folder", "test_folder.cr"].to_s, expected_relative_to: Path["test_files", "test_folder"].to_s it "includes 'lib' by default" do with_env("CRYSTAL_PATH": nil) do diff --git a/spec/compiler/semantic/require_spec.cr b/spec/compiler/semantic/require_spec.cr index 82d9c1210721..bfbffb43f015 100644 --- a/spec/compiler/semantic/require_spec.cr +++ b/spec/compiler/semantic/require_spec.cr @@ -1,10 +1,37 @@ require "../../spec_helper" describe "Semantic: require" do - it "raises crystal exception if can't find require (#7385)" do - node = parse(%(require "file_that_doesnt_exist")) - expect_raises ::Crystal::Exception do - semantic(node) + describe "file not found" do + it "require" do + error = assert_error %(require "file_that_doesnt_exist"), + "can't find file 'file_that_doesnt_exist'", + inject_primitives: false + + error.message.not_nil!.should contain "If you're trying to require a shard:" + end + + it "relative require" do + error = assert_error %(require "./file_that_doesnt_exist"), + "can't find file './file_that_doesnt_exist'", + inject_primitives: false + + error.message.not_nil!.should_not contain "If you're trying to require a shard:" + end + + it "wildcard" do + error = assert_error %(require "file_that_doesnt_exist/*"), + "can't find file 'file_that_doesnt_exist/*'", + inject_primitives: false + + error.message.not_nil!.should contain "If you're trying to require a shard:" + end + + it "relative wildcard" do + error = assert_error %(require "./file_that_doesnt_exist/*"), + "can't find file './file_that_doesnt_exist/*'", + inject_primitives: false + + error.message.not_nil!.should_not contain "If you're trying to require a shard:" end end end diff --git a/src/compiler/crystal/crystal_path.cr b/src/compiler/crystal/crystal_path.cr index 9e3190618370..828e04a9b312 100644 --- a/src/compiler/crystal/crystal_path.cr +++ b/src/compiler/crystal/crystal_path.cr @@ -3,7 +3,12 @@ require "./exception" module Crystal struct CrystalPath - class Error < LocationlessException + class NotFoundError < LocationlessException + getter filename + getter relative_to + + def initialize(@filename : String, @relative_to : String?) + end end private DEFAULT_LIB_PATH = "lib" @@ -46,7 +51,9 @@ module Crystal result = find_in_crystal_path(filename) end - cant_find_file filename, relative_to unless result + unless result + raise NotFoundError.new(filename, relative_to) + end result = [result] if result.is_a?(String) result @@ -159,23 +166,5 @@ module Crystal nil end - - private def cant_find_file(filename, relative_to) - error = "can't find file '#{filename}'" - - if filename.starts_with? '.' - error += " relative to '#{relative_to}'" if relative_to - else - error = <<-NOTE - #{error} - - If you're trying to require a shard: - - Did you remember to run `shards install`? - - Did you make sure you're running the compiler in the same directory as your shard.yml? - NOTE - end - - raise Error.new(error) - end end end diff --git a/src/compiler/crystal/macros/macros.cr b/src/compiler/crystal/macros/macros.cr index c93c4392ec51..f16460c5b86f 100644 --- a/src/compiler/crystal/macros/macros.cr +++ b/src/compiler/crystal/macros/macros.cr @@ -160,7 +160,7 @@ class Crystal::Program begin files = @program.find_in_path(recorded_require.filename, recorded_require.relative_to) required_files.concat(files) if files - rescue Crystal::CrystalPath::Error + rescue Crystal::CrystalPath::NotFoundError # Maybe the file is gone next end diff --git a/src/compiler/crystal/semantic/semantic_visitor.cr b/src/compiler/crystal/semantic/semantic_visitor.cr index 0a92fea1d212..966475700650 100644 --- a/src/compiler/crystal/semantic/semantic_visitor.cr +++ b/src/compiler/crystal/semantic/semantic_visitor.cr @@ -68,6 +68,24 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor node.expanded = expanded node.bind_to(expanded) false + rescue ex : CrystalPath::NotFoundError + message = "can't find file '#{ex.filename}'" + notes = [] of String + + # FIXME: as(String) should not be necessary + if ex.filename.as(String).starts_with? '.' + if relative_to + message += " relative to '#{relative_to}'" + end + else + notes << <<-NOTE + If you're trying to require a shard: + - Did you remember to run `shards install`? + - Did you make sure you're running the compiler in the same directory as your shard.yml? + NOTE + end + + node.raise "#{message}\n\n#{notes.join("\n")}" rescue ex : Crystal::Exception node.raise "while requiring \"#{node.string}\"", ex rescue ex From f584ff64eefb61c17fb407b4c77df8612532444f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 29 May 2020 18:02:40 +0200 Subject: [PATCH 074/263] Refactor spec_helper (#9367) * Rename automcatic_cast.cr to automatic_cast_spec.cr * Remove unused assert_no_warnings helper * Move assert_after_cleanup to cleanup_spec * Refactor spec helper codegen to reuse semantic helper * Refactor spec helper assert_error to reuse semantic helper * Remove spec helper semantic_result and reuse semantic helper * Fix line numbers with inject_primitives in semantic * Fix line numbers with inject_primitives in assert_warning * fixup! Fix line numbers with inject_primitives in assert_warning * Remove useless inject_primitives from assert_warning --- spec/compiler/codegen/warnings_spec.cr | 97 +++++++++---------- spec/compiler/semantic/abstract_def_spec.cr | 30 +++--- ...tomatic_cast.cr => automatic_cast_spec.cr} | 0 spec/compiler/semantic/cleanup_spec.cr | 6 ++ spec/compiler/semantic/concrete_types_spec.cr | 2 +- spec/compiler/semantic/lib_spec.cr | 12 +-- spec/compiler/semantic/warnings_spec.cr | 22 ++--- spec/spec_helper.cr | 89 +++++++---------- spec/support/syntax.cr | 5 +- 9 files changed, 124 insertions(+), 139 deletions(-) rename spec/compiler/semantic/{automatic_cast.cr => automatic_cast_spec.cr} (100%) diff --git a/spec/compiler/codegen/warnings_spec.cr b/spec/compiler/codegen/warnings_spec.cr index 18fcc137b1ad..c0291e9826ce 100644 --- a/spec/compiler/codegen/warnings_spec.cr +++ b/spec/compiler/codegen/warnings_spec.cr @@ -2,29 +2,29 @@ require "../spec_helper" describe "Code gen: warnings" do it "detects top-level deprecated methods" do - assert_warning %( + assert_warning <<-CR, @[Deprecated("Do not use me")] def foo end foo - ), "warning in line 6\nWarning: Deprecated top-level foo. Do not use me", - inject_primitives: false + CR + "warning in line 5\nWarning: Deprecated top-level foo. Do not use me" end it "deprecation reason is optional" do - assert_warning %( + assert_warning <<-CR, @[Deprecated] def foo end foo - ), "warning in line 6\nWarning: Deprecated top-level foo.", - inject_primitives: false + CR + "warning in line 5\nWarning: Deprecated top-level foo." end it "detects deprecated instance methods" do - assert_warning %( + assert_warning <<-CR, class Foo @[Deprecated("Do not use me")] def m @@ -32,12 +32,12 @@ describe "Code gen: warnings" do end Foo.new.m - ), "warning in line 8\nWarning: Deprecated Foo#m. Do not use me", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo#m. Do not use me" end it "detects deprecated class methods" do - assert_warning %( + assert_warning <<-CR, class Foo @[Deprecated("Do not use me")] def self.m @@ -45,12 +45,12 @@ describe "Code gen: warnings" do end Foo.m - ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" end it "detects deprecated generic instance methods" do - assert_warning %( + assert_warning <<-CR, class Foo(T) @[Deprecated("Do not use me")] def m @@ -58,12 +58,12 @@ describe "Code gen: warnings" do end Foo(Int32).new.m - ), "warning in line 8\nWarning: Deprecated Foo(Int32)#m. Do not use me", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo(Int32)#m. Do not use me" end it "detects deprecated generic class methods" do - assert_warning %( + assert_warning <<-CR, class Foo(T) @[Deprecated("Do not use me")] def self.m @@ -71,12 +71,12 @@ describe "Code gen: warnings" do end Foo(Int32).m - ), "warning in line 8\nWarning: Deprecated Foo(Int32).m. Do not use me", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo(Int32).m. Do not use me" end it "detects deprecated module methods" do - assert_warning %( + assert_warning <<-CR, module Foo @[Deprecated("Do not use me")] def self.m @@ -84,23 +84,23 @@ describe "Code gen: warnings" do end Foo.m - ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" end it "detects deprecated methods with named arguments" do - assert_warning %( + assert_warning <<-CR, @[Deprecated] def foo(*, a) end foo(a: 2) - ), "warning in line 6\nWarning: Deprecated top-level foo:a.", - inject_primitives: false + CR + "warning in line 5\nWarning: Deprecated top-level foo:a." end it "detects deprecated initialize" do - assert_warning %( + assert_warning <<-CR, class Foo @[Deprecated] def initialize @@ -108,12 +108,12 @@ describe "Code gen: warnings" do end Foo.new - ), "warning in line 8\nWarning: Deprecated Foo.new.", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo.new." end it "detects deprecated initialize with named arguments" do - assert_warning %( + assert_warning <<-CR, class Foo @[Deprecated] def initialize(*, a) @@ -121,12 +121,12 @@ describe "Code gen: warnings" do end Foo.new(a: 2) - ), "warning in line 8\nWarning: Deprecated Foo.new:a.", - inject_primitives: false + CR + "warning in line 7\nWarning: Deprecated Foo.new:a." end it "informs warnings once per call site location (a)" do - warning_failures = warnings_result %( + warning_failures = warnings_result <<-CR class Foo @[Deprecated("Do not use me")] def m @@ -139,13 +139,12 @@ describe "Code gen: warnings" do Foo.new.b Foo.new.b - ), inject_primitives: false - + CR warning_failures.size.should eq(1) end it "informs warnings once per call site location (b)" do - warning_failures = warnings_result %( + warning_failures = warnings_result <<-CR class Foo @[Deprecated("Do not use me")] def m @@ -154,13 +153,13 @@ describe "Code gen: warnings" do Foo.new.m Foo.new.m - ), inject_primitives: false + CR warning_failures.size.should eq(2) end it "informs warnings once per yield" do - warning_failures = warnings_result %( + warning_failures = warnings_result <<-CR class Foo @[Deprecated("Do not use me")] def m @@ -173,13 +172,13 @@ describe "Code gen: warnings" do end twice { Foo.new.m } - ), inject_primitives: false + CR warning_failures.size.should eq(1) end it "informs warnings once per target type" do - warning_failures = warnings_result %( + warning_failures = warnings_result <<-CR class Foo(T) @[Deprecated("Do not use me")] def m @@ -192,7 +191,7 @@ describe "Code gen: warnings" do Foo(Int32).new.b Foo(Int64).new.b - ), inject_primitives: false + CR warning_failures.size.should eq(2) end @@ -210,13 +209,13 @@ describe "Code gen: warnings" do output_filename = File.join(path, "main") Dir.cd(path) do - File.write main_filename, %( + File.write main_filename, <<-CR require "./lib/foo" bar foo - ) - File.write File.join(path, "lib", "foo.cr"), %( + CR + File.write File.join(path, "lib", "foo.cr"), <<-CR @[Deprecated("Do not use me")] def foo end @@ -224,7 +223,7 @@ describe "Code gen: warnings" do def bar foo end - ) + CR compiler = create_spec_compiler compiler.warnings = Warnings::All @@ -238,29 +237,29 @@ describe "Code gen: warnings" do end it "errors if invalid argument type" do - assert_error %( + assert_error <<-CR, @[Deprecated(42)] def foo end - ), + CR "Error: first argument must be a String" end it "errors if too many arguments" do - assert_error %( + assert_error <<-CR, @[Deprecated("Do not use me", "extra arg")] def foo end - ), + CR "Error: wrong number of deprecated annotation arguments (given 2, expected 1)" end it "errors if invalid named arguments" do - assert_error %( + assert_error <<-CR, @[Deprecated(invalid: "Do not use me")] def foo end - ), + CR "Error: too many named arguments (given 1, expected maximum 0)" end end diff --git a/spec/compiler/semantic/abstract_def_spec.cr b/spec/compiler/semantic/abstract_def_spec.cr index f28851e72210..59c3bd41ed84 100644 --- a/spec/compiler/semantic/abstract_def_spec.cr +++ b/spec/compiler/semantic/abstract_def_spec.cr @@ -396,7 +396,7 @@ describe "Semantic: abstract def" do end it "warning if missing return type" do - assert_warning %( + assert_warning <<-CR, abstract class Foo abstract def foo : Int32 end @@ -406,12 +406,12 @@ describe "Semantic: abstract def" do 1 end end - ), - "warning in line 8\nWarning: this method overrides Foo#foo() which has an explicit return type of Int32.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." + CR + "warning in line 6\nWarning: this method overrides Foo#foo() which has an explicit return type of Int32.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." end it "warning if different return type" do - assert_warning %( + assert_warning <<-CR, abstract class Foo abstract def foo : Int32 end @@ -424,8 +424,8 @@ describe "Semantic: abstract def" do 1 end end - ), - "warning in line 11\nWarning: this method must return Int32, which is the return type of the overridden method Foo#foo(), or a subtype of it, not Bar::Int32" + CR + "warning in line 9\nWarning: this method must return Int32, which is the return type of the overridden method Foo#foo(), or a subtype of it, not Bar::Int32" end it "can return a more specific type" do @@ -543,7 +543,7 @@ describe "Semantic: abstract def" do end it "is missing a return type in subclass of generic subclass" do - assert_warning %( + assert_warning <<-CR, abstract class Foo(T) abstract def foo : T end @@ -552,12 +552,12 @@ describe "Semantic: abstract def" do def foo end end - ), - "warning in line 8\nWarning: this method overrides Foo(T)#foo() which has an explicit return type of T.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." + CR + "warning in line 6\nWarning: this method overrides Foo(T)#foo() which has an explicit return type of T.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." end it "can't find parent return type" do - assert_warning %( + assert_warning <<-CR, abstract class Foo abstract def foo : Unknown end @@ -566,12 +566,12 @@ describe "Semantic: abstract def" do def foo end end - ), - "warning in line 4\nWarning: can't resolve return type Unknown" + CR + "warning in line 2\nWarning: can't resolve return type Unknown" end it "can't find child return type" do - assert_warning %( + assert_warning <<-CR, abstract class Foo abstract def foo : Int32 end @@ -580,8 +580,8 @@ describe "Semantic: abstract def" do def foo : Unknown end end - ), - "warning in line 8\nWarning: can't resolve return type Unknown" + CR + "warning in line 6\nWarning: can't resolve return type Unknown" end it "doesn't crash when abstract method is implemented by supertype (#8031)" do diff --git a/spec/compiler/semantic/automatic_cast.cr b/spec/compiler/semantic/automatic_cast_spec.cr similarity index 100% rename from spec/compiler/semantic/automatic_cast.cr rename to spec/compiler/semantic/automatic_cast_spec.cr diff --git a/spec/compiler/semantic/cleanup_spec.cr b/spec/compiler/semantic/cleanup_spec.cr index 0e7704c78833..a221b45275ba 100644 --- a/spec/compiler/semantic/cleanup_spec.cr +++ b/spec/compiler/semantic/cleanup_spec.cr @@ -1,5 +1,11 @@ require "../../spec_helper" +private def assert_after_cleanup(before, after) + node = Parser.parse(before) + result = semantic node + result.node.to_s.strip.should eq(after.strip) +end + describe "cleanup" do it "errors if assigning var to itself" do assert_error "a = 1; a = a", "expression has no effect" diff --git a/spec/compiler/semantic/concrete_types_spec.cr b/spec/compiler/semantic/concrete_types_spec.cr index 4b793c3acb56..f540244e39ba 100644 --- a/spec/compiler/semantic/concrete_types_spec.cr +++ b/spec/compiler/semantic/concrete_types_spec.cr @@ -1,7 +1,7 @@ require "../../spec_helper" private def assert_concrete_types(str) - result = semantic_result("struct Witness;end\n\n#{str}", nil, inject_primitives: false) + result = semantic("struct Witness;end\n\n#{str}", inject_primitives: false) program = result.program type, expected_concrete_types = yield program.types, program diff --git a/spec/compiler/semantic/lib_spec.cr b/spec/compiler/semantic/lib_spec.cr index b91936fa1f4b..9e8feabd1b30 100644 --- a/spec/compiler/semantic/lib_spec.cr +++ b/spec/compiler/semantic/lib_spec.cr @@ -377,21 +377,21 @@ describe "Semantic: lib" do end it "warns if @[Link(static: true)] is specified" do - assert_warning %( + assert_warning <<-CR, @[Link("foo", static: true)] lib Foo end - ), - "warning in line 3\nWarning: specifying static linking for individual libraries is deprecated" + CR + "warning in line 1\nWarning: specifying static linking for individual libraries is deprecated" end it "warns if Link annotations use positional arguments" do - assert_warning %( + assert_warning <<-CR, @[Link("foo", "bar")] lib Foo end - ), - "warning in line 3\nWarning: using non-named arguments for Link annotations is deprecated" + CR + "warning in line 1\nWarning: using non-named arguments for Link annotations is deprecated" end it "allows invoking lib call without obj inside lib" do diff --git a/spec/compiler/semantic/warnings_spec.cr b/spec/compiler/semantic/warnings_spec.cr index 79e91c665d6f..d5cc0d3c5970 100644 --- a/spec/compiler/semantic/warnings_spec.cr +++ b/spec/compiler/semantic/warnings_spec.cr @@ -8,8 +8,7 @@ describe "Semantic: warnings" do end foo - ), "warning in line 6\nWarning: Deprecated top-level foo. Do not use me", - inject_primitives: false + ), "warning in line 6\nWarning: Deprecated top-level foo. Do not use me" end it "deprecation reason is optional" do @@ -19,8 +18,7 @@ describe "Semantic: warnings" do end foo - ), "warning in line 6\nWarning: Deprecated top-level foo.", - inject_primitives: false + ), "warning in line 6\nWarning: Deprecated top-level foo." end it "detects deprecated class macros" do @@ -32,8 +30,7 @@ describe "Semantic: warnings" do end Foo.m - ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", - inject_primitives: false + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me" end it "detects deprecated generic class macros" do @@ -45,8 +42,7 @@ describe "Semantic: warnings" do end Foo.m - ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", - inject_primitives: false + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me" end it "detects deprecated module macros" do @@ -58,8 +54,7 @@ describe "Semantic: warnings" do end Foo.m - ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me", - inject_primitives: false + ), "warning in line 8\nWarning: Deprecated Foo.m. Do not use me" end it "detects deprecated macros with named arguments" do @@ -69,8 +64,7 @@ describe "Semantic: warnings" do end foo(a: 2) - ), "warning in line 6\nWarning: Deprecated top-level foo.", - inject_primitives: false + ), "warning in line 6\nWarning: Deprecated top-level foo." end it "informs warnings once per call site location (a)" do @@ -87,7 +81,7 @@ describe "Semantic: warnings" do Foo.b Foo.b - ), inject_primitives: false + ) warning_failures.size.should eq(1) end @@ -102,7 +96,7 @@ describe "Semantic: warnings" do Foo.m Foo.m - ), inject_primitives: false + ) warning_failures.size.should eq(2) end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 363da3952707..f3b158c463b3 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -33,39 +33,46 @@ end record SemanticResult, program : Program, - node : ASTNode, - type : Type + node : ASTNode -def assert_type(str, flags = nil, inject_primitives = true) - result = semantic_result(str, flags, inject_primitives: inject_primitives) +def assert_type(str, *, inject_primitives = true, flags = nil, file = __FILE__, line = __LINE__) + result = semantic(str, flags: flags, inject_primitives: inject_primitives) program = result.program expected_type = with program yield program - result.type.should eq(expected_type) + node = result.node + if node.is_a?(Expressions) + node = node.last + end + node.type.should eq(expected_type), file: file, line: line result end -def semantic(code : String, wants_doc = false, inject_primitives = true) - code = inject_primitives(code) if inject_primitives - semantic parse(code, wants_doc: wants_doc), wants_doc: wants_doc +def semantic(code : String, wants_doc = false, inject_primitives = true, flags = nil, filename = nil) + node = parse(code, wants_doc: wants_doc, filename: filename) + node = inject_primitives(node) if inject_primitives + semantic node, wants_doc: wants_doc, flags: flags end -def semantic(node : ASTNode, wants_doc = false) - program = new_program - program.wants_doc = wants_doc - node = program.normalize node - node = program.semantic node - SemanticResult.new(program, node, node.type) +private def inject_primitives(node : ASTNode) + req = Crystal::Require.new("primitives") + case node + when Crystal::Expressions + node.expressions.unshift req + node + when Crystal::Nop + node + else + Crystal::Expressions.new [req, node] of ASTNode + end end -def semantic_result(str, flags = nil, inject_primitives = true) - str = inject_primitives(str) if inject_primitives +def semantic(node : ASTNode, wants_doc = false, flags = nil) program = new_program program.flags.concat(flags.split) if flags - input = parse str - input = program.normalize input - input = program.semantic input - input_type = input.is_a?(Expressions) ? input.last.type : input.type - SemanticResult.new(program, input, input_type) + program.wants_doc = wants_doc + node = program.normalize node + node = program.semantic node + SemanticResult.new(program, node) end def assert_normalize(from, to, flags = nil) @@ -97,18 +104,9 @@ def assert_expand_third(from : String, to) assert_expand node, to end -def assert_after_cleanup(before, after) - # before = inject_primitives(before) - node = Parser.parse(before) - result = semantic node - result.node.to_s.strip.should eq(after.strip) -end - def assert_error(str, message, inject_primitives = true, file = __FILE__, line = __LINE__) - str = inject_primitives(str) if inject_primitives - nodes = parse str expect_raises TypeException, message, file, line do - semantic nodes + semantic str, inject_primitives: inject_primitives end end @@ -116,9 +114,7 @@ def assert_no_errors(*args) semantic(*args) end -def warnings_result(code, inject_primitives = true) - code = inject_primitives(code) if inject_primitives - +def warnings_result(code) output_filename = Crystal.temp_executable("crystal-spec-output") compiler = create_spec_compiler @@ -132,17 +128,12 @@ def warnings_result(code, inject_primitives = true) result.program.warning_failures end -def assert_warning(code, message, inject_primitives = true, file = __FILE__, line = __LINE__) - warning_failures = warnings_result(code, inject_primitives) +def assert_warning(code, message, file = __FILE__, line = __LINE__) + warning_failures = warnings_result(code) warning_failures.size.should eq(1), file, line warning_failures[0].should start_with(message), file, line end -def assert_no_warnings(code, inject_primitives = true, file = __FILE__, line = __LINE__) - warning_failures = warnings_result(code, inject_primitives) - warning_failures.size.should eq(0), file, line -end - def assert_macro(macro_args, macro_body, call_args, expected, expected_pragmas = nil, flags = nil) assert_macro(macro_args, macro_body, expected, expected_pragmas, flags) { call_args } end @@ -166,13 +157,7 @@ def assert_macro_internal(program, sub_node, macro_args, macro_body, expected, e end def codegen(code, inject_primitives = true, debug = Crystal::Debug::None, filename = __FILE__) - code = inject_primitives(code) if inject_primitives - parser = Parser.new(code) - parser.filename = filename - parser.wants_doc = false - node = parser.parse - - result = semantic node + result = semantic code, inject_primitives: inject_primitives, filename: filename result.program.codegen(result.node, single_module: false, debug: debug)[""].mod end @@ -236,7 +221,9 @@ def create_spec_compiler end def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil) - code = inject_primitives(code) if inject_primitives + if inject_primitives + code = %(require "primitives"\n#{code}) + end # Code that requires the prelude doesn't run in LLVM's MCJIT # because of missing linked functions (which are available @@ -314,7 +301,3 @@ def test_c(c_code, crystal_code) File.delete(o_filename) end end - -private def inject_primitives(code) - %(require "primitives"\n) + code -end diff --git a/spec/support/syntax.cr b/spec/support/syntax.cr index 8d036541256a..67634180f565 100644 --- a/spec/support/syntax.cr +++ b/spec/support/syntax.cr @@ -154,8 +154,11 @@ def assert_syntax_error(str, message = nil, line = nil, column = nil, metafile = end end -def parse(string, wants_doc = false) +def parse(string, wants_doc = false, filename = nil) parser = Parser.new(string) parser.wants_doc = wants_doc + if filename + parser.filename = filename + end parser.parse end From 11075c436eae31768325aaed2aa3092f7557d170 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 1 Jun 2020 08:35:58 -0300 Subject: [PATCH 075/263] Add Int#digits (#9383) * Add Int#digits * Fix BigInt#digits and make signature compatible to Int#digits * Don't overflow on maximum values * Better to be safe than sorry --- spec/std/big/big_int_spec.cr | 28 ++++++++++++++++++++++++++ spec/std/int_spec.cr | 39 ++++++++++++++++++++++++++++++++++++ src/big/big_int.cr | 12 ++++++++--- src/int.cr | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/spec/std/big/big_int_spec.cr b/spec/std/big/big_int_spec.cr index 2646f75ad942..c58873757dac 100644 --- a/spec/std/big/big_int_spec.cr +++ b/spec/std/big/big_int_spec.cr @@ -434,6 +434,34 @@ describe "BigInt" do it "has unsafe_shr (#8691)" do BigInt.new(8).unsafe_shr(1).should eq(4) end + + describe "#digits" do + it "works for positive numbers or zero" do + 0.to_big_i.digits.should eq([0]) + 1.to_big_i.digits.should eq([1]) + 10.to_big_i.digits.should eq([0, 1]) + 123.to_big_i.digits.should eq([3, 2, 1]) + 123456789.to_big_i.digits.should eq([9, 8, 7, 6, 5, 4, 3, 2, 1]) + end + + it "works with a base" do + 123.to_big_i.digits(16).should eq([11, 7]) + end + + it "raises for invalid base" do + [1, 0, -1].each do |base| + expect_raises(ArgumentError, "Invalid base #{base}") do + 123.to_big_i.digits(base) + end + end + end + + it "raises for negative numbers" do + expect_raises(ArgumentError, "Can't request digits of negative number") do + -123.to_big_i.digits + end + end + end end describe "BigInt Math" do diff --git a/spec/std/int_spec.cr b/spec/std/int_spec.cr index 5f4eac7be322..bccae5a8292d 100644 --- a/spec/std/int_spec.cr +++ b/spec/std/int_spec.cr @@ -799,4 +799,43 @@ describe "Int" do (10.to_big_i ** 3010).bit_length.should eq(10000) end end + + describe "#digits" do + it "works for positive numbers or zero" do + 0.digits.should eq([0]) + 1.digits.should eq([1]) + 10.digits.should eq([0, 1]) + 123.digits.should eq([3, 2, 1]) + 123456789.digits.should eq([9, 8, 7, 6, 5, 4, 3, 2, 1]) + end + + it "works for maximums" do + Int32::MAX.digits.should eq(Int32::MAX.to_s.chars.map(&.to_i).reverse) + Int64::MAX.digits.should eq(Int64::MAX.to_s.chars.map(&.to_i).reverse) + UInt64::MAX.digits.should eq(UInt64::MAX.to_s.chars.map(&.to_i).reverse) + end + + it "works for non-Int32" do + digits = 123_i64.digits + digits.should eq([3, 2, 1]) + end + + it "works with a base" do + 123.digits(16).should eq([11, 7]) + end + + it "raises for invalid base" do + [1, 0, -1].each do |base| + expect_raises(ArgumentError, "Invalid base #{base}") do + 123.digits(base) + end + end + end + + it "raises for negative numbers" do + expect_raises(ArgumentError, "Can't request digits of negative number") do + -123.digits + end + end + end end diff --git a/src/big/big_int.cr b/src/big/big_int.cr index 18e9f824fc95..04fa1e142b03 100644 --- a/src/big/big_int.cr +++ b/src/big/big_int.cr @@ -433,14 +433,20 @@ struct BigInt < Int # BigInt.new("123456789101101987654321").to_s(36) # => "k3qmt029k48nmpd" # ``` def to_s(base : Int) : String - raise "Invalid base #{base}" unless 2 <= base <= 36 + raise ArgumentError.new("Invalid base #{base}") unless 2 <= base <= 36 cstr = LibGMP.get_str(nil, base, self) String.new(cstr) end - def digits : Array(Int32) + # :nodoc: + def digits(base = 10) : Array(Int32) + if self < 0 + raise ArgumentError.new("Can't request digits of negative number") + end + ary = [] of Int32 - self.to_s.each_char { |c| ary << c - '0' } + self.to_s(base).each_char { |c| ary << c.to_i(base) } + ary.reverse! ary end diff --git a/src/int.cr b/src/int.cr index 455892562c2a..27d8b75bf4f3 100644 --- a/src/int.cr +++ b/src/int.cr @@ -562,6 +562,41 @@ struct Int self % other end + # Returns the digits of a number in a given base. + # The digits are returned as an array with the least significant digit as the first array element. + # + # ``` + # 12345.digits # => [5, 4, 3, 2, 1] + # 12345.digits(7) # => [4, 6, 6, 0, 5] + # 12345.digits(100) # => [45, 23, 1] + # + # -12345.digits(7) # => ArgumentError + # ``` + def digits(base = 10) : Array(Int32) + if base < 2 + raise ArgumentError.new("Invalid base #{base}") + end + + if self < 0 + raise ArgumentError.new("Can't request digits of negative number") + end + + if self == 0 + return [0] + end + + num = self + + digits_count = (Math.log(self.to_f + 1) / Math.log(base)).ceil.to_i + + ary = Array(Int32).new(digits_count) + while num != 0 + ary << num.remainder(base).to_i + num = num.tdiv(base) + end + ary + end + private DIGITS_DOWNCASE = "0123456789abcdefghijklmnopqrstuvwxyz" private DIGITS_UPCASE = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" private DIGITS_BASE62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" From a2031afb7040dca69ee2d6d6445cdac3a7583155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 1 Jun 2020 16:06:31 +0200 Subject: [PATCH 076/263] Fix double string escape in XML::Node#content= (#9300) Libxml2 already escapes special characters in node content, there's no need for the custom `escape` method. It lead to double escape which turns `` content into `&ltfoo&>`. --- spec/std/xml/xml_spec.cr | 25 +++++++++++++++++++++++++ src/xml/node.cr | 16 +++------------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/spec/std/xml/xml_spec.cr b/spec/std/xml/xml_spec.cr index 2c6c6071b1f8..7eed384c0062 100644 --- a/spec/std/xml/xml_spec.cr +++ b/spec/std/xml/xml_spec.cr @@ -242,6 +242,31 @@ describe XML do end end + it "escapes content" do + doc = XML.parse(<<-XML) + + John + XML + + root = doc.root.not_nil! + root.text = "" + root.text.should eq("") + + root.to_xml.should eq(%(<foo>)) + end + + it "escapes content HTML fragment" do + doc = XML.parse_html(<<-XML, XML::HTMLParserOptions.default | XML::HTMLParserOptions::NOIMPLIED | XML::HTMLParserOptions::NODEFDTD) +

foo

+ XML + + node = doc.children.first + node.text = "" + node.text.should eq("") + + node.to_xml.should eq(%(

<foo>

)) + end + it "gets empty content" do doc = XML.parse("") doc.children.first.content.should eq("") diff --git a/src/xml/node.cr b/src/xml/node.cr index 7f4e2d10523a..aff5b97f7999 100644 --- a/src/xml/node.cr +++ b/src/xml/node.cr @@ -92,7 +92,7 @@ struct XML::Node # Sets the Node's content to a Text node containing string. # The string gets XML escaped, not interpreted as markup. def content=(content) - content = escape(content.to_s) + check_no_null_byte(content) LibXML.xmlNodeSetContent(self, content) end @@ -580,19 +580,9 @@ struct XML::Node ptr ? (ptr.as(Array(XML::Error))) : nil end - private SUBSTITUTIONS = { - '>' => ">", - '<' => "<", - '"' => """, - '\'' => "'", - '&' => "&", - } - - private def escape(string) - if string.includes? '\0' + private def check_no_null_byte(string) + if string.includes? Char::ZERO raise XML::Error.new("Cannot escape string containing null character", 0) end - - string.gsub(SUBSTITUTIONS) end end From b02e6d5717da86c49de9ccf124edd2ef908f309e Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 1 Jun 2020 11:15:38 -0300 Subject: [PATCH 077/263] Allow calling `at_exit` inside `at_exit` (#9388) --- spec/std/kernel_spec.cr | 32 +++++++++++++++++++++----------- src/crystal/at_exit_handlers.cr | 6 ------ src/kernel.cr | 4 ++++ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/spec/std/kernel_spec.cr b/spec/std/kernel_spec.cr index b8aa3f9c3af0..998efb9d1d0e 100644 --- a/spec/std/kernel_spec.cr +++ b/spec/std/kernel_spec.cr @@ -172,17 +172,6 @@ describe "at_exit" do error.should eq "Error running at_exit handler: Raised from at_exit handler!\n" end - it "errors when used in an at_exit handler" do - status, output, error = compile_and_run_source <<-CODE - at_exit do - at_exit {} - end - CODE - - status.success?.should be_false - error.should eq "Error running at_exit handler: Cannot use at_exit from an at_exit handler\n" - end - it "shows unhandled exceptions after at_exit handlers" do status, _, error = compile_and_run_source <<-CODE at_exit do @@ -219,6 +208,27 @@ describe "at_exit" do Unhandled exception: Kaboom! OUTPUT end + + it "allows at_exit inside at_exit" do + status, output = compile_and_run_source <<-CODE + at_exit do + puts "1" + at_exit do + puts "2" + end + end + + at_exit do + puts "3" + at_exit do + puts "4" + end + end + CODE + + status.success?.should be_true + output.should eq("3\n4\n1\n2\n") + end end describe "seg fault" do diff --git a/src/crystal/at_exit_handlers.cr b/src/crystal/at_exit_handlers.cr index 8ef75c1430a1..7cd7b8cbc244 100644 --- a/src/crystal/at_exit_handlers.cr +++ b/src/crystal/at_exit_handlers.cr @@ -1,18 +1,12 @@ # :nodoc: module Crystal::AtExitHandlers - @@running = false - private class_getter(handlers) { [] of Int32, ::Exception? -> } def self.add(handler) - raise "Cannot use at_exit from an at_exit handler" if @@running - handlers << handler end def self.run(status, exception = nil) - @@running = true - if handlers = @@handlers # Run the registered handlers in reverse order while handler = handlers.pop? diff --git a/src/kernel.cr b/src/kernel.cr index 74410b97dc6c..b0d5cfc38eca 100644 --- a/src/kernel.cr +++ b/src/kernel.cr @@ -467,6 +467,10 @@ end # passed as the second argument to the block, if the program terminates # normally or `exit(status)` is called explicitly, then the second argument # will be `nil`. +# +# NOTE: If `at_exit` is called inside an `at_exit` handler, it will be called +# right after the current `at_exit` handler ends, and then other handlers +# will be invoked. def at_exit(&handler : Int32, Exception? ->) : Nil Crystal::AtExitHandlers.add(handler) end From cdc98295cc7185449120835be2ce33c70a203830 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 2 Jun 2020 02:58:29 +0900 Subject: [PATCH 078/263] Parser: allow ->@ivar.foo and ->@@cvar.foo (#9268) * Parser: allow ->@ivar.foo and ->@@cvar.foo Fixed #9239 * Add semantic specs for ->@ivar.foo and ->@@cvar.foo syntax https://github.com/crystal-lang/crystal/pull/9268#issuecomment-634719489 --- spec/compiler/formatter/formatter_spec.cr | 3 ++ spec/compiler/parser/parser_spec.cr | 2 ++ spec/compiler/semantic/proc_spec.cr | 38 +++++++++++++++++++++++ src/compiler/crystal/syntax/parser.cr | 12 +++++++ 4 files changed, 55 insertions(+) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index c5dacd27d36c..cec3ecc9e2f9 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -722,6 +722,9 @@ describe Crystal::Formatter do assert_format "->( x , y ) { x }", "->(x, y) { x }" assert_format "->( x : Int32 , y ) { x }", "->(x : Int32, y) { x }" + assert_format "->@foo.foo" + assert_format "->@@foo.foo" + {:+, :-, :*, :/, :^, :>>, :<<, :|, :&, :&+, :&-, :&*, :&**}.each do |sym| assert_format ":#{sym}" end diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index b79f32aab62c..116b5c9dc047 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1227,6 +1227,8 @@ module Crystal it_parses "[] of ->\n", ArrayLiteral.new(of: ProcNotation.new) it_parses "->foo=", ProcPointer.new(nil, "foo=") it_parses "foo = 1; ->foo.foo=", [Assign.new("foo".var, 1.int32), ProcPointer.new("foo".var, "foo=")] + it_parses "->@foo.foo", [ProcPointer.new("@foo".instance_var, "foo")] + it_parses "->@@foo.foo", [ProcPointer.new("@@foo".class_var, "foo")] it_parses "foo &->bar", Call.new(nil, "foo", block_arg: ProcPointer.new(nil, "bar")) diff --git a/spec/compiler/semantic/proc_spec.cr b/spec/compiler/semantic/proc_spec.cr index 03b0c7fae9d9..701517c024ca 100644 --- a/spec/compiler/semantic/proc_spec.cr +++ b/spec/compiler/semantic/proc_spec.cr @@ -1069,4 +1069,42 @@ describe "Semantic: proc" do foo )) { proc_of nil_type } end + + it "can use @ivar as pointer syntax receiver (#9239)" do + assert_type(%( + class Foo + def foo + 1 + end + end + + class Bar + @foo = Foo.new + + def foo + ->@foo.foo + end + end + + Bar.new.foo + )) { proc_of int32 } + end + + it "can use @@cvar as pointer syntax receiver (#9239)" do + assert_type(%( + class Foo + @@foo = new + + def self.foo + ->@@foo.foo + end + + def foo + 1 + end + end + + Foo.foo + )) { proc_of int32 } + end end diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 2cb5843d8cc5..077f4de49288 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -1905,6 +1905,18 @@ module Crystal check :"." name = consume_def_or_macro_name next_token_skip_space + when :INSTANCE_VAR + obj = InstanceVar.new(@token.value.to_s) + next_token_skip_space + check :"." + name = consume_def_or_macro_name + next_token_skip_space + when :CLASS_VAR + obj = ClassVar.new(@token.value.to_s) + next_token_skip_space + check :"." + name = consume_def_or_macro_name + next_token_skip_space else unexpected_token end From 766c927becd6dda90b38d866e136be0119c5e54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 1 Jun 2020 20:00:04 +0200 Subject: [PATCH 079/263] Remove HTTP::Params::Builder#to_s (#9319) A method that appends an internal IO to an external IO is useless when you can just write to the external IO directly. --- src/http/params.cr | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/http/params.cr b/src/http/params.cr index 9a5de41f3ecb..3e0359872ebe 100644 --- a/src/http/params.cr +++ b/src/http/params.cr @@ -127,9 +127,9 @@ module HTTP # params # => "color=black&name=crystal&year=2012+-+today" # ``` def self.build(&block : Builder ->) : String - form_builder = Builder.new - yield form_builder - form_builder.to_s + String.build do |io| + yield Builder.new(io) + end end protected getter raw_params @@ -330,10 +330,7 @@ module HTTP # Every parameter added is directly written to an `IO`, # where keys and values are properly escaped. class Builder - @io : IO - @first : Bool - - def initialize(@io = IO::Memory.new) + def initialize(@io : IO) @first = true end @@ -352,10 +349,6 @@ module HTTP values.each { |value| add(key, value) } self end - - def to_s(io : IO) : Nil - io << @io.to_s - end end end end From bffe450044a9f7e0540dc21a1ccf70c79145e396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 1 Jun 2020 20:01:26 +0200 Subject: [PATCH 080/263] Fix flush methods to always flush underlying IO (#9320) * Fix flush methods to always flush underlying IO * fixup! Fix flush methods to always flush underlying IO --- src/compress/deflate/writer.cr | 1 + src/pretty_print.cr | 2 ++ src/xml/builder.cr | 5 +++-- src/yaml/builder.cr | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/compress/deflate/writer.cr b/src/compress/deflate/writer.cr index 0df595422e04..22b02905ae22 100644 --- a/src/compress/deflate/writer.cr +++ b/src/compress/deflate/writer.cr @@ -60,6 +60,7 @@ class Compress::Deflate::Writer < IO return if @closed consume_output LibZ::Flush::SYNC_FLUSH + @output.flush end # Closes this writer. Must be invoked after all data has been written. diff --git a/src/pretty_print.cr b/src/pretty_print.cr index 0b32809504a5..7dbf4be6a7cf 100644 --- a/src/pretty_print.cr +++ b/src/pretty_print.cr @@ -191,6 +191,8 @@ class PrettyPrint end @buffer.clear @buffer_width = 0 + + @output.flush end private class Text diff --git a/src/xml/builder.cr b/src/xml/builder.cr index b5e30addcd97..4597314ed99d 100644 --- a/src/xml/builder.cr +++ b/src/xml/builder.cr @@ -11,7 +11,7 @@ struct XML::Builder @box : Void* # Creates a builder that writes to the given *io*. - def initialize(io : IO) + def initialize(@io : IO) @box = Box.box(io) buffer = LibXML.xmlOutputBufferCreateIO( ->(ctx, buffer, len) { @@ -249,6 +249,8 @@ struct XML::Builder # this writer's `IO`. def flush call Flush + + @io.flush end # Sets the indent string. @@ -376,7 +378,6 @@ module XML # when StartDocument is omitted. xml.end_document xml.flush - io.flush v end end diff --git a/src/yaml/builder.cr b/src/yaml/builder.cr index c93cb8817091..cc925fc5a107 100644 --- a/src/yaml/builder.cr +++ b/src/yaml/builder.cr @@ -183,6 +183,8 @@ class YAML::Builder # Flushes any pending data to the underlying `IO`. def flush LibYAML.yaml_emitter_flush(@emitter) + + @io.flush end def finalize From 9dbc88e6303afe5582b4bd64238774cb8a15b359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 1 Jun 2020 20:19:28 +0200 Subject: [PATCH 081/263] Add file and line references in spec_helper asserts (#9059) --- spec/spec_helper.cr | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index f3b158c463b3..0d17a2118e94 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -75,36 +75,36 @@ def semantic(node : ASTNode, wants_doc = false, flags = nil) SemanticResult.new(program, node) end -def assert_normalize(from, to, flags = nil) +def assert_normalize(from, to, flags = nil, *, file = __FILE__, line = __LINE__) program = new_program program.flags.concat(flags.split) if flags normalizer = Normalizer.new(program) from_nodes = Parser.parse(from) to_nodes = program.normalize(from_nodes) - to_nodes.to_s.strip.should eq(to.strip) + to_nodes.to_s.strip.should eq(to.strip), file: file, line: line to_nodes end -def assert_expand(from : String, to) - assert_expand Parser.parse(from), to +def assert_expand(from : String, to, *, file = __FILE__, line = __LINE__) + assert_expand Parser.parse(from), to, file: file, line: line end -def assert_expand(from_nodes : ASTNode, to) +def assert_expand(from_nodes : ASTNode, to, *, file = __FILE__, line = __LINE__) to_nodes = LiteralExpander.new(new_program).expand(from_nodes) - to_nodes.to_s.strip.should eq(to.strip) + to_nodes.to_s.strip.should eq(to.strip), file: file, line: line end -def assert_expand_second(from : String, to) +def assert_expand_second(from : String, to, *, file = __FILE__, line = __LINE__) node = (Parser.parse(from).as(Expressions))[1] - assert_expand node, to + assert_expand node, to, file: file, line: line end -def assert_expand_third(from : String, to) +def assert_expand_third(from : String, to, *, file = __FILE__, line = __LINE__) node = (Parser.parse(from).as(Expressions))[2] - assert_expand node, to + assert_expand node, to, file: file, line: line end -def assert_error(str, message, inject_primitives = true, file = __FILE__, line = __LINE__) +def assert_error(str, message, *, inject_primitives = true, file = __FILE__, line = __LINE__) expect_raises TypeException, message, file, line do semantic str, inject_primitives: inject_primitives end @@ -128,31 +128,31 @@ def warnings_result(code) result.program.warning_failures end -def assert_warning(code, message, file = __FILE__, line = __LINE__) +def assert_warning(code, message, *, file = __FILE__, line = __LINE__) warning_failures = warnings_result(code) warning_failures.size.should eq(1), file, line warning_failures[0].should start_with(message), file, line end -def assert_macro(macro_args, macro_body, call_args, expected, expected_pragmas = nil, flags = nil) - assert_macro(macro_args, macro_body, expected, expected_pragmas, flags) { call_args } +def assert_macro(macro_args, macro_body, call_args, expected, expected_pragmas = nil, flags = nil, file = __FILE__, line = __LINE__) + assert_macro(macro_args, macro_body, expected, expected_pragmas, flags, file: file, line: line) { call_args } end -def assert_macro(macro_args, macro_body, expected, expected_pragmas = nil, flags = nil) +def assert_macro(macro_args, macro_body, expected, expected_pragmas = nil, flags = nil, file = __FILE__, line = __LINE__) program = new_program program.flags.concat(flags.split) if flags sub_node = yield program - assert_macro_internal program, sub_node, macro_args, macro_body, expected, expected_pragmas + assert_macro_internal program, sub_node, macro_args, macro_body, expected, expected_pragmas, file: file, line: line end -def assert_macro_internal(program, sub_node, macro_args, macro_body, expected, expected_pragmas) +def assert_macro_internal(program, sub_node, macro_args, macro_body, expected, expected_pragmas, file = __FILE__, line = __LINE__) macro_def = "macro foo(#{macro_args});#{macro_body};end" a_macro = Parser.parse(macro_def).as(Macro) call = Call.new(nil, "", sub_node) result, result_pragmas = program.expand_macro a_macro, call, program, program result = result.chomp(';') - result.should eq(expected) + result.should eq(expected), file: file, line: line result_pragmas.should eq(expected_pragmas) if expected_pragmas end From 13a9d3dda58d5087896b29526e91af76f7bb92e8 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 2 Jun 2020 03:21:17 +0900 Subject: [PATCH 082/263] Doc: show input type path instead of full qualified path on generic (#9302) * Doc: show input type path instead full qualified path on generic * Build program for spec instead of parsing and analyzing https://github.com/crystal-lang/crystal/pull/9302#discussion_r429137588 * Revert "Build program for spec instead of parsing and analyzing" This reverts commit db4b94c7b1ecdcbf29b6f1b485b51784b0fb500a. * Remove unnecessary wants_docs option --- spec/compiler/crystal/tools/doc/type_spec.cr | 54 ++++++++++++++++++++ src/compiler/crystal/tools/doc/type.cr | 19 +------ 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/type_spec.cr b/spec/compiler/crystal/tools/doc/type_spec.cr index f9651b4632c7..87a9e7a19bc0 100644 --- a/spec/compiler/crystal/tools/doc/type_spec.cr +++ b/spec/compiler/crystal/tools/doc/type_spec.cr @@ -44,4 +44,58 @@ describe Doc::Type do foo.lookup_class_method("new").should_not be_nil foo.lookup_class_method("new", 1).should_not be_nil end + + describe "#node_to_html" do + it "shows relative path" do + program = semantic(<<-CODE).program + class Foo + class Bar + end + end + CODE + + generator = Doc::Generator.new program, [""] + foo = generator.type(program.types["Foo"]) + foo.node_to_html("Bar".path).should eq(%(Bar)) + end + + it "shows relative generic" do + program = semantic(<<-CODE).program + class Foo + class Bar(T) + end + end + CODE + + generator = Doc::Generator.new program, [""] + foo = generator.type(program.types["Foo"]) + foo.node_to_html(Generic.new("Bar".path, ["Foo".path] of ASTNode)).should eq(%(Bar(Foo))) + end + + it "shows generic path with necessary colons" do + program = semantic(<<-CODE).program + class Foo + class Foo + end + end + CODE + + generator = Doc::Generator.new program, [""] + foo = generator.type(program.types["Foo"]) + foo.node_to_html("Foo".path(global: true)).should eq(%(::Foo)) + end + + it "shows generic path with unnecessary colons" do + program = semantic(<<-CODE).program + class Foo + class Bar + end + end + CODE + + generator = Doc::Generator.new program, [""] + foo = generator.type(program.types["Foo"]) + foo.node_to_html("Foo".path(global: true)).should eq(%(Foo)) + end + end end diff --git a/src/compiler/crystal/tools/doc/type.cr b/src/compiler/crystal/tools/doc/type.cr index e9b27cfca745..8acbc80753ad 100644 --- a/src/compiler/crystal/tools/doc/type.cr +++ b/src/compiler/crystal/tools/doc/type.cr @@ -496,24 +496,7 @@ class Crystal::Doc::Type end def node_to_html(node : Generic, io, links = true) - match = lookup_path(node.name.as(Path)) - if match - if match.must_be_included? - if links - io << %() - end - match.full_name_without_type_vars(io) - if links - io << "" - end - else - io << node.name - end - else - io << node.name - end + node_to_html node.name, io, links: links io << '(' node.type_vars.join(io, ", ") do |type_var| node_to_html type_var, io, links: links From babe00e4e5a60ad29bd8a7ad1970c6ad7b5083d2 Mon Sep 17 00:00:00 2001 From: Kubo Takehiro Date: Tue, 2 Jun 2020 03:30:08 +0900 Subject: [PATCH 083/263] Improve File.utime precision from second to 100-nanosecond on Windows (#9344) --- src/crystal/system/win32/file.cr | 26 +++++++++++++++++----- src/crystal/system/win32/time.cr | 9 ++++++++ src/lib_c/x86_64-windows-msvc/c/fileapi.cr | 4 ++++ src/lib_c/x86_64-windows-msvc/c/winnt.cr | 3 ++- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index b6b9c2d51396..6832d77d37f7 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -209,12 +209,26 @@ module Crystal::System::File end def self.utime(access_time : ::Time, modification_time : ::Time, path : String) : Nil - times = LibC::Utimbuf64.new - times.actime = access_time.to_unix - times.modtime = modification_time.to_unix - - if LibC._wutime64(to_windows_path(path), pointerof(times)) != 0 - raise ::File::Error.from_errno("Error setting time on file", file: path) + atime = Crystal::System::Time.to_filetime(access_time) + mtime = Crystal::System::Time.to_filetime(modification_time) + handle = LibC.CreateFileW( + to_windows_path(path), + LibC::FILE_WRITE_ATTRIBUTES, + LibC::FILE_SHARE_READ | LibC::FILE_SHARE_WRITE | LibC::FILE_SHARE_DELETE, + nil, + LibC::OPEN_EXISTING, + LibC::FILE_ATTRIBUTE_NORMAL, + LibC::HANDLE.null + ) + if handle == LibC::INVALID_HANDLE_VALUE + raise ::File::Error.from_winerror("Error setting time on file", file: path) + end + begin + if LibC.SetFileTime(handle, nil, pointerof(atime), pointerof(mtime)) == 0 + raise ::File::Error.from_winerror("Error setting time on file", file: path) + end + ensure + LibC.CloseHandle(handle) end end diff --git a/src/crystal/system/win32/time.cr b/src/crystal/system/win32/time.cr index 535a846d4074..c3afe2cd06ab 100644 --- a/src/crystal/system/win32/time.cr +++ b/src/crystal/system/win32/time.cr @@ -35,6 +35,15 @@ module Crystal::System::Time ::Time.utc(seconds: seconds, nanoseconds: nanoseconds) end + def self.to_filetime(time : ::Time) : LibC::FILETIME + span = time - ::Time.utc(seconds: WINDOWS_EPOCH_IN_SECONDS, nanoseconds: 0) + ticks = span.to_i.to_u64 * FILETIME_TICKS_PER_SECOND + span.nanoseconds // NANOSECONDS_PER_FILETIME_TICK + filetime = uninitialized LibC::FILETIME + filetime.dwHighDateTime = (ticks >> 32).to_u32 + filetime.dwLowDateTime = ticks.to_u32! + filetime + end + def self.filetime_to_f64secs(filetime) : Float64 ((filetime.dwHighDateTime.to_u64 << 32) | filetime.dwLowDateTime.to_u64).to_f64 / FILETIME_TICKS_PER_SECOND.to_f64 end diff --git a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr index 72e05dd8b7ed..1b7ef08ce276 100644 --- a/src/lib_c/x86_64-windows-msvc/c/fileapi.cr +++ b/src/lib_c/x86_64-windows-msvc/c/fileapi.cr @@ -32,6 +32,7 @@ lib LibC OPEN_EXISTING = 3 + FILE_ATTRIBUTE_NORMAL = 0x80 FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 FILE_SHARE_READ = 0x1 @@ -79,4 +80,7 @@ lib LibC fun FindFirstFileW(lpFileName : LPWSTR, lpFindFileData : WIN32_FIND_DATAW*) : HANDLE fun FindNextFileW(hFindFile : HANDLE, lpFindFileData : WIN32_FIND_DATAW*) : BOOL fun FindClose(hFindFile : HANDLE) : BOOL + + fun SetFileTime(hFile : HANDLE, lpCreationTime : FILETIME*, + lpLastAccessTime : FILETIME*, lpLastWriteTime : FILETIME*) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 2d6b9e8f2869..e60e00064273 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -16,7 +16,8 @@ lib LibC FILE_ATTRIBUTE_READONLY = 0x1 FILE_ATTRIBUTE_REPARSE_POINT = 0x400 - FILE_READ_ATTRIBUTES = 0x80 + FILE_READ_ATTRIBUTES = 0x80 + FILE_WRITE_ATTRIBUTES = 0x0100 # Memory protection constants PAGE_READWRITE = 0x04 From 4762107f0dd1b93e1874d2b1f0d6ccec719ea4a4 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 2 Jun 2020 03:30:54 +0900 Subject: [PATCH 084/263] Format: fix formatting `&.!` (#9391) Fixed #8823 --- spec/compiler/formatter/formatter_spec.cr | 3 +++ src/compiler/crystal/tools/formatter.cr | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index cec3ecc9e2f9..ed0f13d24a13 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -1610,6 +1610,9 @@ describe Crystal::Formatter do CODE assert_format "a.!" + assert_format "a &.!" + assert_format "a &.a.!" + assert_format "a &.!.!" assert_format <<-CODE ->{ diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 6ca82cac6454..b0b72c508f06 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -2987,6 +2987,14 @@ module Crystal clear_object(body) accept body end + when Not + if body.exp.is_a?(Var) + call = Call.new(nil, "!") + accept call + else + clear_object(body) + accept body + end else raise "BUG: unexpected node for &. argument, at #{node.location}, not #{body.class}" end @@ -3027,6 +3035,12 @@ module Crystal else clear_object(node.obj) end + when Not + if node.exp.is_a?(Var) + node.exp = Nop.new + else + clear_object(node.exp) + end else # nothing to do end From fc5c712e17768fbca906e247c8dd47a6b070dbf2 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 2 Jun 2020 03:32:20 +0900 Subject: [PATCH 085/263] Format: avoid formatter crash on heredoc having interpolation of sting literal (#9382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Format: avoid formatter crash on heredoc having interpolation of string literal Fixed #9362 Close #9373 It adds `heredoc_indent` property to `StringInterpolation` to detect the first interpolation on formatting correctly. In addition, it fixes stirng token consuming on heredoc. * Fix comment typo * Fix typo in comment again Co-authored-by: Johannes Müller Co-authored-by: Johannes Müller --- spec/compiler/formatter/formatter_spec.cr | 6 ++++ src/compiler/crystal/syntax/ast.cr | 6 +++- src/compiler/crystal/syntax/parser.cr | 2 ++ src/compiler/crystal/tools/formatter.cr | 37 ++++++++++++----------- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index ed0f13d24a13..2d7a0423278d 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -974,6 +974,12 @@ describe Crystal::Formatter do assert_format "<<-HTML\n \#{1}x\n HTML" assert_format "<<-HTML\n \#{1}x\n y\n HTML" assert_format "<<-HTML\n \#{1}x\n y\n z\n HTML" + assert_format %(<<-HTML\n \#{"foo"}\n HTML) + assert_format %(<<-HTML\n \#{__FILE__}\n HTML) + assert_format %(<<-HTML\n \#{"fo\#{"o"}"}\n HTML) + assert_format %(<<-HTML\n \#{"foo"}\#{1}\n HTML) + assert_format %(<<-HTML\n foo\n \#{"foo"}\n HTML) + assert_format %(<<-HTML\n \#{"foo"}\n \#{"bar"}\n HTML) assert_format " <<-HTML\n foo\n HTML", "<<-HTML\n foo\nHTML" assert_format " <<-HTML\n \#{1}\n HTML", "<<-HTML\n \#{1}\nHTML" diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index 38107b6f992e..271a8320b9b5 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -296,7 +296,11 @@ module Crystal class StringInterpolation < ASTNode property expressions : Array(ASTNode) - def initialize(@expressions : Array(ASTNode)) + # Removed indentation size. + # This property is only available when this is created from heredoc. + property heredoc_indent : Int32 + + def initialize(@expressions : Array(ASTNode), @heredoc_indent = 0) end def accept_children(visitor) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 077f4de49288..9f326e3235f6 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -2139,6 +2139,8 @@ module Crystal node.expressions.push(StringLiteral.new(string).at(node.location).at_end(token_end_location)) end + node.heredoc_indent = delimiter_state.heredoc_indent + node.end_location = token_end_location end diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index b0b72c508f06..60c3858f3441 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -537,6 +537,16 @@ module Crystal heredoc_line = @line heredoc_end = @line + # To detect the first content of interpolation of string literal correctly, + # we should consume the first string token if this token contains only removed indentation of heredoc. + if is_heredoc && @token.type == :STRING + token_is_indent = @token.raw.bytesize == node.heredoc_indent && @token.raw.each_char.all? &.ascii_whitespace? + if token_is_indent + write @token.raw + next_string_token + end + end + node.expressions.each do |exp| if @token.type == :DELIMITER_END # Heredoc cannot contain string continuation, @@ -569,17 +579,19 @@ module Crystal check :"}" write "}" @token.delimiter_state = delimiter_state + next_string_token else - if @token.invalid_escape - write @token.value - else - write @token.raw + loop do + check :STRING + write @token.invalid_escape ? @token.value : @token.raw + next_string_token + + # On heredoc, pieces of contents are combined due to removing indentation. + # Thus, we should consume continuous string tokens at once. + break if !is_heredoc || @token.type != :STRING end end - next_string_token else - skip_strings - check :INTERPOLATION_START write "\#{" delimiter_state = @token.delimiter_state @@ -606,8 +618,6 @@ module Crystal end end - skip_strings - heredoc_end = @line check :DELIMITER_END @@ -632,15 +642,6 @@ module Crystal false end - private def skip_strings - # Heredocs might indice some spaces that are removed - # because of indentation - while @token.type == :STRING - write @token.raw - next_string_token - end - end - private def consume_heredocs @consuming_heredocs = true @lexer.heredocs.reverse! From cca83e674d1b25eb124d842f78027f2a5ac1ad49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 1 Jun 2020 20:36:30 +0200 Subject: [PATCH 086/263] Make compiler specs use locally build crystal (#9393) * Make compiler specs use locally build crystal Do not rely on a globally installed Crystal being available. * Fix config spec to use lchop instead of lstrip --- spec/compiler/config_spec.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/compiler/config_spec.cr b/spec/compiler/config_spec.cr index df1667a38028..77750b33a292 100644 --- a/spec/compiler/config_spec.cr +++ b/spec/compiler/config_spec.cr @@ -3,7 +3,7 @@ require "./spec_helper" describe Crystal::Config do it ".host_target" do - Crystal::Config.host_target.should eq Crystal::Codegen::Target.new({{ `crystal --version`.lines[-1] }}.lstrip("Default target: ")) + Crystal::Config.host_target.should eq Crystal::Codegen::Target.new({{ `bin/crystal --version`.lines[-1] }}.lchop("Default target: ")) end {% if flag?(:linux) %} From 2d8d6491103f1bc1cfdc77d55cfd172081204511 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 1 Jun 2020 20:38:11 +0200 Subject: [PATCH 087/263] Split general ABI specs from x86_64-specific ones, run on every platform (#9384) --- spec/compiler/codegen/c_abi/c_abi_spec.cr | 172 ++++++++++++++++++ .../codegen/c_abi/c_abi_x86_64_spec.cr | 169 ----------------- 2 files changed, 172 insertions(+), 169 deletions(-) create mode 100644 spec/compiler/codegen/c_abi/c_abi_spec.cr diff --git a/spec/compiler/codegen/c_abi/c_abi_spec.cr b/spec/compiler/codegen/c_abi/c_abi_spec.cr new file mode 100644 index 000000000000..2faaee55ce96 --- /dev/null +++ b/spec/compiler/codegen/c_abi/c_abi_spec.cr @@ -0,0 +1,172 @@ +require "../../../spec_helper" + +describe "Code gen: C ABI" do + it "passes struct less than 64 bits (for real)" do + test_c( + %( + struct s { + char x; + short y; + }; + + int foo(struct s a) { + return a.x + a.y; + } + ), + %( + lib LibFoo + struct Struct + x : Int8 + y : Int16 + end + + fun foo(s : Struct) : Int32 + end + + s = LibFoo::Struct.new x: 1_i8, y: 2_i16 + LibFoo.foo(s) + ), &.to_i.should eq(3)) + end + + it "passes struct between 64 and 128 bits (for real)" do + test_c( + %( + struct s { + long x; + short y; + }; + + long foo(struct s a) { + return a.x + a.y; + } + ), + %( + lib LibFoo + struct Struct + x : Int64 + y : Int16 + end + + fun foo(s : Struct) : Int64 + end + + s = LibFoo::Struct.new x: 1_i64, y: 2_i16 + LibFoo.foo(s) + ), &.to_i.should eq(3)) + end + + it "passes struct bigger than128 bits (for real)" do + test_c( + %( + struct s { + long x; + long y; + char z; + }; + + long foo(struct s a) { + return a.x + a.y + a.z; + } + ), + %( + lib LibFoo + struct Struct + x : Int64 + y : Int64 + z : Int8 + end + + fun foo(s : Struct) : Int64 + end + + s = LibFoo::Struct.new x: 1_i64, y: 2_i64, z: 3_i8 + LibFoo.foo(s) + ), &.to_i.should eq(6)) + end + + it "returns struct less than 64 bits (for real)" do + test_c( + %( + struct s { + char x; + short y; + }; + + struct s foo() { + struct s a = {1, 2}; + return a; + } + ), + %( + lib LibFoo + struct Struct + x : Int8 + y : Int16 + end + + fun foo : Struct + end + + str = LibFoo.foo + str.x.to_i + str.y.to_i + ), &.to_i.should eq(3)) + end + + it "returns struct between 64 and 128 bits (for real)" do + test_c( + %( + struct s { + long x; + short y; + }; + + struct s foo() { + struct s a = {1, 2}; + return a; + } + ), + %( + lib LibFoo + struct Struct + x : Int64 + y : Int16 + end + + fun foo : Struct + end + + str = LibFoo.foo + (str.x + str.y).to_i32 + ), &.to_i.should eq(3)) + end + + it "returns struct bigger than 128 bits with sret" do + test_c( + %( + struct s { + long x; + long y; + char z; + }; + + struct s foo(int z) { + struct s a = {1, 2, z}; + return a; + } + ), + %( + lib LibFoo + struct Struct + x : Int64 + y : Int64 + z : Int8 + end + + fun foo(w : Int32) : Struct + end + + str = LibFoo.foo(3) + (str.x + str.y + str.z).to_i32 + ), &.to_i.should eq(6)) + end +end diff --git a/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr b/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr index e01c7189b959..3c0cd2122bd1 100644 --- a/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr @@ -21,33 +21,6 @@ require "../../../spec_helper" str.should contain("declare void @foo({ i64 })") end - it "passes struct less than 64 bits (for real)" do - test_c( - %( - struct s { - char x; - short y; - }; - - int foo(struct s a) { - return a.x + a.y; - } - ), - %( - lib LibFoo - struct Struct - x : Int8 - y : Int16 - end - - fun foo(s : Struct) : Int32 - end - - s = LibFoo::Struct.new x: 1_i8, y: 2_i16 - LibFoo.foo(s) - ), &.to_i.should eq(3)) - end - it "passes struct less than 64 bits as { i64 } in varargs" do mod = codegen(%( lib LibFoo @@ -109,33 +82,6 @@ require "../../../spec_helper" )) end - it "passes struct between 64 and 128 bits (for real)" do - test_c( - %( - struct s { - long x; - short y; - }; - - long foo(struct s a) { - return a.x + a.y; - } - ), - %( - lib LibFoo - struct Struct - x : Int64 - y : Int16 - end - - fun foo(s : Struct) : Int64 - end - - s = LibFoo::Struct.new x: 1_i64, y: 2_i16 - LibFoo.foo(s) - ), &.to_i.should eq(3)) - end - it "passes struct bigger than128 bits with byval" do mod = codegen(%( lib LibFoo @@ -155,35 +101,6 @@ require "../../../spec_helper" str.scan(/byval/).size.should eq(2) end - it "passes struct bigger than128 bits (for real)" do - test_c( - %( - struct s { - long x; - long y; - char z; - }; - - long foo(struct s a) { - return a.x + a.y + a.z; - } - ), - %( - lib LibFoo - struct Struct - x : Int64 - y : Int64 - z : Int8 - end - - fun foo(s : Struct) : Int64 - end - - s = LibFoo::Struct.new x: 1_i64, y: 2_i64, z: 3_i8 - LibFoo.foo(s) - ), &.to_i.should eq(6)) - end - it "returns struct less than 64 bits as { i64 }" do mod = codegen(%( lib LibFoo @@ -202,34 +119,6 @@ require "../../../spec_helper" str.should contain("declare { i64 } @foo()") end - it "returns struct less than 64 bits (for real)" do - test_c( - %( - struct s { - char x; - short y; - }; - - struct s foo() { - struct s a = {1, 2}; - return a; - } - ), - %( - lib LibFoo - struct Struct - x : Int8 - y : Int16 - end - - fun foo : Struct - end - - str = LibFoo.foo - str.x.to_i + str.y.to_i - ), &.to_i.should eq(3)) - end - it "returns struct between 64 and 128 bits as { i64, i64 }" do mod = codegen(%( lib LibFoo @@ -248,34 +137,6 @@ require "../../../spec_helper" str.should contain("declare { i64, i64 } @foo()") end - it "returns struct between 64 and 128 bits (for real)" do - test_c( - %( - struct s { - long x; - short y; - }; - - struct s foo() { - struct s a = {1, 2}; - return a; - } - ), - %( - lib LibFoo - struct Struct - x : Int64 - y : Int16 - end - - fun foo : Struct - end - - str = LibFoo.foo - (str.x + str.y).to_i32 - ), &.to_i.should eq(3)) - end - it "returns struct bigger than 128 bits with sret" do mod = codegen(%( lib LibFoo @@ -294,35 +155,5 @@ require "../../../spec_helper" str.scan(/sret/).size.should eq(2) str.should contain("sret, i32") # sret goes as first argument end - - it "returns struct bigger than 128 bits with sret" do - test_c( - %( - struct s { - long x; - long y; - char z; - }; - - struct s foo(int z) { - struct s a = {1, 2, z}; - return a; - } - ), - %( - lib LibFoo - struct Struct - x : Int64 - y : Int64 - z : Int8 - end - - fun foo(w : Int32) : Struct - end - - str = LibFoo.foo(3) - (str.x + str.y + str.z).to_i32 - ), &.to_i.should eq(6)) - end end {% end %} From e33075f1c85dcd2c3b61021f8f220dfb559daec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 1 Jun 2020 22:51:38 +0200 Subject: [PATCH 088/263] Specify pkgconfig name for libevent (#9395) * Specify pkgconfig name for libevent * !fixup Specify pkgconfig name for libevent --- src/crystal/system/unix/lib_event2.cr | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/unix/lib_event2.cr b/src/crystal/system/unix/lib_event2.cr index f898c528bf27..b219aec307cc 100644 --- a/src/crystal/system/unix/lib_event2.cr +++ b/src/crystal/system/unix/lib_event2.cr @@ -7,11 +7,17 @@ require "c/netdb" {% if flag?(:openbsd) %} @[Link("event_core")] @[Link("event_extra")] +{% elsif compare_versions(Crystal::VERSION, "0.35.0-0") >= 0 %} + @[Link("event", pkg_config: "libevent")] {% else %} @[Link("event")] {% end %} {% if flag?(:preview_mt) %} - @[Link("event_pthreads")] + {% if compare_versions(Crystal::VERSION, "0.35.0-0") >= 0 %} + @[Link("event_pthreads", pkg_config: "libevent_pthreads")] + {% else %} + @[Link("event_pthreads")] + {% end %} {% end %} lib LibEvent2 alias Int = LibC::Int From c77c54a9f88f9e5351ceb6a5d8f8d2c92488d68e Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 1 Jun 2020 19:33:29 -0300 Subject: [PATCH 089/263] Array: fix delete_at bug with negative start index (#9399) * Array: fix delete_at bug with negative start index * Array#delete_at: check index out of bounds before calling self[...] --- spec/std/array_spec.cr | 14 ++++++++++++++ src/array.cr | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/spec/std/array_spec.cr b/spec/std/array_spec.cr index adb9e2d847ec..af5752f3c1fe 100644 --- a/spec/std/array_spec.cr +++ b/spec/std/array_spec.cr @@ -588,6 +588,20 @@ describe "Array" do a.should eq([1, 3, 4]) end + it "deletes negative index with range" do + a = [1, 2, 3, 4, 5, 6] + a.delete_at(-3, 2).should eq([4, 5]) + a.should eq([1, 2, 3, 6]) + end + + it "deletes negative index with range, out of bounds" do + a = [1, 2, 3, 4, 5, 6] + + expect_raises IndexError do + a.delete_at(-7, 2) + end + end + it "deletes out of bounds" do expect_raises IndexError do [1].delete_at(2) diff --git a/src/array.cr b/src/array.cr index c33bedb434ff..31042709f9cd 100644 --- a/src/array.cr +++ b/src/array.cr @@ -751,6 +751,11 @@ class Array(T) # a.delete_at(99, 1) # raises IndexError # ``` def delete_at(index : Int, count : Int) + index += size if index < 0 + unless 0 <= index <= size + raise IndexError.new + end + val = self[index, count] count = index + count <= size ? count : size - index (@buffer + index).move_from(@buffer + index + count, size - index - count) From baddee8a98a52bc81d0f4469ef1b447d07b01e66 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 2 Jun 2020 14:40:29 -0300 Subject: [PATCH 090/263] Log: check severity before backend (#9400) If the log must be discarded, it's better to check the severity first. Checking the backend first is a memory read operation and that's not needed if the severity will not match. --- src/log/log.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/log/log.cr b/src/log/log.cr index 5c93d14cfcd5..831d31a1b4dc 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -44,10 +44,11 @@ class Log } %} # Logs a message if the logger's current severity is lower or equal to `{{severity}}`. def {{method.id}}(*, exception : Exception? = nil) - return unless backend = @backend severity = Severity.new({{severity}}) return unless level <= severity + return unless backend = @backend + dsl = Emitter.new(@source, severity, exception) result = yield dsl entry = From 530ec52d1f841c724c088cccb5c478516548a1e8 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Wed, 3 Jun 2020 02:40:57 +0900 Subject: [PATCH 091/263] Docs: fix syntax highlighting of heredoc (#9396) --- .../crystal/tools/doc/highlighter_spec.cr | 32 +++++++++++++++++++ src/compiler/crystal/tools/doc/highlighter.cr | 28 ++++++++++++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/spec/compiler/crystal/tools/doc/highlighter_spec.cr b/spec/compiler/crystal/tools/doc/highlighter_spec.cr index 2eb188445b14..abce4c82a253 100644 --- a/spec/compiler/crystal/tools/doc/highlighter_spec.cr +++ b/spec/compiler/crystal/tools/doc/highlighter_spec.cr @@ -76,4 +76,36 @@ describe "Crystal::Doc::Highlighter#highlight" do it_highlights "%w(foo bar\n baz)", %(%w(foo bar\n baz)) it_highlights "%w", %(%w<foo bar baz>) it_highlights "%i(foo bar baz)", %(%i(foo bar baz)) + + it_highlights <<-CR, <<-HTML + foo, bar = <<-FOO, <<-BAR + foo + FOO + bar + BAR + CR + foo, bar = <<-FOO, <<-BAR + foo + FOO + bar + BAR + HTML + + it_highlights <<-CR, <<-HTML + foo, bar = <<-FOO, <<-BAR + foo + FOO + CR + foo, bar = <<-FOO, <<-BAR + foo + FOO + HTML + + it_highlights <<-CR, <<-HTML + foo, bar = <<-FOO, <<-BAR + foo + CR + foo, bar = <<-FOO, <<-BAR + foo + HTML end diff --git a/src/compiler/crystal/tools/doc/highlighter.cr b/src/compiler/crystal/tools/doc/highlighter.cr index 640f5096f0e8..6d0e387b76c3 100644 --- a/src/compiler/crystal/tools/doc/highlighter.cr +++ b/src/compiler/crystal/tools/doc/highlighter.cr @@ -17,12 +17,24 @@ module Crystal::Doc::Highlighter private def highlight_normal_state(lexer, io, break_on_rcurly = false) last_is_def = false + heredoc_stack = [] of Token while true token = lexer.next_token case token.type when :NEWLINE io.puts + heredoc_stack.each_with_index do |token, i| + highlight_delimiter_state lexer, token, io, heredoc: true + unless i == heredoc_stack.size - 1 + # Next token to heredoc's end is either NEWLINE or EOF. + # We can't continue highlighting when it is EOF even though + # heredoc tokens still remain. + break if lexer.next_token.type == :EOF + io.puts + end + end + heredoc_stack.clear when :SPACE io << token.value when :COMMENT @@ -36,7 +48,12 @@ module Crystal::Doc::Highlighter when :CONST, :"::" highlight token, "t", io when :DELIMITER_START - highlight_delimiter_state lexer, token, io + if token.delimiter_state.kind == :heredoc + highlight HTML.escape(token.raw), "s", io + heredoc_stack << token.dup + else + highlight_delimiter_state lexer, token, io + end when :STRING_ARRAY_START, :SYMBOL_ARRAY_START highlight_string_array lexer, token, io when :EOF @@ -83,17 +100,16 @@ module Crystal::Doc::Highlighter end end - private def highlight_delimiter_state(lexer, token, io) + private def highlight_delimiter_state(lexer, token, io, heredoc = false) start_highlight_class "s", io - HTML.escape(token.raw, io) + HTML.escape(token.raw, io) unless heredoc while true token = lexer.next_string_token(token.delimiter_state) case token.type when :DELIMITER_END HTML.escape(token.raw, io) - end_highlight_class io break when :INTERPOLATION_START end_highlight_class io @@ -101,12 +117,12 @@ module Crystal::Doc::Highlighter highlight_normal_state lexer, io, break_on_rcurly: true highlight "}", "i", io start_highlight_class "s", io - when :EOF - break else HTML.escape(token.raw, io) end end + ensure # This ensure is necessary to handle unterminated string literal. + end_highlight_class io end private def highlight_string_array(lexer, token, io) From dead87bafb051c13e1e57c6b1b84059f8563719d Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 2 Jun 2020 19:48:50 +0200 Subject: [PATCH 092/263] Improve spec helpers, prepare for compiler specs on Windows (#9351) * Move `pending_win32` macro to a common location * Fix Crystal.normalize_path for Windows * Improve spec helpers for temporary files and executables * Remove `no_win` macro * fixup! Move `pending_win32` macro to a common location --- spec/compiler/codegen/private_spec.cr | 16 +++---- spec/compiler/compiler_spec.cr | 4 +- spec/compiler/util_spec.cr | 20 ++++++++ spec/spec_helper.cr | 67 +++++++-------------------- spec/std/spec_helper.cr | 39 +++++----------- spec/support/tempfile.cr | 9 ++++ spec/support/win32.cr | 19 ++++++++ src/compiler/crystal/util.cr | 8 ++-- src/prelude.cr | 14 +++--- 9 files changed, 94 insertions(+), 102 deletions(-) create mode 100644 spec/compiler/util_spec.cr create mode 100644 spec/support/win32.cr diff --git a/spec/compiler/codegen/private_spec.cr b/spec/compiler/codegen/private_spec.cr index 613d0b0bd162..c2d2452952c7 100644 --- a/spec/compiler/codegen/private_spec.cr +++ b/spec/compiler/codegen/private_spec.cr @@ -14,11 +14,9 @@ describe "Codegen: private" do ] compiler.prelude = "empty" - output_filename = File.tempname("crystal-spec-output") - - compiler.compile sources, output_filename - ensure - File.delete(output_filename) if output_filename + with_temp_executable "crystal-spec-output" do |output_filename| + compiler.compile sources, output_filename + end end it "codegens overloaded private def in same file" do @@ -39,11 +37,9 @@ describe "Codegen: private" do ] compiler.prelude = "empty" - output_filename = File.tempname("crystal-spec-output") - - compiler.compile sources, output_filename - ensure - File.delete(output_filename) if output_filename + with_temp_executable "crystal-spec-output" do |output_filename| + compiler.compile sources, output_filename + end end it "doesn't include filename for private types" do diff --git a/spec/compiler/compiler_spec.cr b/spec/compiler/compiler_spec.cr index 6854302095fc..a0e24c795727 100644 --- a/spec/compiler/compiler_spec.cr +++ b/spec/compiler/compiler_spec.cr @@ -7,7 +7,7 @@ describe "Compiler" do end it "compiles a file" do - with_tempfile "compiler_spec_output" do |path| + with_temp_executable "compiler_spec_output" do |path| Crystal::Command.run ["build"].concat(program_flags_options).concat([compiler_datapath("compiler_sample"), "-o", path]) File.exists?(path).should be_true @@ -18,7 +18,7 @@ describe "Compiler" do it "runs subcommand in preference to a filename " do Dir.cd compiler_datapath do - with_tempfile "compiler_spec_output" do |path| + with_temp_executable "compiler_spec_output" do |path| Crystal::Command.run ["build"].concat(program_flags_options).concat(["compiler_sample", "-o", path]) File.exists?(path).should be_true diff --git a/spec/compiler/util_spec.cr b/spec/compiler/util_spec.cr new file mode 100644 index 000000000000..9a4815a9e588 --- /dev/null +++ b/spec/compiler/util_spec.cr @@ -0,0 +1,20 @@ +require "spec" +require "compiler/crystal/util" + +describe Crystal do + describe "normalize_path" do + sep = {{ flag?(:win32) ? "\\" : "/" }} + + it { Crystal.normalize_path("a").should eq ".#{sep}a" } + it { Crystal.normalize_path("./a/b").should eq ".#{sep}a#{sep}b" } + it { Crystal.normalize_path("../a/b").should eq ".#{sep}..#{sep}a#{sep}b" } + it { Crystal.normalize_path("/foo/bar").should eq "#{sep}foo#{sep}bar" } + + {% if flag?(:win32) %} + it { Crystal.normalize_path("C:\\foo\\bar").should eq "C:\\foo\\bar" } + it { Crystal.normalize_path("C:foo\\bar").should eq "C:foo\\bar" } + it { Crystal.normalize_path("\\foo\\bar").should eq "\\foo\\bar" } + it { Crystal.normalize_path("foo\\bar").should eq ".\\foo\\bar" } + {% end %} + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 0d17a2118e94..0a67b2f9f802 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -4,10 +4,10 @@ ENV["CRYSTAL_PATH"] = "#{__DIR__}/../src" require "spec" -{% skip_file if flag?(:win32) %} - require "../src/compiler/crystal/**" require "./support/syntax" +require "./support/tempfile" +require "./support/win32" class Crystal::Program def union_of(type1, type2, type3) @@ -114,22 +114,23 @@ def assert_no_errors(*args) semantic(*args) end -def warnings_result(code) - output_filename = Crystal.temp_executable("crystal-spec-output") - +def warnings_result(code, *, file = __FILE__) compiler = create_spec_compiler compiler.warnings = Warnings::All compiler.error_on_warnings = false compiler.prelude = "empty" # avoid issues in the current std lib compiler.color = false apply_program_flags(compiler.flags) - result = compiler.compile Compiler::Source.new("code.cr", code), output_filename - result.program.warning_failures + with_temp_executable("crystal-spec-output", file: file) do |output_filename| + result = compiler.compile Compiler::Source.new("code.cr", code), output_filename + + return result.program.warning_failures + end end def assert_warning(code, message, *, file = __FILE__, line = __LINE__) - warning_failures = warnings_result(code) + warning_failures = warnings_result(code, file: file) warning_failures.size.should eq(1), file, line warning_failures[0].should start_with(message), file, line end @@ -220,7 +221,7 @@ def create_spec_compiler compiler end -def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil) +def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug::None, flags = nil, *, file = __FILE__) if inject_primitives code = %(require "primitives"\n#{code}) end @@ -239,53 +240,24 @@ def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug:: ast.expressions[-1] = exps code = ast.to_s - output_filename = Crystal.temp_executable("crystal-spec-output") - compiler = create_spec_compiler compiler.debug = debug compiler.flags.concat flags if flags apply_program_flags(compiler.flags) - compiler.compile Compiler::Source.new("spec", code), output_filename - output = `#{Process.quote(output_filename)}` - File.delete(output_filename) + with_temp_executable("crystal-spec-output", file: file) do |output_filename| + compiler.compile Compiler::Source.new("spec", code), output_filename - SpecRunOutput.new(output) + output = `#{Process.quote(output_filename)}` + return SpecRunOutput.new(output) + end else new_program.run(code, filename: filename, debug: debug) end end -def build(code) - code_file = File.tempname("build_and_run_code") - - # write code to the temp file - File.write(code_file, code) - - binary_file = File.tempname("build_and_run_bin") - - `bin/crystal build #{encode_program_flags} #{Process.quote(code_file.path.to_s)} -o #{Process.quote(binary_file.path.to_s)}` - File.exists?(binary_file).should be_true - - yield binary_file -ensure - File.delete(code_file) if code_file - File.delete(binary_file) if binary_file -end - -def build_and_run(code) - build(code) do |binary_file| - out_io, err_io = IO::Memory.new, IO::Memory.new - status = Process.run(binary_file, output: out_io, error: err_io) - - {status, out_io.to_s, err_io.to_s} - end -end - -def test_c(c_code, crystal_code) - c_filename = "#{__DIR__}/temp_abi.c" - o_filename = "#{__DIR__}/temp_abi.o" - begin +def test_c(c_code, crystal_code, *, file = __FILE__) + with_tempfile("temp_abi.c", "temp_abi.o", file: file) do |c_filename, o_filename| File.write(c_filename, c_code) `#{Crystal::Compiler::CC} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy @@ -293,11 +265,8 @@ def test_c(c_code, crystal_code) yield run(%( require "prelude" - @[Link(ldflags: "#{o_filename}")] + @[Link(ldflags: #{o_filename.inspect})] #{crystal_code} )) - ensure - File.delete(c_filename) - File.delete(o_filename) end end diff --git a/spec/std/spec_helper.cr b/spec/std/spec_helper.cr index faee17463a46..b87590327247 100644 --- a/spec/std/spec_helper.cr +++ b/spec/std/spec_helper.cr @@ -1,29 +1,12 @@ require "spec" require "../support/tempfile" require "../support/fibers" +require "../support/win32" def datapath(*components) File.join("spec", "std", "data", *components) end -{% if flag?(:win32) %} - def pending_win32(description = "assert", file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) - pending("#{description} [win32]", file, line, end_line) - end - - def pending_win32(*, describe, file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) - pending_win32(describe, file, line, end_line) { } - end -{% else %} - def pending_win32(description = "assert", file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) - it(description, file, line, end_line, &block) - end - - def pending_win32(*, describe, file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) - describe(describe, file, line, end_line, &block) - end -{% end %} - private class Witness @checked = false @@ -92,8 +75,8 @@ def spawn_and_check(before : Proc(_), file = __FILE__, line = __LINE__, &block : end end -def compile_file(source_file, flags = %w(--debug)) - with_tempfile("executable_file") do |executable_file| +def compile_file(source_file, flags = %w(--debug), file = __FILE__) + with_temp_executable("executable_file", file: file) do |executable_file| Process.run("bin/crystal", ["build"] + flags + ["-o", executable_file, source_file]) File.exists?(executable_file).should be_true @@ -101,17 +84,17 @@ def compile_file(source_file, flags = %w(--debug)) end end -def compile_source(source, flags = %w(--debug)) - with_tempfile("source_file") do |source_file| +def compile_source(source, flags = %w(--debug), file = __FILE__) + with_tempfile("source_file", file: file) do |source_file| File.write(source_file, source) - compile_file(source_file, flags) do |executable_file| + compile_file(source_file, flags, file: file) do |executable_file| yield executable_file end end end -def compile_and_run_file(source_file, flags = %w(--debug)) - compile_file(source_file) do |executable_file| +def compile_and_run_file(source_file, flags = %w(--debug), file = __FILE__) + compile_file(source_file, file: file) do |executable_file| output, error = IO::Memory.new, IO::Memory.new status = Process.run executable_file, output: output, error: error @@ -119,9 +102,9 @@ def compile_and_run_file(source_file, flags = %w(--debug)) end end -def compile_and_run_source(source, flags = %w(--debug)) - with_tempfile("source_file") do |source_file| +def compile_and_run_source(source, flags = %w(--debug), file = __FILE__) + with_tempfile("source_file", file: file) do |source_file| File.write(source_file, source) - compile_and_run_file(source_file, flags) + compile_and_run_file(source_file, flags, file: file) end end diff --git a/spec/support/tempfile.cr b/spec/support/tempfile.cr index 4f527e35329c..0c47ee765e90 100644 --- a/spec/support/tempfile.cr +++ b/spec/support/tempfile.cr @@ -34,6 +34,15 @@ def with_tempfile(*paths, file = __FILE__) end end +def with_temp_executable(name, file = __FILE__) + {% if flag?(:win32) %} + name += ".exe" + {% end %} + with_tempfile(name, file: file) do |tempname| + yield tempname + end +end + if SPEC_TEMPFILE_CLEANUP at_exit do FileUtils.rm_r(SPEC_TEMPFILE_PATH) if Dir.exists?(SPEC_TEMPFILE_PATH) diff --git a/spec/support/win32.cr b/spec/support/win32.cr new file mode 100644 index 000000000000..da82d1ca24c3 --- /dev/null +++ b/spec/support/win32.cr @@ -0,0 +1,19 @@ +require "spec" + +{% if flag?(:win32) %} + def pending_win32(description = "assert", file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) + pending("#{description} [win32]", file, line, end_line) + end + + def pending_win32(*, describe, file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) + pending_win32(describe, file, line, end_line) { } + end +{% else %} + def pending_win32(description = "assert", file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) + it(description, file, line, end_line, &block) + end + + def pending_win32(*, describe, file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) + describe(describe, file, line, end_line, &block) + end +{% end %} diff --git a/src/compiler/crystal/util.cr b/src/compiler/crystal/util.cr index fe1ef625cb9c..578396c0c888 100644 --- a/src/compiler/crystal/util.cr +++ b/src/compiler/crystal/util.cr @@ -62,10 +62,8 @@ module Crystal end def self.normalize_path(path) - path_start = ".#{File::SEPARATOR}" - unless path.starts_with?(path_start) || path.starts_with?(File::SEPARATOR) - path = path_start + path - end - path.rstrip(File::SEPARATOR) + path = ::Path[path].normalize + path = ::Path["."] / path unless path.anchor + path.to_s end end diff --git a/src/prelude.cr b/src/prelude.cr index 810e27b395fc..5ccd0f959358 100644 --- a/src/prelude.cr +++ b/src/prelude.cr @@ -7,12 +7,6 @@ # to also add them to `docs_main.cr` if their content needs to # appear in the API docs. -private macro no_win(stmt) - {% unless flag?(:win32) %} - {{stmt}} - {% end %} -end - # This list requires ordered statements require "crystal/once" require "lib_c" @@ -57,7 +51,9 @@ require "intrinsics" require "io" require "kernel" require "math/math" -no_win require "mutex" +{% unless flag?(:win32) %} + require "mutex" +{% end %} require "named_tuple" require "nil" require "humanize" @@ -73,7 +69,9 @@ require "range" require "reference" require "regex" require "set" -no_win require "signal" +{% unless flag?(:win32) %} + require "signal" +{% end %} require "slice" require "static_array" require "struct" From c2113aadc078ec9a8ac7b7969a298c63503a7581 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 2 Jun 2020 15:13:37 -0300 Subject: [PATCH 093/263] Hide internal __ fun from docs (#9410) --- src/crystal/compiler_rt/mulodi4.cr | 1 + src/crystal/once.cr | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/crystal/compiler_rt/mulodi4.cr b/src/crystal/compiler_rt/mulodi4.cr index b592a91b69f7..e853b89b13aa 100644 --- a/src/crystal/compiler_rt/mulodi4.cr +++ b/src/crystal/compiler_rt/mulodi4.cr @@ -1,3 +1,4 @@ +# :nodoc: fun __mulodi4(a : Int64, b : Int64, overflow : Int32*) : Int64 n = 64 min = Int64::MIN diff --git a/src/crystal/once.cr b/src/crystal/once.cr index fc915198b6c1..1e6243669809 100644 --- a/src/crystal/once.cr +++ b/src/crystal/once.cr @@ -40,10 +40,12 @@ class Crystal::OnceState {% end %} end +# :nodoc: fun __crystal_once_init : Void* Crystal::OnceState.new.as(Void*) end +# :nodoc: fun __crystal_once(state : Void*, flag : Bool*, initializer : Void*) state.as(Crystal::OnceState).once(flag, initializer) end From ea9d9b14ea2ad0ad2a0a1640180fd779f386d921 Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Wed, 3 Jun 2020 14:33:49 +0200 Subject: [PATCH 094/263] Minor INI improvements (#9408) * Change INI class to module INI is more a module, having no constructor. * Rename `str` argument to `string_or_io` --- src/ini.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ini.cr b/src/ini.cr index 2aaad871d6c0..4936cbecc327 100644 --- a/src/ini.cr +++ b/src/ini.cr @@ -1,4 +1,4 @@ -class INI +module INI # Exception thrown on an INI parse error. class ParseException < Exception getter line_number : Int32 @@ -21,12 +21,12 @@ class INI # # INI.parse("[foo]\na = 1") # => {"foo" => {"a" => "1"}} # ``` - def self.parse(str : String | IO) : Hash(String, Hash(String, String)) + def self.parse(string_or_io : String | IO) : Hash(String, Hash(String, String)) ini = Hash(String, Hash(String, String)).new current_section = ini[""] = Hash(String, String).new lineno = 0 - str.each_line do |line| + string_or_io.each_line do |line| lineno += 1 next if line.empty? From df026e1f1a8e3cbb82b957397aee3165a5295c65 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Wed, 3 Jun 2020 14:35:02 +0200 Subject: [PATCH 095/263] Fix compile time checking of method definition (#9407) * Fix compile time checking of method definition * Use `has_method?` --- src/crystal/system/unix/file_descriptor.cr | 2 +- src/crystal/system/unix/time.cr | 2 +- src/crystal/system/win32/file_descriptor.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crystal/system/unix/file_descriptor.cr b/src/crystal/system/unix/file_descriptor.cr index 40de66b85dbc..10a1bbe0e772 100644 --- a/src/crystal/system/unix/file_descriptor.cr +++ b/src/crystal/system/unix/file_descriptor.cr @@ -90,7 +90,7 @@ module Crystal::System::FileDescriptor end private def system_reopen(other : IO::FileDescriptor) - {% if LibC.methods.includes? "dup3".id %} + {% if LibC.has_method?("dup3") %} # dup doesn't copy the CLOEXEC flag, so copy it manually using dup3 flags = other.close_on_exec? ? LibC::O_CLOEXEC : 0 if LibC.dup3(other.fd, fd, flags) == -1 diff --git a/src/crystal/system/unix/time.cr b/src/crystal/system/unix/time.cr index 5d15f5275070..9af57648aa26 100644 --- a/src/crystal/system/unix/time.cr +++ b/src/crystal/system/unix/time.cr @@ -14,7 +14,7 @@ module Crystal::System::Time UnixEpochInSeconds = 62135596800_i64 def self.compute_utc_seconds_and_nanoseconds : {Int64, Int32} - {% if LibC.methods.includes?("clock_gettime".id) %} + {% if LibC.has_method?("clock_gettime") %} ret = LibC.clock_gettime(LibC::CLOCK_REALTIME, out timespec) raise RuntimeError.from_errno("clock_gettime") unless ret == 0 {timespec.tv_sec.to_i64 + UnixEpochInSeconds, timespec.tv_nsec.to_i} diff --git a/src/crystal/system/win32/file_descriptor.cr b/src/crystal/system/win32/file_descriptor.cr index 62fa4332a35d..0fe008c93814 100644 --- a/src/crystal/system/win32/file_descriptor.cr +++ b/src/crystal/system/win32/file_descriptor.cr @@ -97,7 +97,7 @@ module Crystal::System::FileDescriptor end private def system_reopen(other : IO::FileDescriptor) - {% if LibC.methods.includes? "dup3".id %} + {% if LibC.has_method?("dup3") %} # dup doesn't copy the CLOEXEC flag, so copy it manually using dup3 flags = other.close_on_exec? ? LibC::O_CLOEXEC : 0 if LibC.dup3(other.fd, self.fd, flags) == -1 From 48e9289f5d17ec31e9dd6dda1e37f407877170a8 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Wed, 3 Jun 2020 14:40:31 +0200 Subject: [PATCH 096/263] Basic support for Win64 C lib ABI (#9387) * Split general ABI specs from x86_64-specific ones, run on every platform To confirm that the newly added file only moves things: $ git diff -w HEAD~:spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr spec/compiler/codegen/c_abi/c_abi_spec.cr * Implement basics of Win64 lib ABI Also un-does #5851 --- spec/compiler/codegen/c_abi/c_abi_spec.cr | 16 +++++----- .../codegen/c_abi/c_abi_x86_64_spec.cr | 2 +- spec/compiler/codegen/extern_spec.cr | 6 ++-- spec/compiler/codegen/primitives_spec.cr | 29 ++++++++++--------- src/compiler/crystal/semantic/main_visitor.cr | 3 ++ src/llvm/abi.cr | 1 + src/llvm/abi/x86.cr | 1 + src/llvm/abi/x86_64.cr | 1 + src/llvm/abi/x86_win64.cr | 22 ++++++++++++++ src/llvm/target_machine.cr | 2 ++ 10 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 src/llvm/abi/x86_win64.cr diff --git a/spec/compiler/codegen/c_abi/c_abi_spec.cr b/spec/compiler/codegen/c_abi/c_abi_spec.cr index 2faaee55ce96..1a772b897ae2 100644 --- a/spec/compiler/codegen/c_abi/c_abi_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_spec.cr @@ -32,11 +32,11 @@ describe "Code gen: C ABI" do test_c( %( struct s { - long x; + long long x; short y; }; - long foo(struct s a) { + long long foo(struct s a) { return a.x + a.y; } ), @@ -59,12 +59,12 @@ describe "Code gen: C ABI" do test_c( %( struct s { - long x; - long y; + long long x; + long long y; char z; }; - long foo(struct s a) { + long long foo(struct s a) { return a.x + a.y + a.z; } ), @@ -116,7 +116,7 @@ describe "Code gen: C ABI" do test_c( %( struct s { - long x; + long long x; short y; }; @@ -144,8 +144,8 @@ describe "Code gen: C ABI" do test_c( %( struct s { - long x; - long y; + long long x; + long long y; char z; }; diff --git a/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr b/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr index 3c0cd2122bd1..84a92e7075f6 100644 --- a/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_x86_64_spec.cr @@ -1,6 +1,6 @@ require "../../../spec_helper" -{% if flag?(:x86_64) %} +{% if flag?(:x86_64) && !flag?(:win32) %} describe "Code gen: C ABI x86_64" do it "passes struct less than 64 bits as { i64 }" do mod = codegen(%( diff --git a/spec/compiler/codegen/extern_spec.cr b/spec/compiler/codegen/extern_spec.cr index 535358943e37..e1f0e401e18a 100644 --- a/spec/compiler/codegen/extern_spec.cr +++ b/spec/compiler/codegen/extern_spec.cr @@ -431,9 +431,9 @@ describe "Codegen: extern struct" do test_c( %( struct Struct { - long x; - long y; - long z; + long long x; + long long y; + long long z; }; struct Struct foo(struct Struct (*callback)(struct Struct)) { diff --git a/spec/compiler/codegen/primitives_spec.cr b/spec/compiler/codegen/primitives_spec.cr index fec15c2eeaa9..20e768e66086 100644 --- a/spec/compiler/codegen/primitives_spec.cr +++ b/spec/compiler/codegen/primitives_spec.cr @@ -239,22 +239,25 @@ describe "Code gen: primitives" do end describe "va_arg" do - it "uses llvm's va_arg instruction" do - mod = codegen(%( - struct VaList - @[Primitive(:va_arg)] - def next(type) + # On Windows llvm's va_arg instruction works incorrectly. + {% unless flag?(:win32) %} + it "uses llvm's va_arg instruction" do + mod = codegen(%( + struct VaList + @[Primitive(:va_arg)] + def next(type) + end end - end - list = VaList.new - list.next(Int32) - )) - str = mod.to_s - str.should contain("va_arg %VaList* %list") - end + list = VaList.new + list.next(Int32) + )) + str = mod.to_s + str.should contain("va_arg %VaList* %list") + end + {% end %} - it "works with C code" do + pending_win32 "works with C code" do test_c( %( extern int foo_f(int,...); diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index c6b4ec4009c4..a4cff837c390 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -2431,6 +2431,9 @@ module Crystal end def visit_va_arg(node) + if program.has_flag? "windows" + node.raise "va_arg is not yet supported on Windows" + end arg = call.not_nil!.args[0]? || node.raise("requires type argument") node.type = arg.type.instance_type end diff --git a/src/llvm/abi.cr b/src/llvm/abi.cr index 323269741b37..fffcdf29ad64 100644 --- a/src/llvm/abi.cr +++ b/src/llvm/abi.cr @@ -1,3 +1,4 @@ +# Based on https://github.com/rust-lang/rust/blob/29ac04402d53d358a1f6200bea45a301ff05b2d1/src/librustc_trans/trans/cabi.rs abstract class LLVM::ABI getter target_data : TargetData getter? osx : Bool diff --git a/src/llvm/abi/x86.cr b/src/llvm/abi/x86.cr index 64976863a47c..abc67c4a0471 100644 --- a/src/llvm/abi/x86.cr +++ b/src/llvm/abi/x86.cr @@ -1,5 +1,6 @@ require "../abi" +# Based on https://github.com/rust-lang/rust/blob/29ac04402d53d358a1f6200bea45a301ff05b2d1/src/librustc_trans/trans/cabi_x86.rs class LLVM::ABI::X86 < LLVM::ABI def abi_info(atys : Array(Type), rty : Type, ret_def : Bool, context : Context) ret_ty = compute_return_type(rty, ret_def, context) diff --git a/src/llvm/abi/x86_64.cr b/src/llvm/abi/x86_64.cr index 8248b3c218fc..cb12d74a3364 100644 --- a/src/llvm/abi/x86_64.cr +++ b/src/llvm/abi/x86_64.cr @@ -1,5 +1,6 @@ require "../abi" +# Based on https://github.com/rust-lang/rust/blob/29ac04402d53d358a1f6200bea45a301ff05b2d1/src/librustc_trans/trans/cabi_x86_64.rs class LLVM::ABI::X86_64 < LLVM::ABI def abi_info(atys : Array(Type), rty : Type, ret_def : Bool, context : Context) arg_tys = Array(LLVM::Type).new(atys.size) diff --git a/src/llvm/abi/x86_win64.cr b/src/llvm/abi/x86_win64.cr new file mode 100644 index 000000000000..5a9a07bdb31e --- /dev/null +++ b/src/llvm/abi/x86_win64.cr @@ -0,0 +1,22 @@ +require "../abi" + +# Based on https://github.com/rust-lang/rust/blob/29ac04402d53d358a1f6200bea45a301ff05b2d1/src/librustc_trans/trans/cabi_x86_win64.rs +class LLVM::ABI::X86_Win64 < LLVM::ABI::X86 + private def compute_arg_types(atys, context) + atys.map do |t| + case t.kind + when Type::Kind::Struct + size = target_data.abi_size(t) + case size + when 1 then ArgType.direct(t, context.int8) + when 2 then ArgType.direct(t, context.int16) + when 4 then ArgType.direct(t, context.int32) + when 8 then ArgType.direct(t, context.int64) + else ArgType.indirect(t, LLVM::Attribute::ByVal) + end + else + non_struct(t, context) + end + end + end +end diff --git a/src/llvm/target_machine.cr b/src/llvm/target_machine.cr index e9c8b793aa7f..d5c0fc2c8db6 100644 --- a/src/llvm/target_machine.cr +++ b/src/llvm/target_machine.cr @@ -43,6 +43,8 @@ class LLVM::TargetMachine def abi triple = self.triple case triple + when /x86_64.+windows-msvc/ + ABI::X86_Win64.new(self) when /x86_64|amd64/ ABI::X86_64.new(self) when /i386|i486|i586|i686/ From 52603da8beea0fa38d4b6b129087ed9b1ef4372d Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 3 Jun 2020 10:13:36 -0300 Subject: [PATCH 097/263] Fix http specs on mt (#9412) * Fix race condition in http specs Add timeout to support/fibers * Keep the wait_for listening Otherwise run_server call in http_server#bind specs: * fails after listen * fails after close will yield before the server that listening --- spec/std/http/spec_helper.cr | 4 +++- spec/support/fibers.cr | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/spec/std/http/spec_helper.cr b/spec/std/http/spec_helper.cr index 8fe39b8dcd07..e3c23a5ff448 100644 --- a/spec/std/http/spec_helper.cr +++ b/spec/std/http/spec_helper.cr @@ -1,5 +1,6 @@ require "spec" require "../spec_helper" +require "../../support/fibers" private def wait_for(timeout = 5.seconds) now = Time.monotonic @@ -25,7 +26,7 @@ end def run_server(server) server_done = Channel(Exception?).new - spawn do + f = spawn do server.listen rescue exc server_done.send exc @@ -35,6 +36,7 @@ def run_server(server) begin wait_for { server.listening? } + wait_until_blocked f yield server_done ensure diff --git a/spec/support/fibers.cr b/spec/support/fibers.cr index 4856233c338b..50e5b38e300f 100644 --- a/spec/support/fibers.cr +++ b/spec/support/fibers.cr @@ -1,11 +1,16 @@ -def wait_until_blocked(f : Fiber) +def wait_until_blocked(f : Fiber, timeout = 5.seconds) + now = Time.monotonic + until f.resumable? Fiber.yield + raise "fiber failed to block within #{timeout}" if (Time.monotonic - now) > timeout end end -def wait_until_finished(f : Fiber) +def wait_until_finished(f : Fiber, timeout = 5.seconds) + now = Time.monotonic until f.dead? Fiber.yield + raise "fiber failed to finish within #{timeout}" if (Time.monotonic - now) > timeout end end From e50bc64044eef246931e28414b8d19bce68d24b4 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 3 Jun 2020 10:14:24 -0300 Subject: [PATCH 098/263] WebSocket doesn't reply with same close code (#9313) --- src/http/web_socket.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/web_socket.cr b/src/http/web_socket.cr index 5afe899797de..18ddca94433d 100644 --- a/src/http/web_socket.cr +++ b/src/http/web_socket.cr @@ -159,7 +159,7 @@ class HTTP::WebSocket message = @current_message.gets_to_end @on_close.try &.call(code, message) - close(code) + close @current_message.clear break From 70bec07995f847d115a4bdf12bfd27e41c86af4f Mon Sep 17 00:00:00 2001 From: Todd Sundsted Date: Wed, 3 Jun 2020 09:30:39 -0400 Subject: [PATCH 099/263] Ensure `type_vars` works for generic modules. (#9161) --- spec/compiler/codegen/macro_spec.cr | 11 +++++++++++ src/compiler/crystal/macros/methods.cr | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/compiler/codegen/macro_spec.cr b/spec/compiler/codegen/macro_spec.cr index 8ed6dd31a49e..9f3e6e1211d1 100644 --- a/spec/compiler/codegen/macro_spec.cr +++ b/spec/compiler/codegen/macro_spec.cr @@ -578,6 +578,17 @@ describe "Code gen: macro" do )).to_string.should eq("Int32") end + it "can acccess type variables of a module" do + run(%( + module Foo(T) + def self.foo + {{ @type.type_vars.first.name.stringify }} + end + end + Foo(Int32).foo + )).to_string.should eq("Int32") + end + it "can acccess type variables that are not types" do run(%( class Foo(T) diff --git a/src/compiler/crystal/macros/methods.cr b/src/compiler/crystal/macros/methods.cr index 3a184b72bee2..4bf8f5e91560 100644 --- a/src/compiler/crystal/macros/methods.cr +++ b/src/compiler/crystal/macros/methods.cr @@ -1749,7 +1749,7 @@ module Crystal end def self.type_vars(type) - if type.is_a?(GenericClassInstanceType) + if type.is_a?(GenericClassInstanceType) || type.is_a?(GenericModuleInstanceType) if type.is_a?(TupleInstanceType) if type.tuple_types.empty? empty_no_return_array From 946b4e6f2fdb4cc353423bdac343aef6f3725412 Mon Sep 17 00:00:00 2001 From: Kubo Takehiro Date: Thu, 4 Jun 2020 04:20:55 +0900 Subject: [PATCH 100/263] Implement File#fsync on Windows (#9257) --- src/crystal/system/win32/file.cr | 4 +++- src/file.cr | 3 +++ src/lib_c/x86_64-windows-msvc/c/io.cr | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 6832d77d37f7..a1574733e602 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -255,6 +255,8 @@ module Crystal::System::File end private def system_fsync(flush_metadata = true) : Nil - raise NotImplementedError.new("File#fsync") + if LibC._commit(fd) != 0 + raise IO::Error.from_errno("Error syncing file") + end end end diff --git a/src/file.cr b/src/file.cr index 879cac96c5aa..2ce83afc457e 100644 --- a/src/file.cr +++ b/src/file.cr @@ -809,6 +809,9 @@ class File < IO::FileDescriptor # then the syscall *fdatasync* will be used and only data required for # subsequent data retrieval is flushed. Metadata such as modified time and # access time is not written. + # + # NOTE: Metadata is flushed even when *flush_metadata* is false on Windows + # and DragonFly BSD. def fsync(flush_metadata = true) : Nil flush system_fsync(flush_metadata) diff --git a/src/lib_c/x86_64-windows-msvc/c/io.cr b/src/lib_c/x86_64-windows-msvc/c/io.cr index 0dd6d93bf7e0..dc3dfd345523 100644 --- a/src/lib_c/x86_64-windows-msvc/c/io.cr +++ b/src/lib_c/x86_64-windows-msvc/c/io.cr @@ -15,4 +15,5 @@ lib LibC fun _get_osfhandle(fd : Int) : IntPtrT fun _pipe(pfds : Int*, psize : UInt, textmode : Int) : Int fun _dup2(fd1 : Int, fd2 : Int) : Int + fun _commit(fd : Int) : Int end From 50813d4d3d402c298a6e0f17afd28440fd6ef07e Mon Sep 17 00:00:00 2001 From: Stephanie Wilde-Hobbs Date: Wed, 3 Jun 2020 20:21:36 +0100 Subject: [PATCH 101/263] Improve compiler single-file run syntax (#9171) * Improve compiler single-file run syntax Now the compiler passes all arguments after the first filename to the program. * Fix args_test.cr being required as a spec --- spec/compiler/compiler_spec.cr | 11 +++++++++++ spec/compiler/data/args_test | 7 +++++++ src/compiler/crystal/command.cr | 7 +++++++ 3 files changed, 25 insertions(+) create mode 100644 spec/compiler/data/args_test diff --git a/spec/compiler/compiler_spec.cr b/spec/compiler/compiler_spec.cr index a0e24c795727..ce7c086f99ec 100644 --- a/spec/compiler/compiler_spec.cr +++ b/spec/compiler/compiler_spec.cr @@ -27,4 +27,15 @@ describe "Compiler" do end end end + + it "treats all arguments post-filename as program arguments" do + with_tempfile "args_test" do |path| + `bin/crystal '#{compiler_datapath}/args_test' -Dother_flag -- bar '#{path}'` + + File.read(path).should eq(<<-FILE) + ["-Dother_flag", "--", "bar"] + {other_flag: false} + FILE + end + end end diff --git a/spec/compiler/data/args_test b/spec/compiler/data/args_test new file mode 100644 index 000000000000..86efc6f59fbb --- /dev/null +++ b/spec/compiler/data/args_test @@ -0,0 +1,7 @@ +# Last argument is file path to write +test_path = ARGV.pop +File.open(test_path, "w") do |f| + ARGV.inspect(f) + f.puts + {other_flag: {{flag?(:other_flag)}}}.inspect(f) +end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 497682293486..7a6d5b3f60aa 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -462,6 +462,13 @@ class Crystal::Command filenames << stdin_filename end + if single_file + opts.before_each do |arg| + opts.stop if !arg.starts_with?('-') && arg.ends_with?(".cr") + opts.stop if File.file?(arg) + end + end + opts.unknown_args do |before, after| opt_filenames = before opt_arguments = after From 1a8ea042a96219d4571682c25559d42a8fc6f5c8 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 3 Jun 2020 16:21:57 -0300 Subject: [PATCH 102/263] Fix overflow checking for operations with mixed sign (#9403) * Fix arithmetic operators with overflow * Special case for multiplication within the codegen * Disable overflow specs before 0.35 --- spec/std/overflow_spec.cr | 54 ++++++ src/compiler/crystal/codegen/primitives.cr | 214 ++++++++------------- src/compiler/crystal/program.cr | 24 +++ 3 files changed, 157 insertions(+), 135 deletions(-) create mode 100644 spec/std/overflow_spec.cr diff --git a/spec/std/overflow_spec.cr b/spec/std/overflow_spec.cr new file mode 100644 index 000000000000..460e4882a23a --- /dev/null +++ b/spec/std/overflow_spec.cr @@ -0,0 +1,54 @@ +{% skip_file unless compare_versions(Crystal::VERSION, "0.35.0-0") > 0 %} + +require "big" +require "spec" + +{% for i in Int::Signed.union_types %} + struct {{i}} + TEST_CASES = [MIN, MIN &+ 1, MIN &+ 2, -1, 0, 1, MAX &- 2, MAX &- 1, MAX] of {{i}} + end +{% end %} + +{% for i in Int::Unsigned.union_types %} + struct {{i}} + TEST_CASES = [MIN, MIN &+ 1, MIN &+ 2, MAX &- 2, MAX &- 1, MAX] of {{i}} + end +{% end %} + +macro run_op_tests(t, u, op) + it "overflow test #{{{t}}} #{{{op}}} #{{{u}}}" do + {{t}}::TEST_CASES.each do |lhs| + {{u}}::TEST_CASES.each do |rhs| + result = lhs.to_big_i {{op.id}} rhs.to_big_i + passes = {{t}}::MIN <= result <= {{t}}::MAX + begin + if passes + (lhs {{op.id}} rhs).should eq(lhs &{{op.id}} rhs) + else + expect_raises(OverflowError) { lhs {{op.id}} rhs } + end + rescue e : Spec::AssertionFailed + raise Spec::AssertionFailed.new("#{e.message}: #{lhs} #{{{op}}} #{rhs}", e.file, e.line) + rescue e : OverflowError + raise OverflowError.new("#{e.message}: #{lhs} #{{{op}}} #{rhs}") + end + end + end + end +end + +{% if flag?(:darwin) %} + private OVERFLOW_TEST_TYPES = [Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128] +{% else %} + private OVERFLOW_TEST_TYPES = [Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64] +{% end %} + +describe "overflow" do + {% for t in OVERFLOW_TEST_TYPES %} + {% for u in OVERFLOW_TEST_TYPES %} + run_op_tests {{t}}, {{u}}, :+ + run_op_tests {{t}}, {{u}}, :- + run_op_tests {{t}}, {{u}}, :* + {% end %} + {% end %} +end diff --git a/src/compiler/crystal/codegen/primitives.cr b/src/compiler/crystal/codegen/primitives.cr index 94fc88c44810..13ae41ea4f04 100644 --- a/src/compiler/crystal/codegen/primitives.cr +++ b/src/compiler/crystal/codegen/primitives.cr @@ -131,12 +131,15 @@ class Crystal::CodeGenVisitor else # go on end + case op + when "+", "-", "*" + return codegen_binary_op_with_overflow(op, t1, t2, p1, p2) + else # go on + end + tmax, p1, p2 = codegen_binary_extend_int(t1, t2, p1, p2) case op - when "+" then codegen_binary_op_add(tmax, t1, t2, p1, p2) - when "-" then codegen_binary_op_sub(tmax, t1, t2, p1, p2) - when "*" then codegen_binary_op_mul(tmax, t1, t2, p1, p2) when "&+" then codegen_trunc_binary_op_result(t1, t2, builder.add(p1, p2)) when "&-" then codegen_trunc_binary_op_result(t1, t2, builder.sub(p1, p2)) when "&*" then codegen_trunc_binary_op_result(t1, t2, builder.mul(p1, p2)) @@ -151,6 +154,79 @@ class Crystal::CodeGenVisitor end end + def codegen_binary_op_with_overflow(op, t1, t2, p1, p2) + if op == "*" + if t1.unsigned? && t2.signed? + return codegen_mul_unsigned_signed_with_overflow(t1, t2, p1, p2) + elsif t1.signed? && t2.unsigned? + return codegen_mul_signed_unsigned_with_overflow(t1, t2, p1, p2) + end + end + + calc_signed = t1.signed? || t2.signed? + calc_width = {t1, t2}.map { |t| t.bytes * 8 + ((calc_signed && t.unsigned?) ? 1 : 0) }.max + calc_type = llvm_context.int(calc_width) + + e1 = t1.signed? ? builder.sext(p1, calc_type) : builder.zext(p1, calc_type) + e2 = t2.signed? ? builder.sext(p2, calc_type) : builder.zext(p2, calc_type) + + llvm_op = + case {calc_signed, op} + when {false, "+"} then "uadd" + when {false, "-"} then "usub" + when {false, "*"} then "umul" + when {true, "+"} then "sadd" + when {true, "-"} then "ssub" + when {true, "*"} then "smul" + else raise "BUG: unknown overflow op" + end + + llvm_fun = binary_overflow_fun "llvm.#{llvm_op}.with.overflow.i#{calc_width}", calc_type + res_with_overflow = builder.call(llvm_fun, [e1, e2]) + + result = extract_value res_with_overflow, 0 + overflow = extract_value res_with_overflow, 1 + + if calc_width > t1.bytes * 8 + result_trunc = trunc result, llvm_type(t1) + result_trunc_ext = t1.signed? ? builder.sext(result_trunc, calc_type) : builder.zext(result_trunc, calc_type) + overflow = or(overflow, builder.icmp LLVM::IntPredicate::NE, result, result_trunc_ext) + end + + codegen_raise_overflow_cond overflow + + trunc result, llvm_type(t1) + end + + def codegen_mul_unsigned_signed_with_overflow(t1, t2, p1, p2) + overflow = and( + codegen_binary_op_ne(t1, t1, p1, int(0, t1)), # self != 0 + codegen_binary_op_lt(t2, t2, p2, int(0, t2)) # other < 0 + ) + codegen_raise_overflow_cond overflow + + return codegen_binary_op_with_overflow("*", t1, @program.int_type(false, t2.bytes), p1, p2) + end + + def codegen_mul_signed_unsigned_with_overflow(t1, t2, p1, p2) + negative = codegen_binary_op_lt(t1, t1, p1, int(0, t1)) # self < 0 + minus_p1 = builder.sub int(0, t1), p1 + abs = builder.select negative, minus_p1, p1 + u1 = @program.int_type(false, t1.bytes) + + # tmp is the abs value of the result + # there is overflow when |result| > max + (negative ? 1 : 0) + tmp = codegen_binary_op_with_overflow("*", u1, t2, abs, p2) + _, max = t1.range + max_result = builder.add(int(max, t1), builder.zext(negative, llvm_type(t1))) + overflow = codegen_binary_op_gt(u1, u1, tmp, max_result) + codegen_raise_overflow_cond overflow + + # negate back the result if p1 was negative + minus_tmp = builder.sub int(0, t1), tmp + builder.select negative, minus_tmp, tmp + end + def codegen_binary_extend_int(t1, t2, p1, p2) if t1.normal_rank == t2.normal_rank # Nothing to do @@ -176,138 +252,6 @@ class Crystal::CodeGenVisitor end end - def codegen_binary_op_add(t : IntegerType, t1, t2, p1, p2) - llvm_fun = case t.kind - when :i8 - binary_overflow_fun "llvm.sadd.with.overflow.i8", llvm_context.int8 - when :i16 - binary_overflow_fun "llvm.sadd.with.overflow.i16", llvm_context.int16 - when :i32 - binary_overflow_fun "llvm.sadd.with.overflow.i32", llvm_context.int32 - when :i64 - binary_overflow_fun "llvm.sadd.with.overflow.i64", llvm_context.int64 - when :i128 - binary_overflow_fun "llvm.sadd.with.overflow.i128", llvm_context.int128 - when :u8 - binary_overflow_fun "llvm.uadd.with.overflow.i8", llvm_context.int8 - when :u16 - binary_overflow_fun "llvm.uadd.with.overflow.i16", llvm_context.int16 - when :u32 - binary_overflow_fun "llvm.uadd.with.overflow.i32", llvm_context.int32 - when :u64 - binary_overflow_fun "llvm.uadd.with.overflow.i64", llvm_context.int64 - when :u128 - binary_overflow_fun "llvm.uadd.with.overflow.i128", llvm_context.int128 - else - raise "unreachable" - end - - codegen_binary_overflow_check(llvm_fun, t, t1, t2, p1, p2) - end - - def codegen_binary_op_sub(t : IntegerType, t1, t2, p1, p2) - llvm_fun = case t.kind - when :i8 - binary_overflow_fun "llvm.ssub.with.overflow.i8", llvm_context.int8 - when :i16 - binary_overflow_fun "llvm.ssub.with.overflow.i16", llvm_context.int16 - when :i32 - binary_overflow_fun "llvm.ssub.with.overflow.i32", llvm_context.int32 - when :i64 - binary_overflow_fun "llvm.ssub.with.overflow.i64", llvm_context.int64 - when :i128 - binary_overflow_fun "llvm.ssub.with.overflow.i128", llvm_context.int128 - when :u8 - binary_overflow_fun "llvm.usub.with.overflow.i8", llvm_context.int8 - when :u16 - binary_overflow_fun "llvm.usub.with.overflow.i16", llvm_context.int16 - when :u32 - binary_overflow_fun "llvm.usub.with.overflow.i32", llvm_context.int32 - when :u64 - binary_overflow_fun "llvm.usub.with.overflow.i64", llvm_context.int64 - when :u128 - binary_overflow_fun "llvm.usub.with.overflow.i128", llvm_context.int128 - else - raise "unreachable" - end - - codegen_binary_overflow_check(llvm_fun, t, t1, t2, p1, p2) - end - - def codegen_binary_op_mul(t : IntegerType, t1, t2, p1, p2) - llvm_fun = case t.kind - when :i8 - binary_overflow_fun "llvm.smul.with.overflow.i8", llvm_context.int8 - when :i16 - binary_overflow_fun "llvm.smul.with.overflow.i16", llvm_context.int16 - when :i32 - binary_overflow_fun "llvm.smul.with.overflow.i32", llvm_context.int32 - when :i64 - binary_overflow_fun "llvm.smul.with.overflow.i64", llvm_context.int64 - when :i128 - binary_overflow_fun "llvm.smul.with.overflow.i128", llvm_context.int128 - when :u8 - binary_overflow_fun "llvm.umul.with.overflow.i8", llvm_context.int8 - when :u16 - binary_overflow_fun "llvm.umul.with.overflow.i16", llvm_context.int16 - when :u32 - binary_overflow_fun "llvm.umul.with.overflow.i32", llvm_context.int32 - when :u64 - binary_overflow_fun "llvm.umul.with.overflow.i64", llvm_context.int64 - when :u128 - binary_overflow_fun "llvm.umul.with.overflow.i128", llvm_context.int128 - else - raise "unreachable" - end - - codegen_binary_overflow_check(llvm_fun, t, t1, t2, p1, p2) - end - - # Generates a call to llvm_fun(p1, p2). - # t1, t2 are the original types of p1, p2. - # t is the super type of t1 and t2 where the operation is performed. - # llvm_fun returns {res, o_bit} where the o_bit signals overflow. - # The generated code also performs a range check and truncation of res - # in order to fit in the original type t1 if needed. - # - # ``` - # %res_with_overflow = call {T, i1} (T %p1, T %p2) - # %res = extractvalue {T, i1} %res, 0 - # %o_bit = extractvalue {T, i1} %res, 1 - # ;; if T != T1 - # %out_of_range = %res < T1::MIN || %res > T1::MAX ;; compare T1.range and %res - # br i1 or(%o_bit, %out_of_range), label %overflow, label %normal - # ;; else - # br i1 %o_bit, label %overflow, label %normal - # ;; end - # - # overflow: - # ;; codegen: raise OverflowError.new with caller's location - # - # normal: - # ;; if T != T1 - # ;; %res' is returned - # %res' = trunc T %res to T1 - # ;; else - # ;; %res is returned - # ;; end - # ``` - private def codegen_binary_overflow_check(llvm_fun, t : IntegerType, t1, t2, p1, p2) - res_with_overflow = builder.call(llvm_fun, [p1, p2]) - - res = extract_value res_with_overflow, 0 - o_bit = extract_value res_with_overflow, 1 - - if t != t1 - overflow = or(o_bit, codegen_out_of_range(t1, t, res)) - else - overflow = o_bit - end - - codegen_raise_overflow_cond overflow - codegen_trunc_binary_op_result(t1, t2, res) - end - private def codegen_out_of_range(target_type : IntegerType, arg_type : IntegerType, arg) min_value, max_value = target_type.range # arg < min_value || arg > max_value diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 8325202e8a58..294014758d50 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -486,6 +486,30 @@ module Crystal end end + def int_type(signed, size) + if signed + case size + when 1 then int8 + when 2 then int16 + when 4 then int32 + when 8 then int64 + when 16 then int128 + else + raise "BUG: Invalid int size: #{size}" + end + else + case size + when 1 then uint8 + when 2 then uint16 + when 4 then uint32 + when 8 then uint64 + when 16 then uint128 + else + raise "BUG: Invalid int size: #{size}" + end + end + end + # Returns the `IntegerType` that matches the given Int value def int?(int) case int From f3249054ee0ed55e782b8f49682ebb07de75bb31 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 4 Jun 2020 19:06:52 -0300 Subject: [PATCH 103/263] Tidy up Makefile and crystal env output (#9423) * Return default lib path when CRYSTAL_CONFIG_PATH is empty This will allow a clean CRYSTAL_PATH on wrapper scripts * Makefile: remove CRYSTAL_CONFIG_PATH in release build The wrapper or package will be responsible of setting it to the target location of the std-lib. In this repo since the compiler is used via bin/crystal wrapper script the CRYSTAL_PATH will be set there. * Makefile: remove unused $(BUILD_PATH) It's a leftover between 3ce407a83d583706e8435aaf90b46805e65876a0 and 027556fa5d9543374c3d0b0920535ece4f0c1a63 * Makefile: Allow CRYSTAL_CONFIG_LIBRARY_PATH So package builder can define it to target location without a wrapper script, or leave it really empty. * Extract current CRYSTAL_LIBRARY_PATH from bin/crystal and failsafe * Set CRYSTAL_CONFIG_LIBRARY_PATH when building specs and on wrapper script The changes in the wrapper script are needed to allow running individual compiler specs easily. $ bin/crystal spec spec/compiler/codegen/primitives_spec.cr --- Makefile | 14 +++++++------- bin/crystal | 4 ++++ src/compiler/crystal/crystal_path.cr | 4 +++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index c2be15297353..a764d6dba47f 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,11 @@ SPEC_SOURCES := $(shell find spec -name '*.cr') override FLAGS += $(if $(release),--release )$(if $(stats),--stats )$(if $(progress),--progress )$(if $(threads),--threads $(threads) )$(if $(debug),-d )$(if $(static),--static )$(if $(LDFLAGS),--link-flags="$(LDFLAGS)" ) SPEC_WARNINGS_OFF := --exclude-warnings spec/std --exclude-warnings spec/compiler SPEC_FLAGS := $(if $(verbose),-v )$(if $(junit_output),--junit_output $(junit_output) ) +CRYSTAL_CONFIG_LIBRARY_PATH := $(shell bin/crystal env CRYSTAL_LIBRARY_PATH 2> /dev/null) CRYSTAL_CONFIG_BUILD_COMMIT := $(shell git rev-parse --short HEAD 2> /dev/null) SOURCE_DATE_EPOCH := $(shell (git show -s --format=%ct HEAD || stat -c "%Y" Makefile || stat -f "%m" Makefile) 2> /dev/null) EXPORTS := \ - $(if $(release),,CRYSTAL_CONFIG_PATH="$(PWD)/src") \ - CRYSTAL_CONFIG_LIBRARY_PATH="$(shell crystal env CRYSTAL_LIBRARY_PATH)" \ + CRYSTAL_CONFIG_LIBRARY_PATH="$(CRYSTAL_CONFIG_LIBRARY_PATH)" \ CRYSTAL_CONFIG_BUILD_COMMIT="$(CRYSTAL_CONFIG_BUILD_COMMIT)" \ SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" SHELL = sh @@ -93,7 +93,7 @@ compiler_spec: $(O)/compiler_spec ## Run compiler specs .PHONY: docs docs: ## Generate standard library documentation - $(BUILD_PATH) ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS) --project-name=Crystal --project-version=$(CRYSTAL_VERSION) --source-refname=$(CRYSTAL_CONFIG_BUILD_COMMIT) + ./bin/crystal docs src/docs_main.cr $(DOCS_OPTIONS) --project-name=Crystal --project-version=$(CRYSTAL_VERSION) --source-refname=$(CRYSTAL_CONFIG_BUILD_COMMIT) .PHONY: crystal crystal: $(O)/crystal ## Build the compiler @@ -106,19 +106,19 @@ libcrystal: $(LIB_CRYSTAL_TARGET) $(O)/all_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) - $(EXPORT_CC) $(BUILD_PATH) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/all_spec.cr + $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/all_spec.cr $(O)/std_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) - $(EXPORT_CC) $(BUILD_PATH) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/std_spec.cr + $(EXPORT_CC) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/std_spec.cr $(O)/compiler_spec: $(DEPS) $(SOURCES) $(SPEC_SOURCES) @mkdir -p $(O) - $(EXPORT_CC) $(BUILD_PATH) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr + $(EXPORT_CC) $(EXPORTS) ./bin/crystal build $(FLAGS) $(SPEC_WARNINGS_OFF) -o $@ spec/compiler_spec.cr $(O)/crystal: $(DEPS) $(SOURCES) @mkdir -p $(O) - $(BUILD_PATH) $(EXPORTS) ./bin/crystal build $(FLAGS) -o $@ src/compiler/crystal.cr -D without_openssl -D without_zlib + $(EXPORTS) ./bin/crystal build $(FLAGS) -o $@ src/compiler/crystal.cr -D without_openssl -D without_zlib $(LLVM_EXT_OBJ): $(LLVM_EXT_DIR)/llvm_ext.cc $(CXX) -c $(CXXFLAGS) -o $@ $< $(shell $(LLVM_CONFIG) --cxxflags) diff --git a/bin/crystal b/bin/crystal index a16aa21013e9..65f39e07643f 100755 --- a/bin/crystal +++ b/bin/crystal @@ -141,6 +141,10 @@ CRYSTAL_DIR="$CRYSTAL_ROOT/.build" export CRYSTAL_PATH=lib:$CRYSTAL_ROOT/src export CRYSTAL_HAS_WRAPPER=true +if [ -z "$CRYSTAL_CONFIG_LIBRARY_PATH" ]; then + export CRYSTAL_CONFIG_LIBRARY_PATH="$(crystal env CRYSTAL_LIBRARY_PATH || echo "")" +fi + if [ -x "$CRYSTAL_DIR/crystal" ]; then __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/crystal" exec "$CRYSTAL_DIR/crystal" "$@" diff --git a/src/compiler/crystal/crystal_path.cr b/src/compiler/crystal/crystal_path.cr index 828e04a9b312..9b2bdd982df7 100644 --- a/src/compiler/crystal/crystal_path.cr +++ b/src/compiler/crystal/crystal_path.cr @@ -15,7 +15,9 @@ module Crystal def self.default_path ENV["CRYSTAL_PATH"]? || begin - if Crystal::Config.path.split(Process::PATH_DELIMITER).includes?(DEFAULT_LIB_PATH) + if Crystal::Config.path.blank? + DEFAULT_LIB_PATH + elsif Crystal::Config.path.split(Process::PATH_DELIMITER).includes?(DEFAULT_LIB_PATH) Crystal::Config.path else {DEFAULT_LIB_PATH, Crystal::Config.path}.join(Process::PATH_DELIMITER) From 82ac201f6c740076b844018fcdad159445afe711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Fri, 5 Jun 2020 13:46:41 +0200 Subject: [PATCH 104/263] Remove redundant spec relying on crystal being in PATH (#9402) --- spec/std/process_spec.cr | 3 --- 1 file changed, 3 deletions(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index ab89a43b9314..22934b119975 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -419,9 +419,6 @@ describe Process do (path = Process.find_executable("ls")).should_not be_nil path.not_nil!.should match(/#{File::SEPARATOR}ls$/) - (path = Process.find_executable("crystal")).should_not be_nil - path.not_nil!.should match(/#{File::SEPARATOR}crystal$/) - Process.find_executable("some_very_unlikely_file_to_exist").should be_nil end end From 8a6988e552f3f673c9dbe48015dde2482d2bf439 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Fri, 5 Jun 2020 09:37:46 -0300 Subject: [PATCH 105/263] Ignore response body during WebSocket handshake (#9418) --- spec/std/http/web_socket_spec.cr | 33 ++++++++++++++++++++++++++++++++ src/http/web_socket/protocol.cr | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/spec/std/http/web_socket_spec.cr b/spec/std/http/web_socket_spec.cr index 9d940bb9d4ea..ad3b35c2e167 100644 --- a/spec/std/http/web_socket_spec.cr +++ b/spec/std/http/web_socket_spec.cr @@ -28,6 +28,15 @@ private def assert_packet(packet, opcode, size, final = false) packet.final.should eq(final) end +private class MalformerHandler + include HTTP::Handler + + def call(context) + context.response.headers["Transfer-Encoding"] = "chunked" + call_next(context) + end +end + describe HTTP::WebSocket do describe "receive" do it "can read a small text packet" do @@ -428,6 +437,30 @@ describe HTTP::WebSocket do end end + it "ignores body in upgrade response (malformed)" do + malformer = MalformerHandler.new + ws_handler = HTTP::WebSocketHandler.new do |ws, ctx| + ws.on_message do |str| + ws.send(str) + end + end + http_server = HTTP::Server.new([malformer, ws_handler]) + + address = http_server.bind_unused_port + + run_server(http_server) do + client = HTTP::WebSocket.new("ws://#{address}") + message = nil + client.on_message do |msg| + message = msg + client.close + end + client.send "hello" + client.run + message.should eq("hello") + end + end + describe "handshake fails if server does not verify Sec-WebSocket-Key" do it "Sec-WebSocket-Accept missing" do http_server = HTTP::Server.new do |context| diff --git a/src/http/web_socket/protocol.cr b/src/http/web_socket/protocol.cr index 08e6c2736058..9f7a1f300c59 100644 --- a/src/http/web_socket/protocol.cr +++ b/src/http/web_socket/protocol.cr @@ -298,7 +298,7 @@ class HTTP::WebSocket::Protocol handshake.to_io(socket) socket.flush - handshake_response = HTTP::Client::Response.from_io(socket) + handshake_response = HTTP::Client::Response.from_io(socket, ignore_body: true) unless handshake_response.status.switching_protocols? raise Socket::Error.new("Handshake got denied. Status code was #{handshake_response.status.code}.") end From b8ac9e17e8d0264b0d5c518601c070ce8eccd978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Fri, 5 Jun 2020 14:38:41 +0200 Subject: [PATCH 106/263] specify pkg_config name for libyaml (#9426) --- src/yaml/lib_yaml.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yaml/lib_yaml.cr b/src/yaml/lib_yaml.cr index f766ff6f1dd5..9faa3abf339e 100644 --- a/src/yaml/lib_yaml.cr +++ b/src/yaml/lib_yaml.cr @@ -1,6 +1,10 @@ require "./enums" -@[Link("yaml")] +{% if compare_versions(Crystal::VERSION, "0.35.0-0") >= 0 %} + @[Link("yaml", pkg_config: "yaml-0.1")] +{% else %} + @[Link("yaml")] +{% end %} lib LibYAML alias Int = LibC::Int From 3b268f0f87026059028aab6194e5a09dade581b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 5 Jun 2020 14:39:19 +0200 Subject: [PATCH 107/263] Add support for Path pattern to Dir.glob (#9420) --- spec/std/dir_spec.cr | 18 ++++++++++++++++++ src/dir/glob.cr | 17 ++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 4bf7067f527f..282c68f28e31 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -368,6 +368,24 @@ describe "Dir" do Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: false).size.should eq 0 end end + + context "with path" do + expected = [ + datapath("dir", "f1.txt"), + datapath("dir", "f2.txt"), + datapath("dir", "g2.txt"), + ] + + it "posix path" do + Dir[Path.posix(datapath, "dir", "*.txt")].sort.should eq expected + Dir[[Path.posix(datapath, "dir", "*.txt")]].sort.should eq expected + end + + it "windows path" do + Dir[Path.windows(datapath, "dir", "*.txt")].sort.should eq expected + Dir[[Path.windows(datapath, "dir", "*.txt")]].sort.should eq expected + end + end end describe "cd" do diff --git a/src/dir/glob.cr b/src/dir/glob.cr index b6d86952bb99..4c006320806d 100644 --- a/src/dir/glob.cr +++ b/src/dir/glob.cr @@ -4,12 +4,12 @@ class Dir # The pattern syntax is similar to shell filename globbing, see `File.match?` for details. # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.[](*patterns) : Array(String) + def self.[](*patterns : Path | String) : Array(String) glob(patterns) end # :ditto: - def self.[](patterns : Enumerable(String)) : Array(String) + def self.[](patterns : Enumerable) : Array(String) glob(patterns) end @@ -20,12 +20,12 @@ class Dir # If *match_hidden* is `true` the pattern will match hidden files and folders. # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.glob(*patterns, match_hidden = false) : Array(String) + def self.glob(*patterns : Path | String, match_hidden = false) : Array(String) glob(patterns, match_hidden: match_hidden) end # :ditto: - def self.glob(patterns : Enumerable(String), match_hidden = false) : Array(String) + def self.glob(patterns : Enumerable, match_hidden = false) : Array(String) paths = [] of String glob(patterns, match_hidden: match_hidden) do |path| paths << path @@ -40,14 +40,14 @@ class Dir # If *match_hidden* is `true` the pattern will match hidden files and folders. # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.glob(*patterns, match_hidden = false, &block : String -> _) + def self.glob(*patterns : Path | String, match_hidden = false, &block : String -> _) glob(patterns, match_hidden: match_hidden) do |path| yield path end end # :ditto: - def self.glob(patterns : Enumerable(String), match_hidden = false, &block : String -> _) + def self.glob(patterns : Enumerable, match_hidden = false, &block : String -> _) Globber.glob(patterns, match_hidden: match_hidden) do |path| yield path end @@ -72,8 +72,11 @@ class Dir end alias PatternType = DirectoriesOnly | ConstantEntry | EntryMatch | RecursiveDirectories | ConstantDirectory | RootDirectory | DirectoryMatch - def self.glob(patterns : Enumerable(String), **options, &block : String -> _) + def self.glob(patterns : Enumerable, **options, &block : String -> _) patterns.each do |pattern| + if pattern.is_a?(Path) + pattern = pattern.to_posix.to_s + end sequences = compile(pattern) sequences.each do |sequence| From ab75612778818ada5b42ced019d47374276d190b Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 5 Jun 2020 09:58:06 -0300 Subject: [PATCH 108/263] Compiler: fix `Program#nilable?` to consider union types (#9417) --- spec/compiler/codegen/nilable_cast_spec.cr | 22 ++++++++++++++++++++++ src/compiler/crystal/program.cr | 16 ++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/nilable_cast_spec.cr b/spec/compiler/codegen/nilable_cast_spec.cr index f9b5182efcea..c30ff6fd5c7e 100644 --- a/spec/compiler/codegen/nilable_cast_spec.cr +++ b/spec/compiler/codegen/nilable_cast_spec.cr @@ -147,4 +147,26 @@ describe "Code gen: nilable cast" do x.try &.as?(Foo) )) end + + it "casts union type to nilable type (#9342)" do + run(%( + struct Nil + def foo + 0 + end + end + + class Gen(T) + def initialize(@value : Int32) + end + + def foo + @value + end + end + + a = Gen(String).new(10) || Gen(Int32).new(20) + a.as?(Gen).foo + )).to_i.should eq(10) + end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 294014758d50..7d74964a9873 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -326,10 +326,18 @@ module Crystal # Returns the `Type` for `type | Nil` def nilable(type) - # Nil | Nil # => Nil - return self.nil if type == self.nil - - union_of self.nil, type + case type + when self.nil + # Nil | Nil # => Nil + return self.nil + when UnionType + types = Array(Type).new(type.union_types.size + 1) + types.concat type.union_types + types << self.nil + Type.merge(types) + else + union_of self.nil, type + end end # Returns the `Type` for `type1 | type2` From 18758d58b064ebc3fc049d55fcbcfd2dc4b6a505 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Fri, 5 Jun 2020 22:00:00 +0900 Subject: [PATCH 109/263] Use `Process.quote` for `crystal env` output (#9428) Then we can use `eval "$(crystal env)"` in shell script safely. --- src/compiler/crystal/command/env.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/command/env.cr b/src/compiler/crystal/command/env.cr index 5b6c20bc1999..5e41e0fb0365 100644 --- a/src/compiler/crystal/command/env.cr +++ b/src/compiler/crystal/command/env.cr @@ -16,7 +16,7 @@ class Crystal::Command if ARGV.empty? vars.each do |key, value| - puts "#{key}=#{value.inspect}" + puts "#{key}=#{Process.quote(value)}" end else ARGV.each do |key| From c4b87dc4acec45cb9f4fcd08daff797ad2e9d373 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 5 Jun 2020 10:00:19 -0300 Subject: [PATCH 110/263] Allow executing OpAssign (+=, ||=, etc.) inside macros (#9409) --- spec/compiler/semantic/macro_spec.cr | 14 ++++++++++++++ src/compiler/crystal/macros/interpreter.cr | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/spec/compiler/semantic/macro_spec.cr b/spec/compiler/semantic/macro_spec.cr index aba8a8aff2dd..90bdd6497fc0 100644 --- a/spec/compiler/semantic/macro_spec.cr +++ b/spec/compiler/semantic/macro_spec.cr @@ -1463,4 +1463,18 @@ describe "Semantic: macro" do method = result.program.types["Foo"].lookup_first_def("bar", false).not_nil! method.location.not_nil!.expanded_location.not_nil!.line_number.should eq(10) end + + it "executes OpAssign (#9356)" do + assert_type(%( + {% begin %} + {% a = nil %} + {% a ||= 1 %} + {% if a %} + 1 + {% else %} + 'a' + {% end %} + {% end %} + )) { int32 } + end end diff --git a/src/compiler/crystal/macros/interpreter.cr b/src/compiler/crystal/macros/interpreter.cr index 1c804a301f36..db5bcceff172 100644 --- a/src/compiler/crystal/macros/interpreter.cr +++ b/src/compiler/crystal/macros/interpreter.cr @@ -318,6 +318,11 @@ module Crystal false end + def visit(node : OpAssign) + @program.normalize(node).accept(self) + false + end + def visit(node : And) node.left.accept self if @last.truthy? From 0d4130e7dd7876c2c2fc83bcbed0c355df8b200c Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Mon, 8 Jun 2020 11:43:19 -0400 Subject: [PATCH 111/263] Specify pkgconfig name for libxml2 (#9436) --- src/xml/libxml2.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/xml/libxml2.cr b/src/xml/libxml2.cr index c348eec4a151..63657b2ae000 100644 --- a/src/xml/libxml2.cr +++ b/src/xml/libxml2.cr @@ -4,7 +4,11 @@ require "./parser_options" require "./html_parser_options" require "./save_options" -@[Link("xml2")] +{% if compare_versions(Crystal::VERSION, "0.35.0-0") >= 0 %} + @[Link("xml2", pkg_config: "libxml-2.0")] +{% else %} + @[Link("xml2")] +{% end %} lib LibXML alias Int = LibC::Int From c5b34ae0391d22662e8ea1c8101459fda244e9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 8 Jun 2020 17:44:02 +0200 Subject: [PATCH 112/263] Make YAML specs robust against libyaml 0.2.5 (#9427) --- spec/std/yaml/serializable_spec.cr | 4 ++-- spec/std/yaml/serialization_spec.cr | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/std/yaml/serializable_spec.cr b/spec/std/yaml/serializable_spec.cr index 74001ea667ba..701c2a939ccf 100644 --- a/spec/std/yaml/serializable_spec.cr +++ b/spec/std/yaml/serializable_spec.cr @@ -497,7 +497,7 @@ describe "YAML::Serializable" do it "emit_nulls option" do person = YAMLAttrPersonEmittingNullsByOptions.from_yaml("---\nname: John\n") - person.to_yaml.should eq "---\nname: John\nage: \nvalue1: \n" + person.to_yaml.should match /\A---\nname: John\nage: ?\nvalue1: ?\n\z/ end it "parses yaml with Time::Format converter" do @@ -528,7 +528,7 @@ describe "YAML::Serializable" do it "outputs with converter when nilable when emit_null is true" do yaml = YAMLAttrWithNilableTimeEmittingNull.new - yaml.to_yaml.should eq("---\nvalue: \n") + yaml.to_yaml.should match(/\A---\nvalue: ?\n\z/) end it "outputs YAML with properties key" do diff --git a/spec/std/yaml/serialization_spec.cr b/spec/std/yaml/serialization_spec.cr index 9fd542308f60..85dec4579c20 100644 --- a/spec/std/yaml/serialization_spec.cr +++ b/spec/std/yaml/serialization_spec.cr @@ -386,9 +386,9 @@ describe "YAML serialization" do :null => nil, } - expected = "---\nhello: World\ninteger: 2\nfloat: 3.5\nhash:\n a: 1\n b: 2\narray:\n- 1\n- 2\n- 3\nnull: \n" + expected = /\A---\nhello: World\ninteger: 2\nfloat: 3.5\nhash:\n a: 1\n b: 2\narray:\n- 1\n- 2\n- 3\nnull: ?\n\z/ - data.to_yaml.should eq(expected) + data.to_yaml.should match(expected) end it "writes to a stream" do @@ -409,7 +409,7 @@ describe "YAML serialization" do h[1] = 2 h[h] = h - h.to_yaml.should eq("--- &1\n1: 2\n*1: *1\n") + h.to_yaml.should match(/\A--- &1\n1: 2\n\*1 ?: \*1\n\z/) end end end From c01184e38bdc85a1d7208a52eec333dc103b2eec Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 8 Jun 2020 12:44:18 -0300 Subject: [PATCH 113/263] Compiler: don't duplicate instance var in inherited generic type (#9433) --- spec/compiler/codegen/generic_class_spec.cr | 18 ++++++++++++++++++ .../semantic/type_declaration_processor.cr | 10 +++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/spec/compiler/codegen/generic_class_spec.cr b/spec/compiler/codegen/generic_class_spec.cr index 27172c71f40d..682eca0b5a0f 100644 --- a/spec/compiler/codegen/generic_class_spec.cr +++ b/spec/compiler/codegen/generic_class_spec.cr @@ -401,4 +401,22 @@ describe "Code gen: generic class type" do end )) end + + it "doesn't override guessed instance var in generic type if already declared in superclass (#9431)" do + codegen(%( + class Foo + @x = 0 + end + + class Bar(T) < Foo + @x = 0 + end + + class Baz < Bar(Int32) + @valid = true + end + + Baz.new + )) + end end diff --git a/src/compiler/crystal/semantic/type_declaration_processor.cr b/src/compiler/crystal/semantic/type_declaration_processor.cr index c2fddca266c0..f1a5461bbf05 100644 --- a/src/compiler/crystal/semantic/type_declaration_processor.cr +++ b/src/compiler/crystal/semantic/type_declaration_processor.cr @@ -339,13 +339,13 @@ struct Crystal::TypeDeclarationProcessor # set from uninstantiated generic types return if owner.is_a?(GenericInstanceType) + # If a superclass already defines this variable we ignore + # the guessed type information for subclasses + supervar = owner.lookup_instance_var?(name) + return if supervar + case owner when NonGenericClassType - # If a superclass already defines this variable we ignore - # the guessed type information for subclasses - supervar = owner.lookup_instance_var?(name) - return if supervar - type = type_info.type if nilable_instance_var?(owner, name) type = Type.merge!(type, @program.nil) From 1af66b9344e12b13a0f9309b3d67059b3d74b0d4 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 8 Jun 2020 12:45:09 -0300 Subject: [PATCH 114/263] Compiler: preserve all union types in `as?` (#9435) --- spec/compiler/semantic/nilable_cast_spec.cr | 21 +++++++++++++++++++++ src/compiler/crystal/program.cr | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/spec/compiler/semantic/nilable_cast_spec.cr b/spec/compiler/semantic/nilable_cast_spec.cr index c2ddac878cd0..7382bf42c43c 100644 --- a/spec/compiler/semantic/nilable_cast_spec.cr +++ b/spec/compiler/semantic/nilable_cast_spec.cr @@ -47,4 +47,25 @@ describe "Semantic: nilable cast" do end )) { string } end + + it "casts to module" do + assert_type(%( + module Moo + end + + class Base + end + + class Foo < Base + include Moo + end + + class Bar < Base + include Moo + end + + base = (Foo.new || Bar.new) + base.as?(Moo) + )) { union_of([types["Foo"], types["Bar"], nil_type] of Type) } + end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 7d74964a9873..16783bc7b2c7 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -333,8 +333,8 @@ module Crystal when UnionType types = Array(Type).new(type.union_types.size + 1) types.concat type.union_types - types << self.nil - Type.merge(types) + types << self.nil unless types.includes? self.nil + union_of types else union_of self.nil, type end From 9057e8d8074bf6dd8fb5fc09f96264201e0abd61 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 8 Jun 2020 15:53:28 -0300 Subject: [PATCH 115/263] Make IOBackend formatter a named argument (#9434) --- src/log/io_backend.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log/io_backend.cr b/src/log/io_backend.cr index ac9189db1b44..e9265c503f75 100644 --- a/src/log/io_backend.cr +++ b/src/log/io_backend.cr @@ -3,7 +3,7 @@ class Log::IOBackend < Log::Backend property io : IO property formatter : Formatter - def initialize(@io = STDOUT, @formatter : Formatter = ShortFormat) + def initialize(@io = STDOUT, *, @formatter : Formatter = ShortFormat) @mutex = Mutex.new(:unchecked) end From ce1b36cdce7d96849961046ca9af2a0558d379f1 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 9 Jun 2020 10:38:54 -0300 Subject: [PATCH 116/263] Update distribution-scripts (#9446) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3067787c916..98b5dc397d62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - run: | git clone https://github.com/crystal-lang/distribution-scripts.git ~/distribution-scripts cd ~/distribution-scripts - git checkout 1f2d5d937d431ff86355a276fc259478b3df73ac + git checkout c8495fb1799c395b81923bd6d040fc8e75afbe7e # persist relevant information for build process - run: | cd ~/distribution-scripts From 3c48f311f98e95964d425abe23d2b353b7da07d1 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 9 Jun 2020 11:21:41 -0300 Subject: [PATCH 117/263] Release 0.35.0 (#9317) --- CHANGELOG.md | 245 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/VERSION | 2 +- 2 files changed, 246 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12023d944a5e..a70cbc9dcd64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,248 @@ +# 0.35.0 (2020-06-09) + +## Language changes + +- **(breaking-change)** Let `case when` be non-exhaustive, introduce `case in` as exhaustive. ([#9258](https://github.com/crystal-lang/crystal/pull/9258), [#9045](https://github.com/crystal-lang/crystal/pull/9045), thanks @asterite) +- Allow `->@ivar.foo` and `->@@cvar.foo` expressions. ([#9268](https://github.com/crystal-lang/crystal/pull/9268), thanks @MakeNowJust) + +### Macros + +- Allow executing OpAssign (`+=`, `||=`, etc.) inside macros. ([#9409](https://github.com/crystal-lang/crystal/pull/9409), thanks @asterite) + +## Standard library + +- **(breaking-change)** Refactor to standardize on first argument for methods receiving `IO`. ([#9134](https://github.com/crystal-lang/crystal/pull/9134), [#9289](https://github.com/crystal-lang/crystal/pull/9289), [#9303](https://github.com/crystal-lang/crystal/pull/9303), [#9318](https://github.com/crystal-lang/crystal/pull/9318), thanks @straight-shoota, @bcardiff, @oprypin) +- **(breaking-change)** Cleanup Digest and OpenSSL::Digest. ([#8426](https://github.com/crystal-lang/crystal/pull/8426), thanks @didactic-drunk) +- Fix `Enum#to_s` for private enum. ([#9126](https://github.com/crystal-lang/crystal/pull/9126), thanks @straight-shoota) +- Refactor `Benchmark::IPS::Entry` to use `UInt64` in `bytes_per_op`. ([#9081](https://github.com/crystal-lang/crystal/pull/9081), thanks @jhass) +- Add `Experimental` annotation and doc label. ([#9244](https://github.com/crystal-lang/crystal/pull/9244), thanks @bcardiff) +- Add subcommands to `OptionParser`. ([#9009](https://github.com/crystal-lang/crystal/pull/9009), [#9133](https://github.com/crystal-lang/crystal/pull/9133), thanks @RX14, @Sija) +- Make `NamedTuple#sorted_keys` public. ([#9263](https://github.com/crystal-lang/crystal/pull/9263), thanks @waj) +- Fix example codes in multiple places. ([#9203](https://github.com/crystal-lang/crystal/pull/9203), thanks @maiha) + +### Macros + +- **(breaking-change)** Remove top-level `assert_responds_to` macro. ([#9085](https://github.com/crystal-lang/crystal/pull/9085), thanks @bcardiff) +- **(breaking-change)** Drop top-level `parallel` macro. ([#9097](https://github.com/crystal-lang/crystal/pull/9097), thanks @bcardiff) +- Fix lazy property not forwarding annotations. ([#9140](https://github.com/crystal-lang/crystal/pull/9140), thanks @asterite) +- Add `host_flag?` macro method, not affected by cross-compilation. ([#9049](https://github.com/crystal-lang/crystal/pull/9049), thanks @oprypin) +- Add `.each` and `.each_with_index` to various macro types. ([#9120](https://github.com/crystal-lang/crystal/pull/9120), thanks @Blacksmoke16) +- Add `StringLiteral#titleize` macro method. ([#9269](https://github.com/crystal-lang/crystal/pull/9269), thanks @MakeNowJust) +- Add `TypeNode` methods to check what "type" the node is. ([#9270](https://github.com/crystal-lang/crystal/pull/9270), thanks @Blacksmoke16) +- Fix support `TypeNode.name(generic_args: false)` for generic instances. ([#9224](https://github.com/crystal-lang/crystal/pull/9224), thanks @Blacksmoke16) + +### Numeric + +- **(breaking-change)** Add `Int#digits`, reverse `BigInt#digits` result. ([#9383](https://github.com/crystal-lang/crystal/pull/9383), thanks @asterite) +- Fix overflow checking for operations with mixed sign. ([#9403](https://github.com/crystal-lang/crystal/pull/9403), thanks @waj) +- Add `BigInt#factorial` using GMP. ([#9132](https://github.com/crystal-lang/crystal/pull/9132), thanks @peheje) + +### Text + +- Add `String#titleize`. ([#9204](https://github.com/crystal-lang/crystal/pull/9204), thanks @hugopl) +- Add `Regex#matches?` and `String#matches?`. ([#8989](https://github.com/crystal-lang/crystal/pull/8989), thanks @MakeNowJust) +- Add `IO` overloads to various `String` case methods. ([#9236](https://github.com/crystal-lang/crystal/pull/9236), thanks @Blacksmoke16) +- Improve docs examples regarding `Regex::MatchData`. ([#9010](https://github.com/crystal-lang/crystal/pull/9010), thanks @MakeNowJust) +- Improve docs on `String` methods. ([#8447](https://github.com/crystal-lang/crystal/pull/8447), thanks @jan-zajic) + +### Collections + +- **(breaking-change)** Add `Enumerable#first` with fallback block. ([#8999](https://github.com/crystal-lang/crystal/pull/8999), thanks @MakeNowJust) +- Fix `Array#delete_at` bug with negative start index. ([#9399](https://github.com/crystal-lang/crystal/pull/9399), thanks @asterite) +- Fix `Enumerable#{zip,zip?}` when self is an `Iterator`. ([#9330](https://github.com/crystal-lang/crystal/pull/9330), thanks @mneumann) +- Make `Range#each` and `Range#reverse_each` work better with end/begin-less values. ([#9325](https://github.com/crystal-lang/crystal/pull/9325), thanks @asterite) +- Improve docs on `Hash`. ([#8887](https://github.com/crystal-lang/crystal/pull/8887), thanks @rdp) + +### Serialization + +- **(breaking-change)** Deprecate `JSON.mapping` and `YAML.mapping`. ([#9272](https://github.com/crystal-lang/crystal/pull/9272), thanks @straight-shoota) +- **(breaking-change)** Make `INI` a module. ([#9408](https://github.com/crystal-lang/crystal/pull/9408), thanks @j8r) +- Fix integration between `record` macro and `JSON::Serializable`/`YAML::Serializable` regarding default values. ([#9063](https://github.com/crystal-lang/crystal/pull/9063), thanks @Blacksmoke16) +- Fix `XML.parse` invalid mem access in multi-thread. ([#9098](https://github.com/crystal-lang/crystal/pull/9098), thanks @bcardiff, @asterite) +- Fix double string escape in `XML::Node#content=`. ([#9300](https://github.com/crystal-lang/crystal/pull/9300), thanks @straight-shoota) +- Improve xpath regarding namespaces. ([#9288](https://github.com/crystal-lang/crystal/pull/9288), thanks @asterite) +- Escape CDATA end sequences. ([#9230](https://github.com/crystal-lang/crystal/pull/9230), thanks @Blacksmoke16) +- Add `JSON` and `YAML` serialization to `Path`. ([#9156](https://github.com/crystal-lang/crystal/pull/9156), thanks @straight-shoota) +- Specify pkg-config name for `libyaml`. ([#9426](https://github.com/crystal-lang/crystal/pull/9426), thanks @jhass) +- Specify pkg-config name for `libxml2`. ([#9436](https://github.com/crystal-lang/crystal/pull/9436), thanks @Blacksmoke16) +- Make YAML specs robust against libyaml 0.2.5. ([#9427](https://github.com/crystal-lang/crystal/pull/9427), thanks @jhass) + +### Time + +- **(breaking-change)** Support different number of fraction digits for RFC3339 time format. ([#9283](https://github.com/crystal-lang/crystal/pull/9283), thanks @waj) +- Fix parsing AM/PM hours. ([#9334](https://github.com/crystal-lang/crystal/pull/9334), thanks @straight-shoota) +- Improve `File.utime` precision from second to 100-nanosecond on Windows. ([#9344](https://github.com/crystal-lang/crystal/pull/9344), thanks @kubo) + +### Files + +- **(breaking-change)** Move `Flate`, `Gzip`, `Zip`, `Zlib` to `Compress`. ([#8886](https://github.com/crystal-lang/crystal/pull/8886), thanks @bcardiff) +- **(breaking-change)** Cleanup `File` & `FileUtils`. ([#9175](https://github.com/crystal-lang/crystal/pull/9175), thanks @bcardiff) +- Fix realpath on macOS 10.15 (Catalina). ([#9296](https://github.com/crystal-lang/crystal/pull/9296), thanks @waj) +- Fix `File#pos`, `File#seek` and `File#truncate` over 2G on Windows. ([#9015](https://github.com/crystal-lang/crystal/pull/9015), thanks @kubo) +- Fix `File.rename` to overwrite the destination file on Windows, like elsewhere. ([#9038](https://github.com/crystal-lang/crystal/pull/9038), thanks @oprypin) +- Fix `File`'s specs and related exception types on Windows. ([#9037](https://github.com/crystal-lang/crystal/pull/9037), thanks @oprypin) +- Add support for `Path` arguments to multiple methods. ([#9153](https://github.com/crystal-lang/crystal/pull/9153), thanks @straight-shoota) +- Add `Path#each_part` iterator. ([#9138](https://github.com/crystal-lang/crystal/pull/9138), thanks @straight-shoota) +- Add `Path#relative_to`. ([#9169](https://github.com/crystal-lang/crystal/pull/9169), thanks @straight-shoota) +- Add support for `Path` pattern to `Dir.glob`. ([#9420](https://github.com/crystal-lang/crystal/pull/9420), thanks @straight-shoota) +- Implement `File#fsync` on Windows. ([#9257](https://github.com/crystal-lang/crystal/pull/9257), thanks @kubo) +- Refactor `Path` regarding empty and `.`. ([#9137](https://github.com/crystal-lang/crystal/pull/9137), thanks @straight-shoota) + +### Networking + +- **(breaking-change)** Make `IO#skip`, `IO#write` returns the number of bytes it skipped/written as `Int64`. ([#9233](https://github.com/crystal-lang/crystal/pull/9233), [#9363](https://github.com/crystal-lang/crystal/pull/9363), thanks @bcardiff) +- **(breaking-change)** Improve error handling and logging in `HTTP::Server`. ([#9115](https://github.com/crystal-lang/crystal/pull/9115), [#9034](https://github.com/crystal-lang/crystal/pull/9034), thanks @waj, @straight-shoota) +- **(breaking-change)** Change `HTTP::Request#remote_address` type to `Socket::Address?`. ([#9210](https://github.com/crystal-lang/crystal/pull/9210), thanks @waj) +- Fix `flush` methods to always flush underlying `IO`. ([#9320](https://github.com/crystal-lang/crystal/pull/9320), thanks @straight-shoota) +- Fix `HTTP::Server` sporadic failure in SSL handshake. ([#9177](https://github.com/crystal-lang/crystal/pull/9177), thanks @waj) +- `WebSocket` shouldn't reply with same close code. ([#9313](https://github.com/crystal-lang/crystal/pull/9313), thanks @waj) +- Ignore response body during `WebSocket` handshake. ([#9418](https://github.com/crystal-lang/crystal/pull/9418), thanks @waj) +- Treat cookies which expire in this instant as expired. ([#9061](https://github.com/crystal-lang/crystal/pull/9061), thanks @RX14) +- Set `sync` or `flush_on_newline` for standard I/O on Windows. ([#9207](https://github.com/crystal-lang/crystal/pull/9207), thanks @kubo) +- Prefer HTTP basic authentication in OAuth2 client. ([#9127](https://github.com/crystal-lang/crystal/pull/9127), thanks @crush-157) +- Defer request upgrade in `HTTP::Server` (aka: WebSockets). ([#9243](https://github.com/crystal-lang/crystal/pull/9243), thanks @waj) +- Improve `URI::Punycode`, `HTTP::WebSocketHandler`, `HTTP::Status` documentation. ([#9068](https://github.com/crystal-lang/crystal/pull/9068), [#9130](https://github.com/crystal-lang/crystal/pull/9130), [#9180](https://github.com/crystal-lang/crystal/pull/9180), thanks @Blacksmoke16, @dscottboggs, @wontruefree) +- Remove `HTTP::Params::Builder#to_s`, use underlying `IO` directly. ([#9319](https://github.com/crystal-lang/crystal/pull/9319), thanks @straight-shoota) +- Fixed some regular failing specs in multi-thread mode. ([#9412](https://github.com/crystal-lang/crystal/pull/9412), thanks @bcardiff) + +### Crypto + +- **(security)** Update SSL server secure defaults. ([#9026](https://github.com/crystal-lang/crystal/pull/9026), thanks @straight-shoota) +- Add LibSSL `NO_TLS_V1_3` option. ([#9350](https://github.com/crystal-lang/crystal/pull/9350), thanks @lun-4) + +### Logging + +- **(breaking-change)** Rename `Log::Severity::Warning` to `Warn`. Drop `Verbose`. Add `Trace` and `Notice`. ([#9293](https://github.com/crystal-lang/crystal/pull/9293), [#9107](https://github.com/crystal-lang/crystal/pull/9107), [#9316](https://github.com/crystal-lang/crystal/pull/9316), thanks @bcardiff, @paulcsmith) +- **(breaking-change)** Allow local data on entries via `Log::Metadata` and redesign `Log::Context`. ([#9118](https://github.com/crystal-lang/crystal/pull/9118), [#9227](https://github.com/crystal-lang/crystal/pull/9227), [#9150](https://github.com/crystal-lang/crystal/pull/9150), [#9157](https://github.com/crystal-lang/crystal/pull/9157), thanks @bcardiff, @waj) +- **(breaking-change)** Split top-level `Log::Metadata` from `Log::Metadata::Value`, drop immutability via clone, improve performance. ([#9295](https://github.com/crystal-lang/crystal/pull/9295), thanks @bcardiff) +- **(breaking-change)** Rework `Log.setup_from_env` and defaults. ([#9145](https://github.com/crystal-lang/crystal/pull/9145), [#9240](https://github.com/crystal-lang/crystal/pull/9240), thanks @bcardiff) +- Add `Log.capture` spec helper. ([#9201](https://github.com/crystal-lang/crystal/pull/9201), thanks @bcardiff) +- Redesign `Log::Formatter`. ([#9211](https://github.com/crystal-lang/crystal/pull/9211), thanks @waj) +- Add `to_json` for `Log::Context`. ([#9101](https://github.com/crystal-lang/crystal/pull/9101), thanks @paulcsmith) +- Add `Log::IOBackend#new` with `formatter` named argument. ([#9105](https://github.com/crystal-lang/crystal/pull/9105), [#9434](https://github.com/crystal-lang/crystal/pull/9434), thanks @paulcsmith, @bcardiff) +- Allow `nil` as context raw values. ([#9121](https://github.com/crystal-lang/crystal/pull/9121), thanks @bcardiff) +- Add missing `Log#with_context`. ([#9058](https://github.com/crystal-lang/crystal/pull/9058), thanks @bcardiff) +- Fix types referred in documentation. ([#9117](https://github.com/crystal-lang/crystal/pull/9117), thanks @bcardiff) +- Allow override context within logging calls. ([#9146](https://github.com/crystal-lang/crystal/pull/9146), thanks @bcardiff) +- Check severity before backend. ([#9400](https://github.com/crystal-lang/crystal/pull/9400), thanks @asterite) + +### Concurrency + +- **(breaking-change)** Drop `Concurrent::Future` and top-level methods `delay`, `future`, `lazy`. Use [crystal-community/future.cr](https://github.com/crystal-community/future.cr). ([#9093](https://github.com/crystal-lang/crystal/pull/9093), thanks @bcardiff) + +### System + +- **(breaking-change)** Deprecate `Process#kill`, use `Process#signal`. ([#9006](https://github.com/crystal-lang/crystal/pull/9006), thanks @oprypin, @jan-zajic) +- `Process` raises `IO::Error` (or subclasses). ([#9340](https://github.com/crystal-lang/crystal/pull/9340), thanks @waj) +- Add `Process.quote` and fix shell usages in the compiler. ([#9043](https://github.com/crystal-lang/crystal/pull/9043), [#9369](https://github.com/crystal-lang/crystal/pull/9369), thanks @oprypin, @bcardiff) +- Implement `Process` support on Windows. ([#9047](https://github.com/crystal-lang/crystal/pull/9047), [#9021](https://github.com/crystal-lang/crystal/pull/9021), [#9122](https://github.com/crystal-lang/crystal/pull/9122), [#9112](https://github.com/crystal-lang/crystal/pull/9112), [#9149](https://github.com/crystal-lang/crystal/pull/9149), [#9310](https://github.com/crystal-lang/crystal/pull/9310), thanks @oprypin, @RX14, @kubo, @jan-zajic) +- Fix compile-time checking of `dup3`/`clock_gettime` methods definition. ([#9407](https://github.com/crystal-lang/crystal/pull/9407), thanks @asterite) +- Use `Int64` as portable `Process.pid` type. ([#9019](https://github.com/crystal-lang/crystal/pull/9019), thanks @oprypin) + +### Runtime + +- **(breaking-change)** Deprecate top-level `fork`. ([#9136](https://github.com/crystal-lang/crystal/pull/9136), thanks @bcardiff) +- **(breaking-change)** Move `Debug` to `Crystal` namespace. ([#9176](https://github.com/crystal-lang/crystal/pull/9176), thanks @bcardiff) +- Fix segfaults when static linking with musl. ([#9238](https://github.com/crystal-lang/crystal/pull/9238), thanks @waj) +- Allow calling `at_exit` inside `at_exit`. ([#9388](https://github.com/crystal-lang/crystal/pull/9388), thanks @asterite) +- Rework DWARF loading and fix empty backtraces in musl. ([#9267](https://github.com/crystal-lang/crystal/pull/9267), thanks @waj) +- Add DragonFly(BSD) support. ([#9178](https://github.com/crystal-lang/crystal/pull/9178), thanks @mneumann) +- Add `Crystal::System::Process` to split out system-specific implementations. ([#9035](https://github.com/crystal-lang/crystal/pull/9035), thanks @oprypin) +- Move internal `CallStack` to `Exception::CallStack`. ([#9076](https://github.com/crystal-lang/crystal/pull/9076), thanks @bcardiff) +- Specify pkg-config name for `libevent`. ([#9395](https://github.com/crystal-lang/crystal/pull/9395), thanks @jhass) + +### Spec + +- Reference global Spec in `be_a` macro. ([#9066](https://github.com/crystal-lang/crystal/pull/9066), thanks @asterite) +- Add `-h` short flag to spec runner. ([#9164](https://github.com/crystal-lang/crystal/pull/9164), thanks @straight-shoota) +- Fix `crystal spec` file paths on Windows. ([#9234](https://github.com/crystal-lang/crystal/pull/9234), thanks @oprypin) +- Refactor spec hooks. ([#9090](https://github.com/crystal-lang/crystal/pull/9090), thanks @straight-shoota) + +## Compiler + +- **(breaking-change)** Improve compiler single-file run syntax to make it shebang-friendly `#!`. ([#9171](https://github.com/crystal-lang/crystal/pull/9171), thanks @RX14) +- **(breaking-change)** Use `Process.quote` for `crystal env` output. ([#9428](https://github.com/crystal-lang/crystal/pull/9428), thanks @MakeNowJust) +- **(breaking-change)** Simplify `Link` annotation handling. ([#8972](https://github.com/crystal-lang/crystal/pull/8972), thanks @RX14) +- Fix parsing of `foo:"bar"` inside call or named tuple. ([#9033](https://github.com/crystal-lang/crystal/pull/9033), thanks @asterite) +- Fix parsing of anonymous splat and block arg. ([#9113](https://github.com/crystal-lang/crystal/pull/9113), thanks @MakeNowJust) +- Fix parsing of `unless` inside macro. ([#9024](https://github.com/crystal-lang/crystal/pull/9024), [#9167](https://github.com/crystal-lang/crystal/pull/9167), thanks @MakeNowJust) +- Fix parsing of `\ ` (backslash + space) inside regex literal to ` ` (space). ([#9079](https://github.com/crystal-lang/crystal/pull/9079), thanks @MakeNowJust) +- Fix parsing of ambiguous '+' and '-'. ([#9194](https://github.com/crystal-lang/crystal/pull/9194), thanks @max-codeware) +- Fix parsing of capitalized named argument. ([#9232](https://github.com/crystal-lang/crystal/pull/9232), thanks @asterite) +- Fix parsing of `{[] of Foo, self.foo}` expressions. ([#9329](https://github.com/crystal-lang/crystal/pull/9329), thanks @MakeNowJust) +- Fix cast fun function pointer to Proc. ([#9287](https://github.com/crystal-lang/crystal/pull/9287), thanks @asterite) +- Make compiler warn on deprecated macros. ([#9343](https://github.com/crystal-lang/crystal/pull/9343), thanks @bcardiff) +- Basic support for Win64 C lib ABI. ([#9387](https://github.com/crystal-lang/crystal/pull/9387), thanks @oprypin) +- Make the compiler able to run on Windows and compile itself. ([#9054](https://github.com/crystal-lang/crystal/pull/9054), [#9062](https://github.com/crystal-lang/crystal/pull/9062), [#9095](https://github.com/crystal-lang/crystal/pull/9095), [#9106](https://github.com/crystal-lang/crystal/pull/9106), [#9307](https://github.com/crystal-lang/crystal/pull/9307), thanks @oprypin, @Sija) +- Add docs regarding `CRYSTAL_OPTS`. ([#9018](https://github.com/crystal-lang/crystal/pull/9018), thanks @straight-shoota) +- Remove `Process.run("which")` from compiler. ([#9141](https://github.com/crystal-lang/crystal/pull/9141), thanks @straight-shoota) +- Refactor type parser. ([#9208](https://github.com/crystal-lang/crystal/pull/9208), thanks @MakeNowJust) +- Refactor & clean-up in compiler. ([#8781](https://github.com/crystal-lang/crystal/pull/8781), [#9195](https://github.com/crystal-lang/crystal/pull/9195), thanks @rhysd, @straight-shoota) +- Refactor `CrystalPath::Error`. ([#9359](https://github.com/crystal-lang/crystal/pull/9359), thanks @straight-shoota) +- Refactor and improvements on spec_helper. ([#9367](https://github.com/crystal-lang/crystal/pull/9367), [#9059](https://github.com/crystal-lang/crystal/pull/9059), [#9393](https://github.com/crystal-lang/crystal/pull/9393), [#9351](https://github.com/crystal-lang/crystal/pull/9351), [#9402](https://github.com/crystal-lang/crystal/pull/9402), thanks @straight-shoota, @jhass, @oprypin) +- Split general ABI specs from x86_64-specific ones, run on every platform. ([#9384](https://github.com/crystal-lang/crystal/pull/9384), thanks @oprypin) + +### Language semantics + +- Fix `RegexLiteral#to_s` output when first character of literal is whitespace. ([#9017](https://github.com/crystal-lang/crystal/pull/9017), thanks @MakeNowJust) +- Fix autocasting in multidispatch. ([#9004](https://github.com/crystal-lang/crystal/pull/9004), thanks @asterite) +- Fix propagation of annotations into other scopes. ([#9125](https://github.com/crystal-lang/crystal/pull/9125), thanks @asterite) +- Fix yield computation inside macro code. ([#9324](https://github.com/crystal-lang/crystal/pull/9324), thanks @asterite) +- Fix incorrect type generated with `as?` when type is a union. ([#9417](https://github.com/crystal-lang/crystal/pull/9417), [#9435](https://github.com/crystal-lang/crystal/pull/9435), thanks @asterite) +- Don't duplicate instance var in inherited generic type. ([#9433](https://github.com/crystal-lang/crystal/pull/9433), thanks @asterite) +- Ensure `type_vars` works for generic modules. ([#9161](https://github.com/crystal-lang/crystal/pull/9161), thanks @toddsundsted) +- Make autocasting work in default values against unions. ([#9366](https://github.com/crystal-lang/crystal/pull/9366), thanks @asterite) +- Skip no closure check for non-Crystal procs. ([#9248](https://github.com/crystal-lang/crystal/pull/9248), thanks @jhass) + +### Debugger + +- Improve debugging support. ([#8538](https://github.com/crystal-lang/crystal/pull/8538), thanks @skuznetsov) +- Move recent additions to `DIBuilder` to `LLVMExt`. ([#9114](https://github.com/crystal-lang/crystal/pull/9114), thanks @bcardiff) + +## Tools + +### Formatter + +- Fix formatting of regex after some comments. ([#9109](https://github.com/crystal-lang/crystal/pull/9109), thanks @MakeNowJust) +- Fix formatting of `&.!`. ([#9391](https://github.com/crystal-lang/crystal/pull/9391), thanks @MakeNowJust) +- Avoid crash on heredoc with interpolations. ([#9382](https://github.com/crystal-lang/crystal/pull/9382), thanks @MakeNowJust) +- Refactor: code clean-up. ([#9231](https://github.com/crystal-lang/crystal/pull/9231), thanks @MakeNowJust) + +### Doc generator + +- Fix links to methods with `String` default values. ([#9200](https://github.com/crystal-lang/crystal/pull/9200), thanks @bcardiff) +- Fix syntax highlighting of heredoc. ([#9396](https://github.com/crystal-lang/crystal/pull/9396), thanks @MakeNowJust) +- Correctly attach docs before annotations to following types. ([#9332](https://github.com/crystal-lang/crystal/pull/9332), thanks @asterite) +- Allow annotations and `:ditto:` in macro. ([#9341](https://github.com/crystal-lang/crystal/pull/9341), thanks @bcardiff) +- Add project name and version to API docs. ([#8792](https://github.com/crystal-lang/crystal/pull/8792), thanks @straight-shoota) +- Add version selector to API docs. ([#9074](https://github.com/crystal-lang/crystal/pull/9074), [#9187](https://github.com/crystal-lang/crystal/pull/9187), [#9250](https://github.com/crystal-lang/crystal/pull/9250), [#9252](https://github.com/crystal-lang/crystal/pull/9252), [#9254](https://github.com/crystal-lang/crystal/pull/9254), thanks @straight-shoota, @bcardiff) +- Show input type path instead of full qualified path on generic. ([#9302](https://github.com/crystal-lang/crystal/pull/9302), thanks @MakeNowJust) +- Remove README link in API docs. ([#9082](https://github.com/crystal-lang/crystal/pull/9082), thanks @straight-shoota) +- Remove special handling for version tags in docs generator. ([#9083](https://github.com/crystal-lang/crystal/pull/9083), thanks @straight-shoota) +- Refactor `is_crystal_repo` based on project name. ([#9070](https://github.com/crystal-lang/crystal/pull/9070), thanks @straight-shoota) +- Refactor `Docs::Generator` source link generation. ([#9119](https://github.com/crystal-lang/crystal/pull/9119), [#9305](https://github.com/crystal-lang/crystal/pull/9305), thanks @straight-shoota) + +### Playground + +- Allow building compiler without 'playground', to avoid dependency on sockets. ([#9031](https://github.com/crystal-lang/crystal/pull/9031), thanks @oprypin) +- Add support to jquery version 3. ([#9028](https://github.com/crystal-lang/crystal/pull/9028), thanks @deiv) + +## Others + +- CI improvements and housekeeping. ([#9012](https://github.com/crystal-lang/crystal/pull/9012), [#9129](https://github.com/crystal-lang/crystal/pull/9129), [#9242](https://github.com/crystal-lang/crystal/pull/9242), [#9370](https://github.com/crystal-lang/crystal/pull/9370), thanks @bcardiff) +- Update to Shards 0.11.1. ([#9446](https://github.com/crystal-lang/crystal/pull/9446), thanks @bcardiff) +- Tidy up Makefile and crystal env output. ([#9423](https://github.com/crystal-lang/crystal/pull/9423), thanks @bcardiff) +- Always include `lib` directory in the `CRYSTAL_PATH`. ([#9315](https://github.com/crystal-lang/crystal/pull/9315), thanks @waj) +- Use `SOURCE_DATE_EPOCH` only to determine compiler date. ([#9088](https://github.com/crystal-lang/crystal/pull/9088), thanks @straight-shoota) +- Win CI: Bootstrap Crystal, build things on Windows, publish the binary. ([#9123](https://github.com/crystal-lang/crystal/pull/9123), [#9155](https://github.com/crystal-lang/crystal/pull/9155), [#9144](https://github.com/crystal-lang/crystal/pull/9144), [#9346](https://github.com/crystal-lang/crystal/pull/9346), thanks @oprypin) +- Regenerate implementation tool sample. ([#9003](https://github.com/crystal-lang/crystal/pull/9003), thanks @nulty) +- Avoid requiring non std-lib spec spec_helper. ([#9294](https://github.com/crystal-lang/crystal/pull/9294), thanks @bcardiff) +- Improve grammar and fix typos. ([#9087](https://github.com/crystal-lang/crystal/pull/9087), [#9212](https://github.com/crystal-lang/crystal/pull/9212), [#9368](https://github.com/crystal-lang/crystal/pull/9368), thanks @MakeNowJust, @j8r) +- Hide internal functions in docs. ([#9410](https://github.com/crystal-lang/crystal/pull/9410), thanks @bcardiff) +- Advertise full `crystal spec` command for running a particular spec. ([#9103](https://github.com/crystal-lang/crystal/pull/9103), thanks @paulcsmith) +- Update README. ([#9225](https://github.com/crystal-lang/crystal/pull/9225), [#9163](https://github.com/crystal-lang/crystal/pull/9163), thanks @danimiba, @straight-shoota) +- Fix LICENSE and add NOTICE file. ([#3903](https://github.com/crystal-lang/crystal/pull/3903), thanks @MakeNowJust) + # 0.34.0 (2020-04-06) ## Language changes diff --git a/src/VERSION b/src/VERSION index cd748687dc99..7b52f5e5178d 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -0.35.0-dev +0.35.0 From 452fee05a769e8d3f67cc1b7bee35042f68b3466 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 10 Jun 2020 18:14:23 -0300 Subject: [PATCH 118/263] Set VERSION to 0.35.1-dev (#9455) --- src/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VERSION b/src/VERSION index 7b52f5e5178d..ac40018a669e 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -0.35.0 +0.35.1-dev From 53917547512e3e5b00bc9c79a21dae7265b37bff Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 10 Jun 2020 18:14:40 -0300 Subject: [PATCH 119/263] Revert Hash#each restriction added in #8887 (#9456) --- spec/std/hash_spec.cr | 12 ++++++++++++ src/hash.cr | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/spec/std/hash_spec.cr b/spec/std/hash_spec.cr index e241f09404a0..35672eb5804d 100644 --- a/spec/std/hash_spec.cr +++ b/spec/std/hash_spec.cr @@ -14,6 +14,14 @@ end private alias RecursiveType = String | Int32 | Array(RecursiveType) | Hash(Symbol, RecursiveType) +private class HashWrapper(K, V) + include Enumerable({K, V}) + + @hash = {} of K => V + + delegate each, to: @hash +end + describe "Hash" do describe "empty" do it "size should be zero" do @@ -1225,4 +1233,8 @@ describe "Hash" do h.clone.compare_by_identity?.should be_true end end + + it "can be wrapped" do + HashWrapper(Int32, Int32).new.to_a.should be_empty + end end diff --git a/src/hash.cr b/src/hash.cr index 98bf0a796916..c5577b8d4a33 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -1255,7 +1255,7 @@ class Hash(K, V) # ``` # # The enumeration follows the order the keys were inserted. - def each(& : Tuple(K, V) ->) : Nil + def each : Nil each_entry_with_index do |entry, i| yield({entry.key, entry.value}) end From d08b646e1c98efae75e9dd4684aae2fa2ca34234 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Thu, 11 Jun 2020 06:18:14 +0900 Subject: [PATCH 120/263] Parser: fix parsing of `{foo: X, typeof: Y}` type (#9453) --- spec/compiler/parser/parser_spec.cr | 1 + src/compiler/crystal/syntax/parser.cr | 1 + 2 files changed, 2 insertions(+) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 116b5c9dc047..0a13970cdaff 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -565,6 +565,7 @@ module Crystal it_parses "Foo({x: X, y: Y})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path), NamedArgument.new("y", "Y".path)])] of ASTNode) it_parses "Foo({X: X, Y: Y})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("X", "X".path), NamedArgument.new("Y", "Y".path)])] of ASTNode) it_parses "Foo(T, {x: X})", Generic.new("Foo".path, ["T".path, Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path)])] of ASTNode) + it_parses "Foo({x: X, typeof: Y})", Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("x", "X".path), NamedArgument.new("typeof", "Y".path)])] of ASTNode) assert_syntax_error "Foo({x: X, x: Y})", "duplicated key: x" it_parses %(Foo({"foo bar": X})), Generic.new("Foo".path, [Generic.new(Path.global("NamedTuple"), [] of ASTNode, named_args: [NamedArgument.new("foo bar", "X".path)])] of ASTNode) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 9f326e3235f6..d4dd4a62f968 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -5057,6 +5057,7 @@ module Crystal case @token.type when :IDENT + return false if named_tuple_start? case @token.value when :typeof true From 3a4be658c55b71c401b97a7525fbc33f92e7880c Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Thu, 11 Jun 2020 23:04:30 +0900 Subject: [PATCH 121/263] Parser: fix parsing of proc in hash `of` key type (#9458) Fixed #9451 --- spec/compiler/parser/parser_spec.cr | 1 + src/compiler/crystal/syntax/parser.cr | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 0a13970cdaff..cebb4295b14d 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -1067,6 +1067,7 @@ module Crystal assert_syntax_error "{a: 1, a: 2}", "duplicated key: a" it_parses "{} of Int => Double", HashLiteral.new([] of HashLiteral::Entry, of: HashLiteral::Entry.new("Int".path, "Double".path)) + it_parses "{} of Int32 -> Int32 => Int32", HashLiteral.new([] of HashLiteral::Entry, of: HashLiteral::Entry.new(ProcNotation.new(["Int32".path] of ASTNode, "Int32".path), "Int32".path)) it_parses "require \"foo\"", Require.new("foo") it_parses "require \"foo\"; [1]", [Require.new("foo"), ([1.int32] of ASTNode).array] diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index d4dd4a62f968..f687ea3c4be4 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -5103,7 +5103,7 @@ module Crystal # They are conflicted with operators, so more look-ahead is needed. next_token_skip_space delimiter_or_type_suffix? - when :"->", :"|", :",", :NEWLINE, :EOF, :"=", :";", :"(", :")", :"[", :"]" + when :"->", :"|", :",", :"=>", :NEWLINE, :EOF, :"=", :";", :"(", :")", :"[", :"]" true else false From 224e41bdb5dd2ad612d0d39a9195705b946eca54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 12 Jun 2020 14:31:42 +0200 Subject: [PATCH 122/263] Use less strict cipher compatibility for OpenSSL client context (#9459) --- spec/std/openssl/ssl/context_spec.cr | 2 +- src/openssl/ssl/context.cr | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/spec/std/openssl/ssl/context_spec.cr b/spec/std/openssl/ssl/context_spec.cr index aa357375f77c..df71c4dfa127 100644 --- a/spec/std/openssl/ssl/context_spec.cr +++ b/spec/std/openssl/ssl/context_spec.cr @@ -100,7 +100,7 @@ describe OpenSSL::SSL::Context do pending "uses intermediate default ciphers" do # Can't be checked because `Context#ciphers` is not implemented. - OpenSSL::SSL::Context::Client.new.ciphers.should eq OpenSSL::SSL::Context::CIPHERS_INTERMEDIATE + OpenSSL::SSL::Context::Client.new.ciphers.should eq OpenSSL::SSL::Context::CIPHERS_OLD OpenSSL::SSL::Context::Server.new.ciphers.should eq OpenSSL::SSL::Context::CIPHERS_INTERMEDIATE end diff --git a/src/openssl/ssl/context.cr b/src/openssl/ssl/context.cr index bfe3707f7fcf..da4eb8d78bf1 100644 --- a/src/openssl/ssl/context.cr +++ b/src/openssl/ssl/context.cr @@ -5,8 +5,6 @@ require "uri/punycode" # For both server and client applications exist more specialized subclassses # `SSL::Context::Server` and `SSL::Context::Client` which need to be instantiated # appropriately. -# -# All instances use `CIPHERS_INTERMEDIATE` ciphers by default. abstract class OpenSSL::SSL::Context # :nodoc: def self.default_method @@ -18,6 +16,8 @@ abstract class OpenSSL::SSL::Context end class Client < Context + @hostname : String? + # Generates a new TLS client context with sane defaults for a client connection. # # Defaults to `TLS_method` or `SSLv23_method` (depending on OpenSSL version) @@ -36,9 +36,8 @@ abstract class OpenSSL::SSL::Context # context = OpenSSL::SSL::Context::Client.new # context.add_options(OpenSSL::SSL::Options::NO_SSL_V2 | OpenSSL::SSL::Options::NO_SSL_V3) # ``` - - @hostname : String? - + # + # It uses `CIPHERS_OLD` compatibility level by default. def initialize(method : LibSSL::SSLMethod = Context.default_method) super(method) @@ -46,6 +45,8 @@ abstract class OpenSSL::SSL::Context {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.0.2") >= 0 %} self.default_verify_param = "ssl_server" {% end %} + + self.ciphers = CIPHERS_OLD end # Returns a new TLS client context with only the given method set. @@ -115,6 +116,8 @@ abstract class OpenSSL::SSL::Context # context = OpenSSL::SSL::Context::Server.new # context.add_options(OpenSSL::SSL::Options::NO_SSL_V2 | OpenSSL::SSL::Options::NO_SSL_V3) # ``` + # + # It uses `CIPHERS_INTERMEDIATE` compatibility level by default. def initialize(method : LibSSL::SSLMethod = Context.default_method) super(method) @@ -124,6 +127,8 @@ abstract class OpenSSL::SSL::Context {% end %} set_tmp_ecdh_key(curve: LibCrypto::NID_X9_62_prime256v1) + + self.ciphers = CIPHERS_INTERMEDIATE end # Returns a new TLS server context with only the given method set. @@ -187,8 +192,6 @@ abstract class OpenSSL::SSL::Context )) add_modes(OpenSSL::SSL::Modes.flags(AUTO_RETRY, RELEASE_BUFFERS)) - - self.ciphers = CIPHERS_INTERMEDIATE end # Overriding initialize or new in the child classes as public methods, From 5c68a02581c3da8165bf269ed49bf047c53bf890 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 12 Jun 2020 12:56:38 -0300 Subject: [PATCH 123/263] Compiler: show warnings even if there are errors (#9461) --- spec/compiler/codegen/warnings_spec.cr | 265 ------------------ spec/compiler/semantic/warnings_spec.cr | 262 +++++++++++++++++ src/compiler/crystal/codegen/call.cr | 2 - src/compiler/crystal/codegen/warnings.cr | 109 ------- src/compiler/crystal/command.cr | 30 +- src/compiler/crystal/command/docs.cr | 6 +- src/compiler/crystal/command/eval.cr | 2 +- src/compiler/crystal/command/spec.cr | 6 +- src/compiler/crystal/compiler.cr | 5 +- src/compiler/crystal/program.cr | 12 - .../crystal/semantic/cleanup_transformer.cr | 2 + src/compiler/crystal/semantic/warnings.cr | 127 ++++++++- 12 files changed, 421 insertions(+), 407 deletions(-) delete mode 100644 spec/compiler/codegen/warnings_spec.cr delete mode 100644 src/compiler/crystal/codegen/warnings.cr diff --git a/spec/compiler/codegen/warnings_spec.cr b/spec/compiler/codegen/warnings_spec.cr deleted file mode 100644 index c0291e9826ce..000000000000 --- a/spec/compiler/codegen/warnings_spec.cr +++ /dev/null @@ -1,265 +0,0 @@ -require "../spec_helper" - -describe "Code gen: warnings" do - it "detects top-level deprecated methods" do - assert_warning <<-CR, - @[Deprecated("Do not use me")] - def foo - end - - foo - CR - "warning in line 5\nWarning: Deprecated top-level foo. Do not use me" - end - - it "deprecation reason is optional" do - assert_warning <<-CR, - @[Deprecated] - def foo - end - - foo - CR - "warning in line 5\nWarning: Deprecated top-level foo." - end - - it "detects deprecated instance methods" do - assert_warning <<-CR, - class Foo - @[Deprecated("Do not use me")] - def m - end - end - - Foo.new.m - CR - "warning in line 7\nWarning: Deprecated Foo#m. Do not use me" - end - - it "detects deprecated class methods" do - assert_warning <<-CR, - class Foo - @[Deprecated("Do not use me")] - def self.m - end - end - - Foo.m - CR - "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" - end - - it "detects deprecated generic instance methods" do - assert_warning <<-CR, - class Foo(T) - @[Deprecated("Do not use me")] - def m - end - end - - Foo(Int32).new.m - CR - "warning in line 7\nWarning: Deprecated Foo(Int32)#m. Do not use me" - end - - it "detects deprecated generic class methods" do - assert_warning <<-CR, - class Foo(T) - @[Deprecated("Do not use me")] - def self.m - end - end - - Foo(Int32).m - CR - "warning in line 7\nWarning: Deprecated Foo(Int32).m. Do not use me" - end - - it "detects deprecated module methods" do - assert_warning <<-CR, - module Foo - @[Deprecated("Do not use me")] - def self.m - end - end - - Foo.m - CR - "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" - end - - it "detects deprecated methods with named arguments" do - assert_warning <<-CR, - @[Deprecated] - def foo(*, a) - end - - foo(a: 2) - CR - "warning in line 5\nWarning: Deprecated top-level foo:a." - end - - it "detects deprecated initialize" do - assert_warning <<-CR, - class Foo - @[Deprecated] - def initialize - end - end - - Foo.new - CR - "warning in line 7\nWarning: Deprecated Foo.new." - end - - it "detects deprecated initialize with named arguments" do - assert_warning <<-CR, - class Foo - @[Deprecated] - def initialize(*, a) - end - end - - Foo.new(a: 2) - CR - "warning in line 7\nWarning: Deprecated Foo.new:a." - end - - it "informs warnings once per call site location (a)" do - warning_failures = warnings_result <<-CR - class Foo - @[Deprecated("Do not use me")] - def m - end - - def b - m - end - end - - Foo.new.b - Foo.new.b - CR - warning_failures.size.should eq(1) - end - - it "informs warnings once per call site location (b)" do - warning_failures = warnings_result <<-CR - class Foo - @[Deprecated("Do not use me")] - def m - end - end - - Foo.new.m - Foo.new.m - CR - - warning_failures.size.should eq(2) - end - - it "informs warnings once per yield" do - warning_failures = warnings_result <<-CR - class Foo - @[Deprecated("Do not use me")] - def m - end - end - - def twice - yield - yield - end - - twice { Foo.new.m } - CR - - warning_failures.size.should eq(1) - end - - it "informs warnings once per target type" do - warning_failures = warnings_result <<-CR - class Foo(T) - @[Deprecated("Do not use me")] - def m - end - - def b - m - end - end - - Foo(Int32).new.b - Foo(Int64).new.b - CR - - warning_failures.size.should eq(2) - end - - it "ignore deprecation excluded locations" do - with_tempfile("check_warnings_excludes") do |path| - FileUtils.mkdir_p File.join(path, "lib") - - # NOTE tempfile might be created in symlinked folder - # which affects how to match current dir /var/folders/... - # with the real path /private/var/folders/... - path = File.real_path(path) - - main_filename = File.join(path, "main.cr") - output_filename = File.join(path, "main") - - Dir.cd(path) do - File.write main_filename, <<-CR - require "./lib/foo" - - bar - foo - CR - File.write File.join(path, "lib", "foo.cr"), <<-CR - @[Deprecated("Do not use me")] - def foo - end - - def bar - foo - end - CR - - compiler = create_spec_compiler - compiler.warnings = Warnings::All - compiler.warnings_exclude << Crystal.normalize_path "lib" - compiler.prelude = "empty" - result = compiler.compile Compiler::Source.new(main_filename, File.read(main_filename)), output_filename - - result.program.warning_failures.size.should eq(1) - end - end - end - - it "errors if invalid argument type" do - assert_error <<-CR, - @[Deprecated(42)] - def foo - end - CR - "Error: first argument must be a String" - end - - it "errors if too many arguments" do - assert_error <<-CR, - @[Deprecated("Do not use me", "extra arg")] - def foo - end - CR - "Error: wrong number of deprecated annotation arguments (given 2, expected 1)" - end - - it "errors if invalid named arguments" do - assert_error <<-CR, - @[Deprecated(invalid: "Do not use me")] - def foo - end - CR - "Error: too many named arguments (given 1, expected maximum 0)" - end -end diff --git a/spec/compiler/semantic/warnings_spec.cr b/spec/compiler/semantic/warnings_spec.cr index d5cc0d3c5970..4230df9f4113 100644 --- a/spec/compiler/semantic/warnings_spec.cr +++ b/spec/compiler/semantic/warnings_spec.cr @@ -1,6 +1,268 @@ require "../spec_helper" describe "Semantic: warnings" do + it "detects top-level deprecated methods" do + assert_warning <<-CR, + @[Deprecated("Do not use me")] + def foo + end + + foo + CR + "warning in line 5\nWarning: Deprecated top-level foo. Do not use me" + end + + it "deprecation reason is optional" do + assert_warning <<-CR, + @[Deprecated] + def foo + end + + foo + CR + "warning in line 5\nWarning: Deprecated top-level foo." + end + + it "detects deprecated instance methods" do + assert_warning <<-CR, + class Foo + @[Deprecated("Do not use me")] + def m + end + end + + Foo.new.m + CR + "warning in line 7\nWarning: Deprecated Foo#m. Do not use me" + end + + it "detects deprecated class methods" do + assert_warning <<-CR, + class Foo + @[Deprecated("Do not use me")] + def self.m + end + end + + Foo.m + CR + "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" + end + + it "detects deprecated generic instance methods" do + assert_warning <<-CR, + class Foo(T) + @[Deprecated("Do not use me")] + def m + end + end + + Foo(Int32).new.m + CR + "warning in line 7\nWarning: Deprecated Foo(Int32)#m. Do not use me" + end + + it "detects deprecated generic class methods" do + assert_warning <<-CR, + class Foo(T) + @[Deprecated("Do not use me")] + def self.m + end + end + + Foo(Int32).m + CR + "warning in line 7\nWarning: Deprecated Foo(Int32).m. Do not use me" + end + + it "detects deprecated module methods" do + assert_warning <<-CR, + module Foo + @[Deprecated("Do not use me")] + def self.m + end + end + + Foo.m + CR + "warning in line 7\nWarning: Deprecated Foo.m. Do not use me" + end + + it "detects deprecated methods with named arguments" do + assert_warning <<-CR, + @[Deprecated] + def foo(*, a) + end + + foo(a: 2) + CR + "warning in line 5\nWarning: Deprecated top-level foo:a." + end + + it "detects deprecated initialize" do + assert_warning <<-CR, + class Foo + @[Deprecated] + def initialize + end + end + + Foo.new + CR + "warning in line 7\nWarning: Deprecated Foo.new." + end + + it "detects deprecated initialize with named arguments" do + assert_warning <<-CR, + class Foo + @[Deprecated] + def initialize(*, a) + end + end + + Foo.new(a: 2) + CR + "warning in line 7\nWarning: Deprecated Foo.new:a." + end + + it "informs warnings once per call site location (a)" do + warning_failures = warnings_result <<-CR + class Foo + @[Deprecated("Do not use me")] + def m + end + + def b + m + end + end + + Foo.new.b + Foo.new.b + CR + warning_failures.size.should eq(1) + end + + it "informs warnings once per call site location (b)" do + warning_failures = warnings_result <<-CR + class Foo + @[Deprecated("Do not use me")] + def m + end + end + + Foo.new.m + Foo.new.m + CR + + warning_failures.size.should eq(2) + end + + it "informs warnings once per yield" do + warning_failures = warnings_result <<-CR + class Foo + @[Deprecated("Do not use me")] + def m + end + end + + def twice + yield + yield + end + + twice { Foo.new.m } + CR + + warning_failures.size.should eq(1) + end + + it "informs warnings once per target type" do + warning_failures = warnings_result <<-CR + class Foo(T) + @[Deprecated("Do not use me")] + def m + end + + def b + m + end + end + + Foo(Int32).new.b + Foo(Int64).new.b + CR + + warning_failures.size.should eq(2) + end + + it "ignore deprecation excluded locations" do + with_tempfile("check_warnings_excludes") do |path| + FileUtils.mkdir_p File.join(path, "lib") + + # NOTE tempfile might be created in symlinked folder + # which affects how to match current dir /var/folders/... + # with the real path /private/var/folders/... + path = File.real_path(path) + + main_filename = File.join(path, "main.cr") + output_filename = File.join(path, "main") + + Dir.cd(path) do + File.write main_filename, <<-CR + require "./lib/foo" + + bar + foo + CR + File.write File.join(path, "lib", "foo.cr"), <<-CR + @[Deprecated("Do not use me")] + def foo + end + + def bar + foo + end + CR + + compiler = create_spec_compiler + compiler.warnings = Warnings::All + compiler.warnings_exclude << Crystal.normalize_path "lib" + compiler.prelude = "empty" + result = compiler.compile Compiler::Source.new(main_filename, File.read(main_filename)), output_filename + + result.program.warning_failures.size.should eq(1) + end + end + end + + it "errors if invalid argument type" do + assert_error <<-CR, + @[Deprecated(42)] + def foo + end + CR + "Error: first argument must be a String" + end + + it "errors if too many arguments" do + assert_error <<-CR, + @[Deprecated("Do not use me", "extra arg")] + def foo + end + CR + "Error: wrong number of deprecated annotation arguments (given 2, expected 1)" + end + + it "errors if invalid named arguments" do + assert_error <<-CR, + @[Deprecated(invalid: "Do not use me")] + def foo + end + CR + "Error: too many named arguments (given 1, expected maximum 0)" + end + it "detects top-level deprecated marcos" do assert_warning %( @[Deprecated("Do not use me")] diff --git a/src/compiler/crystal/codegen/call.cr b/src/compiler/crystal/codegen/call.cr index 78cadf4b39ac..567107050686 100644 --- a/src/compiler/crystal/codegen/call.cr +++ b/src/compiler/crystal/codegen/call.cr @@ -16,8 +16,6 @@ class Crystal::CodeGenVisitor return false end - check_call_to_deprecated_method node - owner = node.name == "super" ? node.scope : node.target_def.owner call_args, has_out = prepare_call_args node, owner diff --git a/src/compiler/crystal/codegen/warnings.cr b/src/compiler/crystal/codegen/warnings.cr deleted file mode 100644 index 5ddee37c273a..000000000000 --- a/src/compiler/crystal/codegen/warnings.cr +++ /dev/null @@ -1,109 +0,0 @@ -module Crystal - class Program - def ignore_warning_due_to_location?(location : Location?) - return false unless location - - filename = location.original_filename - return false unless filename - - @program.warnings_exclude.any? do |path| - filename.starts_with?(path) - end - end - end - - struct DeprecatedAnnotation - getter message : String? - - def initialize(@message = nil) - end - - def self.from(ann : Annotation) - args = ann.args - named_args = ann.named_args - - if named_args - ann.raise "too many named arguments (given #{named_args.size}, expected maximum 0)" - end - - message = nil - count = 0 - - args.each do |arg| - case count - when 0 - arg.raise "first argument must be a String" unless arg.is_a?(StringLiteral) - message = arg.value - else - ann.wrong_number_of "deprecated annotation arguments", args.size, "1" - end - - count += 1 - end - - new(message) - end - end - - class Def - def short_reference - case owner - when Program - "top-level #{name}" - when .metaclass? - "#{owner.instance_type}.#{name}" - else - "#{owner}##{name}" - end - end - end - - class CodeGenVisitor - @deprecated_methods_detected = Set(String).new - - def check_call_to_deprecated_method(node : Call) - return unless @program.warnings.all? - - if (ann = node.target_def.annotation(@program.deprecated_annotation)) && - (deprecated_annotation = DeprecatedAnnotation.from(ann)) - return if compiler_expanded_call(node) - return if @program.ignore_warning_due_to_location?(node.location) - short_reference = node.target_def.short_reference - warning_key = node.location.try { |l| "#{short_reference} #{l}" } - - # skip warning if the call site was already informed - # if there is no location information just inform it. - return if !warning_key || @deprecated_methods_detected.includes?(warning_key) - @deprecated_methods_detected.add(warning_key) if warning_key - - message = deprecated_annotation.message - message = message ? " #{message}" : "" - - full_message = node.warning "Deprecated #{short_reference}.#{message}" - - @program.warning_failures << full_message - end - end - - private def compiler_expanded_call(node : Call) - # Compiler generates a `_.initialize` call in `new` - node.obj.as?(Var).try { |v| v.name == "_" } && node.name == "initialize" - end - end - - class Command - def report_warnings(result : Compiler::Result) - return if result.program.warning_failures.empty? - - result.program.warning_failures.each do |message| - STDERR.puts message - STDERR.puts "\n" - end - STDERR.puts "A total of #{result.program.warning_failures.size} warnings were found." - end - - def warnings_fail_on_exit?(result : Compiler::Result) - result.program.error_on_warnings && result.program.warning_failures.size > 0 - end - end -end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 7a6d5b3f60aa..9f722944e12f 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -50,6 +50,7 @@ class Crystal::Command end private getter options + @compiler : Compiler? def initialize(@options : Array(String)) @color = ENV["TERM"]? != "dumb" @@ -69,10 +70,9 @@ class Crystal::Command when "build".starts_with?(command) options.shift use_crystal_opts - result = build - report_warnings result - exit 1 if warnings_fail_on_exit?(result) - result + build + report_warnings + exit 1 if warnings_fail_on_exit? when "play".starts_with?(command) options.shift {% if flag?(:without_playground) %} @@ -122,8 +122,12 @@ class Crystal::Command end end rescue ex : Crystal::LocationlessException + report_warnings + error ex.message rescue ex : Crystal::Exception + report_warnings + ex.color = @color ex.error_trace = @error_trace if @config.try(&.output_format) == "json" @@ -135,6 +139,8 @@ class Crystal::Command rescue ex : OptionParser::Exception error ex.message rescue ex + report_warnings + ex.inspect_with_backtrace STDERR error "you've found a bug in the Crystal compiler. Please open an issue, including source code that will allow us to reproduce the bug: https://github.com/crystal-lang/crystal/issues" end @@ -190,9 +196,9 @@ class Crystal::Command private def run_command(single_file = false) config = create_compiler "run", run: true, single_file: single_file if config.specified_output - result = config.compile - report_warnings result - exit 1 if warnings_fail_on_exit?(result) + config.compile + report_warnings + exit 1 if warnings_fail_on_exit? return end @@ -201,8 +207,8 @@ class Crystal::Command result = config.compile output_filename unless config.compiler.no_codegen? - report_warnings result - exit 1 if warnings_fail_on_exit?(result) + report_warnings + exit 1 if warnings_fail_on_exit? execute output_filename, config.arguments, config.compiler end @@ -297,7 +303,7 @@ class Crystal::Command private def create_compiler(command, no_codegen = false, run = false, hierarchy = false, cursor_command = false, single_file = false) - compiler = Compiler.new + compiler = new_compiler compiler.progress_tracker = @progress_tracker link_flags = [] of String filenames = [] of String @@ -616,4 +622,8 @@ class Crystal::Command private def use_crystal_opts @options = ENV.fetch("CRYSTAL_OPTS", "").split.concat(options) end + + private def new_compiler + @compiler = Compiler.new + end end diff --git a/src/compiler/crystal/command/docs.cr b/src/compiler/crystal/command/docs.cr index 61447c8dc527..573bb3a4fc63 100644 --- a/src/compiler/crystal/command/docs.cr +++ b/src/compiler/crystal/command/docs.cr @@ -14,7 +14,7 @@ class Crystal::Command sitemap_changefreq = "never" project_info = Doc::ProjectInfo.new - compiler = Compiler.new + compiler = new_compiler OptionParser.parse(options) do |opts| opts.banner = <<-'BANNER' @@ -140,7 +140,7 @@ class Crystal::Command Doc::Generator.new(result.program, included_dirs, output_directory, output_format, sitemap_base_url, sitemap_priority, sitemap_changefreq, project_info).run - report_warnings result - exit 1 if warnings_fail_on_exit?(result) + report_warnings + exit 1 if warnings_fail_on_exit? end end diff --git a/src/compiler/crystal/command/eval.cr b/src/compiler/crystal/command/eval.cr index f982d19f6efb..efed166c358c 100644 --- a/src/compiler/crystal/command/eval.cr +++ b/src/compiler/crystal/command/eval.cr @@ -2,7 +2,7 @@ class Crystal::Command private def eval - compiler = Compiler.new + compiler = new_compiler OptionParser.parse(options) do |opts| opts.banner = "Usage: crystal eval [options] [source]\n\nOptions:" setup_simple_compiler_options compiler, opts diff --git a/src/compiler/crystal/command/spec.cr b/src/compiler/crystal/command/spec.cr index 41785ccc6af3..0af8808a205a 100644 --- a/src/compiler/crystal/command/spec.cr +++ b/src/compiler/crystal/command/spec.cr @@ -10,7 +10,7 @@ class Crystal::Command private def spec - compiler = Compiler.new + compiler = new_compiler OptionParser.parse(options) do |opts| opts.banner = "Usage: crystal spec [options] [files]\n\nOptions:" setup_simple_compiler_options compiler, opts @@ -74,7 +74,7 @@ class Crystal::Command output_filename = Crystal.temp_executable "spec" result = compiler.compile sources, output_filename - report_warnings result - execute output_filename, options, compiler, error_on_exit: warnings_fail_on_exit?(result) + report_warnings + execute output_filename, options, compiler, error_on_exit: warnings_fail_on_exit? end end diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index c017281eb54a..57bdb9d597ea 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -153,6 +153,9 @@ module Crystal # Whether to use llvm ThinLTO for linking property thin_lto = false + # Program that was created for the last compilation. + property! program : Program + # Compiles the given *source*, with *output_filename* as the name # of the generated executable. # @@ -198,7 +201,7 @@ module Crystal end private def new_program(sources) - program = Program.new + @program = program = Program.new program.filename = sources.first.filename program.cache_dir = CacheDir.instance.directory_for(sources) program.codegen_target = codegen_target diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index 16783bc7b2c7..cd576c8f57bd 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -118,18 +118,6 @@ module Crystal property codegen_target = Config.host_target - # Which kind of warnings wants to be detected. - property warnings : Warnings = Warnings::All - - # Paths to ignore for warnings detection. - property warnings_exclude : Array(String) = [] of String - - # Detected warning failures. - property warning_failures = [] of String - - # If `true` compiler will error if warnings are found. - property error_on_warnings : Bool = false - def initialize super(self, self, "main") diff --git a/src/compiler/crystal/semantic/cleanup_transformer.cr b/src/compiler/crystal/semantic/cleanup_transformer.cr index a0d1ffb39364..c35d4030e8df 100644 --- a/src/compiler/crystal/semantic/cleanup_transformer.cr +++ b/src/compiler/crystal/semantic/cleanup_transformer.cr @@ -326,6 +326,8 @@ module Crystal return expanded.transform self end + @program.check_call_to_deprecated_method(node) + # Need to transform these manually because node.block doesn't # need to be transformed if it has a fun_literal # ~~~ diff --git a/src/compiler/crystal/semantic/warnings.cr b/src/compiler/crystal/semantic/warnings.cr index c7ff6b393a35..48fa1b504ef8 100644 --- a/src/compiler/crystal/semantic/warnings.cr +++ b/src/compiler/crystal/semantic/warnings.cr @@ -1,5 +1,20 @@ module Crystal class Program + # Which kind of warnings wants to be detected. + property warnings : Warnings = Warnings::All + + # Paths to ignore for warnings detection. + property warnings_exclude : Array(String) = [] of String + + # Detected warning failures. + property warning_failures = [] of String + + # If `true` compiler will error if warnings are found. + property error_on_warnings : Bool = false + + @deprecated_methods_detected = Set(String).new + @deprecated_macros_detected = Set(String).new + def report_warning(node : ASTNode, message : String) return unless self.warnings.all? return if self.ignore_warning_due_to_location?(node.location) @@ -22,7 +37,16 @@ module Crystal self.warning_failures << message end - @deprecated_macros_detected = Set(String).new + def ignore_warning_due_to_location?(location : Location?) + return false unless location + + filename = location.original_filename + return false unless filename + + @program.warnings_exclude.any? do |path| + filename.starts_with?(path) + end + end def check_call_to_deprecated_macro(a_macro : Macro, call : Call) return unless self.warnings.all? @@ -48,6 +72,37 @@ module Crystal self.warning_failures << full_message end end + + def check_call_to_deprecated_method(node : Call) + return unless @warnings.all? + + node.target_defs.try &.each do |target_def| + if (ann = target_def.annotation(deprecated_annotation)) && + (deprecated_annotation = DeprecatedAnnotation.from(ann)) + return if compiler_expanded_call(node) + return if ignore_warning_due_to_location?(node.location) + short_reference = target_def.short_reference + warning_key = node.location.try { |l| "#{short_reference} #{l}" } + + # skip warning if the call site was already informed + # if there is no location information just inform it. + return if !warning_key || @deprecated_methods_detected.includes?(warning_key) + @deprecated_methods_detected.add(warning_key) if warning_key + + message = deprecated_annotation.message + message = message ? " #{message}" : "" + + full_message = node.warning "Deprecated #{short_reference}.#{message}" + + self.warning_failures << full_message + end + end + end + + private def compiler_expanded_call(node : Call) + # Compiler generates a `_.initialize` call in `new` + node.obj.as?(Var).try { |v| v.name == "_" } && node.name == "initialize" + end end class Macro @@ -62,4 +117,74 @@ module Crystal end end end + + struct DeprecatedAnnotation + getter message : String? + + def initialize(@message = nil) + end + + def self.from(ann : Annotation) + args = ann.args + named_args = ann.named_args + + if named_args + ann.raise "too many named arguments (given #{named_args.size}, expected maximum 0)" + end + + message = nil + count = 0 + + args.each do |arg| + case count + when 0 + arg.raise "first argument must be a String" unless arg.is_a?(StringLiteral) + message = arg.value + else + ann.wrong_number_of "deprecated annotation arguments", args.size, "1" + end + + count += 1 + end + + new(message) + end + end + + class Def + def short_reference + case owner + when Program + "top-level #{name}" + when .metaclass? + "#{owner.instance_type}.#{name}" + else + "#{owner}##{name}" + end + end + end + + class Command + def report_warnings + compiler = @compiler + return unless compiler + + program = compiler.program + return if program.warning_failures.empty? + + program.warning_failures.each do |message| + STDERR.puts message + STDERR.puts "\n" + end + STDERR.puts "A total of #{program.warning_failures.size} warnings were found." + end + + def warnings_fail_on_exit? + compiler = @compiler + return false unless compiler + + program = compiler.program + program.error_on_warnings && program.warning_failures.size > 0 + end + end end From 79f30223a2d3b7b3c4e0ff1e527cde6e808c998f Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Sat, 13 Jun 2020 12:23:09 -0300 Subject: [PATCH 124/263] Fix Log.context.set docs for hash based data (#9470) Fixes #9449 --- src/log/main.cr | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/log/main.cr b/src/log/main.cr index 9d7d8f695994..28f956e9ab0f 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -127,13 +127,13 @@ class Log # ``` # Log.context.set a: 1 # Log.context.set b: 2 - # Log.info { %q(message with {"a" => 1, "b" => 2} context") } - # extra = {:c => "3"} - # Log.context.set extra - # Log.info { %q(message with {"a" => 1, "b" => 2, "c" => "3" } context) } - # extra = {"c" => 3} - # Log.context.set extra - # Log.info { %q(message with {"a" => 1, "b" => 2, "c" => 3 } context) } + # Log.info { %q(message with a: 1, b: 2 context") } + # h = {:c => "3"} + # Log.context.set extra: h + # Log.info { %q(message with a: 1, b: 2, extra: {"c" => "3"} context) } + # h = {"c" => 3} + # Log.context.set extra: h + # Log.info { %q(message with a: 1, b: 2, extra: {"c" => 3} context) } # ``` def set(**kwargs) extend_fiber_context(Fiber.current, kwargs) From 68f0f38a22d1470eee722f6c41a2532f3ef2bb78 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Mon, 15 Jun 2020 22:37:02 +0900 Subject: [PATCH 125/263] Remove trailing whitespace in etc/lldb/crystal_formatters.py (#9482) --- etc/lldb/crystal_formatters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/lldb/crystal_formatters.py b/etc/lldb/crystal_formatters.py index e312f88642b3..856dea7c4f55 100644 --- a/etc/lldb/crystal_formatters.py +++ b/etc/lldb/crystal_formatters.py @@ -22,10 +22,10 @@ def get_child_index(self, name): return int(name.lstrip('[').rstrip(']')) except: return -1 - + def get_child_at_index(self,index): if index >= self.size: - return None + return None try: elementType = self.buffer.type.GetPointeeType() offset = elementType.size * index From 0e2a0f48f1bf75b6cc1e91a0999fe8f0bcc2f1ea Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 16 Jun 2020 19:20:46 -0300 Subject: [PATCH 126/263] Restore debug level information in specs to fix 32 bits builds (#9466) * Forward flags in compile_and_run_file collect memory within ensure block fails when --debug is used on 32bits * Restore compiler flags used in specs as before #9294 Something pending is that now the CRYSTAL_SPEC_COMPILER_FLAGS are not used since program_flags_options is not called. * Avoid emitting full debug info on call_stack_spec --- spec/std/spec/hooks_spec.cr | 2 +- spec/std/spec_helper.cr | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/std/spec/hooks_spec.cr b/spec/std/spec/hooks_spec.cr index 7a7caf8a94c5..105f5bd9db0e 100644 --- a/spec/std/spec/hooks_spec.cr +++ b/spec/std/spec/hooks_spec.cr @@ -3,7 +3,7 @@ require "./spec_helper" describe Spec do describe "hooks" do it "runs in correct order" do - compile_and_run_source(<<-CR)[1].lines[..-5].should eq <<-OUT.lines + compile_and_run_source(<<-CR, flags: %w(--no-debug))[1].lines[..-5].should eq <<-OUT.lines require "prelude" require "spec" diff --git a/spec/std/spec_helper.cr b/spec/std/spec_helper.cr index b87590327247..c06dd485768b 100644 --- a/spec/std/spec_helper.cr +++ b/spec/std/spec_helper.cr @@ -75,7 +75,7 @@ def spawn_and_check(before : Proc(_), file = __FILE__, line = __LINE__, &block : end end -def compile_file(source_file, flags = %w(--debug), file = __FILE__) +def compile_file(source_file, flags = %w(), file = __FILE__) with_temp_executable("executable_file", file: file) do |executable_file| Process.run("bin/crystal", ["build"] + flags + ["-o", executable_file, source_file]) File.exists?(executable_file).should be_true @@ -84,7 +84,7 @@ def compile_file(source_file, flags = %w(--debug), file = __FILE__) end end -def compile_source(source, flags = %w(--debug), file = __FILE__) +def compile_source(source, flags = %w(), file = __FILE__) with_tempfile("source_file", file: file) do |source_file| File.write(source_file, source) compile_file(source_file, flags, file: file) do |executable_file| @@ -93,8 +93,8 @@ def compile_source(source, flags = %w(--debug), file = __FILE__) end end -def compile_and_run_file(source_file, flags = %w(--debug), file = __FILE__) - compile_file(source_file, file: file) do |executable_file| +def compile_and_run_file(source_file, flags = %w(), file = __FILE__) + compile_file(source_file, flags, file: file) do |executable_file| output, error = IO::Memory.new, IO::Memory.new status = Process.run executable_file, output: output, error: error @@ -102,7 +102,7 @@ def compile_and_run_file(source_file, flags = %w(--debug), file = __FILE__) end end -def compile_and_run_source(source, flags = %w(--debug), file = __FILE__) +def compile_and_run_source(source, flags = %w(), file = __FILE__) with_tempfile("source_file", file: file) do |source_file| File.write(source_file, source) compile_and_run_file(source_file, flags, file: file) From c41640fc205840a544f23eda33a6fedb4a184088 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 16 Jun 2020 19:21:16 -0300 Subject: [PATCH 127/263] Revert IO#write changes in 0.35.0 and let it return Nil (#9469) * Revert "Change IO#write, IO#skip, IO.copy to return Int64 (#9363)" This reverts commit 6380fa94f9798f440fe92213d7fb6870696063d9. * Revert "Make IO#skip IO#write returns the number of bytes it skipped/written (#9233)" This reverts commit 7f132502126adcc9f3a432bef336ca62a788fda7. * Keep IO::Stabled#skip_to_end * Keep IO#copy returning Int64 * Code cleanup --- spec/std/http/chunked_content_spec.cr | 2 +- spec/std/http/request_spec.cr | 3 +- spec/std/http/server/response_spec.cr | 3 +- spec/std/io/buffered_spec.cr | 4 +- spec/std/io/io_spec.cr | 41 +++----------------- spec/std/io/memory_spec.cr | 6 +-- spec/std/io/sized_spec.cr | 5 +-- spec/std/io/stapled_spec.cr | 4 +- spec/support/io.cr | 3 +- src/compress/deflate/writer.cr | 6 +-- src/compress/gzip/writer.cr | 6 +-- src/compress/zip/checksum_writer.cr | 4 +- src/compress/zlib/writer.cr | 6 +-- src/float.cr | 2 +- src/http/content.cr | 10 ++--- src/http/server/response.cr | 4 +- src/http/web_socket/protocol.cr | 6 +-- src/int.cr | 2 +- src/io.cr | 36 ++++++++--------- src/io/buffered.cr | 29 +++++--------- src/io/byte_format.cr | 56 +++++++++++++-------------- src/io/delimited.cr | 2 +- src/io/encoding.cr | 6 +-- src/io/hexdump.cr | 4 +- src/io/memory.cr | 18 +++------ src/io/multi_writer.cr | 6 +-- src/io/sized.cr | 5 +-- src/io/stapled.cr | 10 ++--- src/openssl/digest/digest_io.cr | 4 +- src/string/builder.cr | 10 ++--- 30 files changed, 112 insertions(+), 191 deletions(-) diff --git a/spec/std/http/chunked_content_spec.cr b/spec/std/http/chunked_content_spec.cr index c97cef02ca1f..2a4e43a3827c 100644 --- a/spec/std/http/chunked_content_spec.cr +++ b/spec/std/http/chunked_content_spec.cr @@ -34,7 +34,7 @@ describe HTTP::ChunkedContent do mem = IO::Memory.new("4\r\n123\n\r\n0\r\n\r\n") content = HTTP::ChunkedContent.new(mem) - content.skip(2).should eq(2) + content.skip(2) content.read_char.should eq('3') expect_raises(IO::EOFError) do diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index 7ae9aa34957e..e06a4c193dc4 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -6,8 +6,7 @@ private class EmptyIO < IO 0 end - def write(slice : Bytes) : Int64 - slice.size.to_i64 + def write(slice : Bytes) : Nil end end diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index b42e537d317e..33bd0aa70ada 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -12,11 +12,10 @@ private class ReverseResponseOutput < IO def initialize(@output : IO) end - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil slice.reverse_each do |byte| @output.write_byte(byte) end - slice.size.to_i64 end def read(slice : Bytes) diff --git a/spec/std/io/buffered_spec.cr b/spec/std/io/buffered_spec.cr index 7c3514cbaaa4..a2d615fa6ccd 100644 --- a/spec/std/io/buffered_spec.cr +++ b/spec/std/io/buffered_spec.cr @@ -429,14 +429,14 @@ describe "IO::Buffered" do it "skips" do str = IO::Memory.new("123456789") io = BufferedWrapper.new(str) - io.skip(3).should eq(3) + io.skip(3) io.read_char.should eq('4') end it "skips big" do str = IO::Memory.new(("a" * 10_000) + "b") io = BufferedWrapper.new(str) - io.skip(10_000).should eq(10_000) + io.skip(10_000) io.read_char.should eq('b') end diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index b09c096e83ea..3b5bf8d175e2 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -46,7 +46,7 @@ private class SimpleIOMemory < IO count end - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil count = slice.size new_bytesize = bytesize + count if new_bytesize > @capacity @@ -55,8 +55,6 @@ private class SimpleIOMemory < IO slice.copy_to(@buffer + @bytesize, count) @bytesize += count - - slice.size.to_i64 end def to_slice @@ -99,8 +97,7 @@ private class OneByOneIO < IO 1 end - def write(slice : Bytes) : Int64 - slice.size.to_i64 + def write(slice : Bytes) : Nil end end @@ -508,7 +505,7 @@ describe IO do it "skips a few bytes" do io = SimpleIOMemory.new io << "hello world" - io.skip(6).should eq(6) + io.skip(6) io.gets_to_end.should eq("world") end @@ -523,14 +520,14 @@ describe IO do it "skips more than 4096 bytes" do io = SimpleIOMemory.new io << "a" * 4100 - io.skip(4099).should eq(4099) + io.skip(4099) io.gets_to_end.should eq("a") end it "skips to end" do io = SimpleIOMemory.new io << "hello" - io.skip_to_end.should eq(5) + io.skip_to_end io.read_byte.should be_nil end @@ -544,34 +541,6 @@ describe IO do end end end - - describe "counts written bytes" do - it "directly" do - with_tempfile("create.txt") do |path| - File.open(path, "w") do |io| - io.write("hello world".to_slice).should eq(11) - io.write_utf8("mañana".to_slice).should eq(7) - end - end - end - - pending_win32 "with encoding" do - with_tempfile("create.txt") do |path| - File.open(path, "w", File::DEFAULT_CREATE_PERMISSIONS, "CP1252") do |io| - # In UTF-8 ñ will use 2 bytes - io.write_utf8("mañana".to_slice).should eq(6) - end - end - end - - it "with byte format" do - io = SimpleIOMemory.new - - io.write_bytes(1u64).should eq(8) - io.write_bytes(1u32).should eq(4) - io.write_bytes(1u8).should eq(1) - end - end end pending_win32 describe: "encoding" do diff --git a/spec/std/io/memory_spec.cr b/spec/std/io/memory_spec.cr index 1aca014e7eef..2e19e5caec17 100644 --- a/spec/std/io/memory_spec.cr +++ b/spec/std/io/memory_spec.cr @@ -353,11 +353,11 @@ describe IO::Memory do it "skips" do io = IO::Memory.new("hello") - io.skip(2).should eq(2) + io.skip(2) io.gets_to_end.should eq("llo") io.rewind - io.skip(5).should eq(5) + io.skip(5) io.gets_to_end.should eq("") io.rewind @@ -369,7 +369,7 @@ describe IO::Memory do it "skips_to_end" do io = IO::Memory.new("hello") - io.skip_to_end.should eq(5) + io.skip_to_end io.gets_to_end.should eq("") end diff --git a/spec/std/io/sized_spec.cr b/spec/std/io/sized_spec.cr index a02eafd1556f..8338063ff9db 100644 --- a/spec/std/io/sized_spec.cr +++ b/spec/std/io/sized_spec.cr @@ -5,8 +5,7 @@ private class NoPeekIO < IO 0 end - def write(bytes : Bytes) : Int64 - 0i64 + def write(bytes : Bytes) : Nil end def peek @@ -139,7 +138,7 @@ describe "IO::Sized" do it "skips" do io = IO::Memory.new "123456789" sized = IO::Sized.new(io, read_size: 6) - sized.skip(3).should eq(3) + sized.skip(3) sized.read_char.should eq('4') expect_raises(IO::EOFError) do diff --git a/spec/std/io/stapled_spec.cr b/spec/std/io/stapled_spec.cr index 261dfcea11bb..6c6a315119f7 100644 --- a/spec/std/io/stapled_spec.cr +++ b/spec/std/io/stapled_spec.cr @@ -80,7 +80,7 @@ describe IO::Stapled do reader = IO::Memory.new "cletus" io = IO::Stapled.new reader, IO::Memory.new io.peek.should eq "cletus".to_slice - io.skip(4).should eq(4) + io.skip(4) io.peek.should eq "us".to_slice end @@ -88,7 +88,7 @@ describe IO::Stapled do reader = IO::Memory.new "cletus" io = IO::Stapled.new reader, IO::Memory.new io.peek.should eq "cletus".to_slice - io.skip_to_end.should eq(6) + io.skip_to_end io.peek.should eq Bytes.empty end diff --git a/spec/support/io.cr b/spec/support/io.cr index 9ce2cd44c944..07d3afa8624a 100644 --- a/spec/support/io.cr +++ b/spec/support/io.cr @@ -8,10 +8,9 @@ class RaiseIOError < IO raise IO::Error.new("...") end - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil @writes += 1 raise IO::Error.new("...") if @raise_on_write - slice.size.to_i64 end def flush diff --git a/src/compress/deflate/writer.cr b/src/compress/deflate/writer.cr index 22b02905ae22..c09b6da2da76 100644 --- a/src/compress/deflate/writer.cr +++ b/src/compress/deflate/writer.cr @@ -43,16 +43,14 @@ class Compress::Deflate::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? @stream.avail_in = slice.size @stream.next_in = slice consume_output LibZ::Flush::NO_FLUSH - - slice.size.to_i64 end # See `IO#flush`. diff --git a/src/compress/gzip/writer.cr b/src/compress/gzip/writer.cr index 19ba228c1fb3..f6ebb8a58d5a 100644 --- a/src/compress/gzip/writer.cr +++ b/src/compress/gzip/writer.cr @@ -68,10 +68,10 @@ class Compress::Gzip::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? flate_io = write_header flate_io.write(slice) @@ -82,8 +82,6 @@ class Compress::Gzip::Writer < IO # Using wrapping addition here because isize is only 32 bits wide but # uncompressed data size can be bigger. @isize &+= slice.size - - slice.size.to_i64 end # Flushes data, forcing writing the gzip header if no diff --git a/src/compress/zip/checksum_writer.cr b/src/compress/zip/checksum_writer.cr index 194e96debe9e..77853958a53d 100644 --- a/src/compress/zip/checksum_writer.cr +++ b/src/compress/zip/checksum_writer.cr @@ -13,8 +13,8 @@ module Compress::Zip raise IO::Error.new "Can't read from Zip::Writer entry" end - def write(slice : Bytes) : Int64 - return 0i64 if slice.empty? + def write(slice : Bytes) : Nil + return if slice.empty? @count += slice.size @crc32 = Digest::CRC32.update(slice, @crc32) if @compute_crc32 diff --git a/src/compress/zlib/writer.cr b/src/compress/zlib/writer.cr index f66605d0fbb6..455ac8df6395 100644 --- a/src/compress/zlib/writer.cr +++ b/src/compress/zlib/writer.cr @@ -44,17 +44,15 @@ class Compress::Zlib::Writer < IO end # See `IO#write`. - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? write_header unless @wrote_header @flate_io.write(slice) @adler32 = Digest::Adler32.update(slice, @adler32) - - slice.size.to_i64 end # Flushes data, forcing writing the zlib header if no diff --git a/src/float.cr b/src/float.cr index 1303ee44888d..bc4fdf0c2540 100644 --- a/src/float.cr +++ b/src/float.cr @@ -93,7 +93,7 @@ struct Float # Writes this float to the given *io* in the given *format*. # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) : Int64 + def to_io(io : IO, format : IO::ByteFormat) format.encode(self, io) end diff --git a/src/http/content.cr b/src/http/content.cr index 1719a0c4c0c0..e7dd653845c2 100644 --- a/src/http/content.cr +++ b/src/http/content.cr @@ -41,7 +41,7 @@ module HTTP super end - def skip(bytes_count : Int) : Int64 + def skip(bytes_count) ensure_send_continue super end @@ -73,7 +73,7 @@ module HTTP @io.peek end - def skip(bytes_count : Int) : Int64 + def skip(bytes_count) ensure_send_continue @io.skip(bytes_count) end @@ -164,18 +164,14 @@ module HTTP peek end - def skip(bytes_count : Int) : Int64 - bytes_count = bytes_count.to_i64 + def skip(bytes_count) ensure_send_continue - if bytes_count <= @chunk_remaining @io.skip(bytes_count) @chunk_remaining -= bytes_count else super end - - bytes_count end # Checks if the last read consumed a chunk and we diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 78f273e16c53..c0e0a18d3b42 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -80,8 +80,8 @@ class HTTP::Server end # See `IO#write(slice)`. - def write(slice : Bytes) : Int64 - return 0i64 if slice.empty? + def write(slice : Bytes) : Nil + return if slice.empty? @output.write(slice) end diff --git a/src/http/web_socket/protocol.cr b/src/http/web_socket/protocol.cr index 9f7a1f300c59..e9cfacfd5646 100644 --- a/src/http/web_socket/protocol.cr +++ b/src/http/web_socket/protocol.cr @@ -52,8 +52,8 @@ class HTTP::WebSocket::Protocol @pos = 0 end - def write(slice : Bytes) : Int64 - return 0i64 if slice.empty? + def write(slice : Bytes) : Nil + return if slice.empty? count = Math.min(@buffer.size - @pos, slice.size) (@buffer + @pos).copy_from(slice.to_unsafe, count) @@ -66,8 +66,6 @@ class HTTP::WebSocket::Protocol if count < slice.size write(slice + count) end - - slice.size.to_i64 end def read(slice : Bytes) diff --git a/src/int.cr b/src/int.cr index 27d8b75bf4f3..2bdb2dbeaf2d 100644 --- a/src/int.cr +++ b/src/int.cr @@ -673,7 +673,7 @@ struct Int # Writes this integer to the given *io* in the given *format*. # # See also: `IO#write_bytes`. - def to_io(io : IO, format : IO::ByteFormat) : Int64 + def to_io(io : IO, format : IO::ByteFormat) format.encode(self, io) end diff --git a/src/io.cr b/src/io.cr index f7b5d687bb7a..96e036e24908 100644 --- a/src/io.cr +++ b/src/io.cr @@ -14,7 +14,7 @@ require "c/errno" # these two methods: # # * `read(slice : Bytes)`: read at most *slice.size* bytes from IO into *slice* and return the number of bytes read -# * `write(slice : Bytes)`: write the whole *slice* into the IO and return the number of bytes written +# * `write(slice : Bytes)`: write the whole *slice* into the IO # # For example, this is a simple `IO` on top of a `Bytes`: # @@ -29,10 +29,9 @@ require "c/errno" # slice.size # end # -# def write(slice : Bytes) : Int64 +# def write(slice : Bytes) : Nil # slice.size.times { |i| @slice[i] = slice[i] } # @slice += slice.size -# slice.size.to_i64 # end # end # @@ -100,7 +99,7 @@ abstract class IO # io.write(slice) # io.to_s # => "abcd" # ``` - abstract def write(slice : Bytes) : Int64 + abstract def write(slice : Bytes) : Nil # Closes this `IO`. # @@ -464,12 +463,14 @@ abstract class IO end # Writes a slice of UTF-8 encoded bytes to this `IO`, using the current encoding. - def write_utf8(slice : Bytes) : Int64 + def write_utf8(slice : Bytes) if encoder = encoder() encoder.write(self, slice) else write(slice) end + + nil end private def encoder @@ -812,27 +813,22 @@ abstract class IO # io.gets # => "world" # io.skip(1) # raises IO::EOFError # ``` - def skip(bytes_count : Int) : Int64 - bytes_count = bytes_count.to_i64 - remaining = bytes_count + def skip(bytes_count : Int) : Nil buffer = uninitialized UInt8[4096] - while remaining > 0 - read_count = read(buffer.to_slice[0, Math.min(remaining, 4096)]) + while bytes_count > 0 + read_count = read(buffer.to_slice[0, Math.min(bytes_count, 4096)]) raise IO::EOFError.new if read_count == 0 - remaining -= read_count + + bytes_count -= read_count end - bytes_count end # Reads and discards bytes from `self` until there # are no more bytes. - def skip_to_end : Int64 - bytes_count = 0i64 + def skip_to_end : Nil buffer = uninitialized UInt8[4096] - while (len = read(buffer.to_slice)) > 0 - bytes_count &+= len + while read(buffer.to_slice) > 0 end - bytes_count end # Writes a single byte into this `IO`. @@ -842,7 +838,7 @@ abstract class IO # io.write_byte 97_u8 # io.to_s # => "a" # ``` - def write_byte(byte : UInt8) : Int64 + def write_byte(byte : UInt8) x = byte write Slice.new(pointerof(x), 1) end @@ -850,7 +846,7 @@ abstract class IO # Writes the given object to this `IO` using the specified *format*. # # This ends up invoking `object.to_io(self, format)`, so any object defining a - # `to_io(io : IO, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : UInt64` + # `to_io(io : IO, format : IO::ByteFormat = IO::ByteFormat::SystemEndian)` # method can be written in this way. # # See `Int#to_io` and `Float#to_io`. @@ -861,7 +857,7 @@ abstract class IO # io.rewind # io.gets(4) # => "\u{4}\u{3}\u{2}\u{1}" # ``` - def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) : Int64 + def write_bytes(object, format : IO::ByteFormat = IO::ByteFormat::SystemEndian) object.to_io(self, format) end diff --git a/src/io/buffered.cr b/src/io/buffered.cr index 008fb466d9dc..be8275db3f9a 100644 --- a/src/io/buffered.cr +++ b/src/io/buffered.cr @@ -110,36 +110,30 @@ module IO::Buffered end # :nodoc: - def skip(bytes_count : Int) : Int64 - bytes_count = bytes_count.to_i64 + def skip(bytes_count) : Nil check_open if bytes_count <= @in_buffer_rem.size @in_buffer_rem += bytes_count - return bytes_count + return end - remaining = bytes_count - remaining -= @in_buffer_rem.size + bytes_count -= @in_buffer_rem.size @in_buffer_rem = Bytes.empty - super(remaining) - bytes_count + super(bytes_count) end # Buffered implementation of `IO#write(slice)`. - def write(slice : Bytes) : Int64 - # NOTE: It returns the bytes written without differencing whether - # they are kept in the buffer or sent to the underlying IO. + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? count = slice.size if sync? - unbuffered_write(slice) - return slice.size.to_i64 + return unbuffered_write(slice) end if flush_on_newline? @@ -155,8 +149,7 @@ module IO::Buffered if count >= @buffer_size flush - unbuffered_write slice[0, count] - return slice.size.to_i64 + return unbuffered_write slice[0, count] end if count > @buffer_size - @out_count @@ -165,12 +158,10 @@ module IO::Buffered slice.copy_to(out_buffer + @out_count, count) @out_count += count - - slice.size.to_i64 end # :nodoc: - def write_byte(byte : UInt8) : Int64 + def write_byte(byte : UInt8) check_open if sync? @@ -186,8 +177,6 @@ module IO::Buffered if flush_on_newline? && byte === '\n' flush end - - 1i64 end # Turns on/off `IO` **write** buffering. When *sync* is set to `true`, no buffering diff --git a/src/io/byte_format.cr b/src/io/byte_format.cr index a8d2d8c5a94d..6f6b6a1082cd 100644 --- a/src/io/byte_format.cr +++ b/src/io/byte_format.cr @@ -33,27 +33,27 @@ # io.to_slice # => Bytes[0x34, 0x12] # ``` module IO::ByteFormat - abstract def encode(int : Int8, io : IO) : Int64 - abstract def encode(int : UInt8, io : IO) : Int64 - abstract def encode(int : Int16, io : IO) : Int64 - abstract def encode(int : UInt16, io : IO) : Int64 - abstract def encode(int : Int32, io : IO) : Int64 - abstract def encode(int : UInt32, io : IO) : Int64 - abstract def encode(int : Int64, io : IO) : Int64 - abstract def encode(int : UInt64, io : IO) : Int64 - abstract def encode(int : Int128, io : IO) : Int64 - abstract def encode(int : UInt128, io : IO) : Int64 - - abstract def encode(int : Int8, bytes : Bytes) : Int64 - abstract def encode(int : UInt8, bytes : Bytes) : Int64 - abstract def encode(int : Int16, bytes : Bytes) : Int64 - abstract def encode(int : UInt16, bytes : Bytes) : Int64 - abstract def encode(int : Int32, bytes : Bytes) : Int64 - abstract def encode(int : UInt32, bytes : Bytes) : Int64 - abstract def encode(int : Int64, bytes : Bytes) : Int64 - abstract def encode(int : UInt64, bytes : Bytes) : Int64 - abstract def encode(int : Int128, bytes : Bytes) : Int64 - abstract def encode(int : UInt128, bytes : Bytes) : Int64 + abstract def encode(int : Int8, io : IO) + abstract def encode(int : UInt8, io : IO) + abstract def encode(int : Int16, io : IO) + abstract def encode(int : UInt16, io : IO) + abstract def encode(int : Int32, io : IO) + abstract def encode(int : UInt32, io : IO) + abstract def encode(int : Int64, io : IO) + abstract def encode(int : UInt64, io : IO) + abstract def encode(int : Int128, io : IO) + abstract def encode(int : UInt128, io : IO) + + abstract def encode(int : Int8, bytes : Bytes) + abstract def encode(int : UInt8, bytes : Bytes) + abstract def encode(int : Int16, bytes : Bytes) + abstract def encode(int : UInt16, bytes : Bytes) + abstract def encode(int : Int32, bytes : Bytes) + abstract def encode(int : UInt32, bytes : Bytes) + abstract def encode(int : Int64, bytes : Bytes) + abstract def encode(int : UInt64, bytes : Bytes) + abstract def encode(int : Int128, bytes : Bytes) + abstract def encode(int : UInt128, bytes : Bytes) abstract def decode(int : Int8.class, io : IO) abstract def decode(int : UInt8.class, io : IO) @@ -77,11 +77,11 @@ module IO::ByteFormat abstract def decode(int : Int128.class, bytes : Bytes) abstract def decode(int : UInt128.class, bytes : Bytes) - def encode(float : Float32, io : IO) : Int64 + def encode(float : Float32, io : IO) encode(float.unsafe_as(Int32), io) end - def encode(float : Float32, bytes : Bytes) : Int64 + def encode(float : Float32, bytes : Bytes) encode(float.unsafe_as(Int32), bytes) end @@ -93,11 +93,11 @@ module IO::ByteFormat decode(Int32, bytes).unsafe_as(Float32) end - def encode(float : Float64, io : IO) : Int64 + def encode(float : Float64, io : IO) encode(float.unsafe_as(Int64), io) end - def encode(float : Float64, bytes : Bytes) : Int64 + def encode(float : Float64, bytes : Bytes) encode(float.unsafe_as(Int64), bytes) end @@ -125,18 +125,16 @@ module IO::ByteFormat {% for type, i in %w(Int8 UInt8 Int16 UInt16 Int32 UInt32 Int64 UInt64 Int128 UInt128) %} {% bytesize = 2 ** (i // 2) %} - def self.encode(int : {{type.id}}, io : IO) : Int64 + def self.encode(int : {{type.id}}, io : IO) buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self io.write(buffer.to_slice) - Int64.new({{bytesize}}) end - def self.encode(int : {{type.id}}, bytes : Bytes) : Int64 + def self.encode(int : {{type.id}}, bytes : Bytes) buffer = int.unsafe_as(StaticArray(UInt8, {{bytesize}})) buffer.reverse! unless SystemEndian == self buffer.to_slice.copy_to(bytes) - Int64.new({{bytesize}}) end def self.decode(type : {{type.id}}.class, io : IO) diff --git a/src/io/delimited.cr b/src/io/delimited.cr index 34d23c07066a..2142391502a0 100644 --- a/src/io/delimited.cr +++ b/src/io/delimited.cr @@ -107,7 +107,7 @@ class IO::Delimited < IO read_bytes end - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil raise IO::Error.new "Can't write to IO::Delimited" end diff --git a/src/io/encoding.cr b/src/io/encoding.cr index aab1c720a0db..174029fbb31c 100644 --- a/src/io/encoding.cr +++ b/src/io/encoding.cr @@ -27,8 +27,7 @@ class IO @closed = false end - def write(io, slice : Bytes) : Int64 - bytes_written = 0i64 + def write(io, slice : Bytes) inbuf_ptr = slice.to_unsafe inbytesleft = LibC::SizeT.new(slice.size) outbuf = uninitialized UInt8[1024] @@ -39,9 +38,8 @@ class IO if err == Crystal::Iconv::ERROR @iconv.handle_invalid(pointerof(inbuf_ptr), pointerof(inbytesleft)) end - bytes_written &+= io.write(outbuf.to_slice[0, outbuf.size - outbytesleft]) + io.write(outbuf.to_slice[0, outbuf.size - outbytesleft]) end - bytes_written end def close diff --git a/src/io/hexdump.cr b/src/io/hexdump.cr index cec29b9a0af0..3df681f62ef8 100644 --- a/src/io/hexdump.cr +++ b/src/io/hexdump.cr @@ -32,8 +32,8 @@ class IO::Hexdump < IO end end - def write(buf : Bytes) : Int64 - return 0i64 if buf.empty? + def write(buf : Bytes) : Nil + return if buf.empty? @io.write(buf).tap do @output.puts buf.hexdump if @write diff --git a/src/io/memory.cr b/src/io/memory.cr index be97cfe1c64a..e447c9f5da93 100644 --- a/src/io/memory.cr +++ b/src/io/memory.cr @@ -82,13 +82,13 @@ class IO::Memory < IO # See `IO#write(slice)`. Raises if this `IO::Memory` is non-writeable, # or if it's non-resizeable and a resize is needed. - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_writeable check_open count = slice.size - return 0i64 if count == 0 + return if count == 0 new_bytesize = @pos + count if new_bytesize > @capacity @@ -104,13 +104,11 @@ class IO::Memory < IO @pos += count @bytesize = @pos if @pos > @bytesize - - slice.size.to_i64 end # See `IO#write_byte`. Raises if this `IO::Memory` is non-writeable, # or if it's non-resizeable and a resize is needed. - def write_byte(byte : UInt8) : Int64 + def write_byte(byte : UInt8) check_writeable check_open @@ -129,7 +127,7 @@ class IO::Memory < IO @pos += 1 @bytesize = @pos if @pos > @bytesize - 1i64 + nil end # :nodoc: @@ -194,8 +192,7 @@ class IO::Memory < IO end # :nodoc: - def skip(bytes_count : Int) : Int64 - bytes_count = bytes_count.to_i64 + def skip(bytes_count) check_open available = @bytesize - @pos @@ -204,16 +201,13 @@ class IO::Memory < IO else raise IO::EOFError.new end - bytes_count end # :nodoc: - def skip_to_end : Int64 + def skip_to_end : Nil check_open - skipped = @bytesize - @pos @pos = @bytesize - skipped.to_i64 end # :nodoc: diff --git a/src/io/multi_writer.cr b/src/io/multi_writer.cr index 7ddf7f42b4a4..c06bc8649ed9 100644 --- a/src/io/multi_writer.cr +++ b/src/io/multi_writer.cr @@ -29,14 +29,12 @@ class IO::MultiWriter < IO @writers = writers.map(&.as(IO)).to_a end - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? @writers.each { |writer| writer.write(slice) } - - slice.size.to_i64 end def read(slice : Bytes) diff --git a/src/io/sized.cr b/src/io/sized.cr index d5ae6e506cf3..253df834a743 100644 --- a/src/io/sized.cr +++ b/src/io/sized.cr @@ -61,8 +61,7 @@ class IO::Sized < IO peek end - def skip(bytes_count : Int) : Int64 - bytes_count = bytes_count.to_i64 + def skip(bytes_count) : Nil check_open if bytes_count <= @read_remaining @@ -71,8 +70,6 @@ class IO::Sized < IO else raise IO::EOFError.new end - - bytes_count end def write(slice : Bytes) : NoReturn diff --git a/src/io/stapled.cr b/src/io/stapled.cr index 1d30bdc2fe98..3fc6e2f92d7d 100644 --- a/src/io/stapled.cr +++ b/src/io/stapled.cr @@ -51,31 +51,31 @@ class IO::Stapled < IO end # Skips `reader`. - def skip(bytes_count : Int) : Int64 + def skip(bytes_count : Int) : Nil check_open @reader.skip(bytes_count) end # Skips `reader`. - def skip_to_end : Int64 + def skip_to_end : Nil check_open @reader.skip_to_end end # Writes a byte to `writer`. - def write_byte(byte : UInt8) : Int64 + def write_byte(byte : UInt8) check_open @writer.write_byte(byte) end # Writes a slice to `writer`. - def write(slice : Bytes) : Int64 + def write(slice : Bytes) : Nil check_open - return 0i64 if slice.empty? + return if slice.empty? @writer.write(slice) end diff --git a/src/openssl/digest/digest_io.cr b/src/openssl/digest/digest_io.cr index b04942736f1f..4093725fefcc 100644 --- a/src/openssl/digest/digest_io.cr +++ b/src/openssl/digest/digest_io.cr @@ -42,8 +42,8 @@ module OpenSSL read_bytes end - def write(slice : Bytes) : Int64 - return 0i64 if slice.empty? + def write(slice : Bytes) : Nil + return if slice.empty? if @mode.write? digest_algorithm.update(slice) diff --git a/src/string/builder.cr b/src/string/builder.cr index 7ba67dd2e029..eb187535420a 100644 --- a/src/string/builder.cr +++ b/src/string/builder.cr @@ -38,8 +38,8 @@ class String::Builder < IO raise "Not implemented" end - def write(slice : Bytes) : Int64 - return 0i64 if slice.empty? + def write(slice : Bytes) : Nil + return if slice.empty? count = slice.size new_bytesize = real_bytesize + count @@ -49,11 +49,9 @@ class String::Builder < IO slice.copy_to(@buffer + real_bytesize, count) @bytesize += count - - slice.size.to_i64 end - def write_byte(byte : UInt8) : Int64 + def write_byte(byte : UInt8) new_bytesize = real_bytesize + 1 if new_bytesize > @capacity resize_to_capacity(Math.pw2ceil(new_bytesize)) @@ -63,7 +61,7 @@ class String::Builder < IO @bytesize += 1 - 1i64 + nil end def buffer From 9dffad33f647385847a2163d0006953e3b2d72b4 Mon Sep 17 00:00:00 2001 From: George Dietrich Date: Wed, 17 Jun 2020 17:07:44 -0400 Subject: [PATCH 128/263] Wrap request handler in `Log.with_context` (#9494) * Wrap request handler in `Log.with_context` * Ensure exceptions are logged within the same context as handler --- .../std/http/server/request_processor_spec.cr | 33 +++++++++++++++++-- src/http/server/request_processor.cr | 2 +- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/spec/std/http/server/request_processor_spec.cr b/spec/std/http/server/request_processor_spec.cr index 72c2f6c3f4a2..089ef70525e0 100644 --- a/spec/std/http/server/request_processor_spec.cr +++ b/spec/std/http/server/request_processor_spec.cr @@ -268,9 +268,9 @@ describe HTTP::Server::RequestProcessor do logs.entry.exception.should be_a(IO::Error) end - it "catches raised error on handler" do + it "catches raised error on handler and retains context from handler" do exception = Exception.new "OH NO" - processor = HTTP::Server::RequestProcessor.new { raise exception } + processor = HTTP::Server::RequestProcessor.new { Log.context.set foo: "bar"; raise exception } input = IO::Memory.new("GET / HTTP/1.1\r\n\r\n") output = IO::Memory.new logs = Log.capture("http.server") do @@ -286,6 +286,7 @@ describe HTTP::Server::RequestProcessor do logs.check(:error, "Unhandled exception on HTTP::Handler") logs.entry.exception.should eq(exception) + logs.entry.context[:foo].should eq "bar" end it "doesn't respond with error when headers were already sent" do @@ -320,4 +321,32 @@ describe HTTP::Server::RequestProcessor do client_response.status_code.should eq(200) client_response.body.should eq("Hello world") end + + it "does not bleed Log::Context between requests" do + processor = HTTP::Server::RequestProcessor.new do |context| + Log.info { "before" } + Log.context.set foo: "bar" + Log.info { "after" } + + context.response.content_type = "text/plain" + context.response.print "Hello world" + end + + logs = Log.capture do + processor.process( + IO::Memory.new("GET / HTTP/1.1\r\n\r\nGET / HTTP/1.1\r\n\r\n"), + IO::Memory.new, + ) + end + + logs.check :info, "before" + logs.entry.context.should be_empty + logs.check :info, "after" + logs.entry.context[:foo].should eq "bar" + + logs.check :info, "before" + logs.entry.context.should be_empty + logs.check :info, "after" + logs.entry.context[:foo].should eq "bar" + end end diff --git a/src/http/server/request_processor.cr b/src/http/server/request_processor.cr index 632d46eb0a38..1e98ad4bfb96 100644 --- a/src/http/server/request_processor.cr +++ b/src/http/server/request_processor.cr @@ -46,7 +46,7 @@ class HTTP::Server::RequestProcessor response.headers["Connection"] = "keep-alive" if request.keep_alive? context = Context.new(request, response) - begin + Log.with_context do @handler.call(context) rescue ex : ClientError Log.debug(exception: ex.cause) { ex.message } From 01dafa33190a8ffacf4a6c6935e013c31f1346fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 18 Jun 2020 17:32:13 +0200 Subject: [PATCH 129/263] Fix Digest::Base block argument type restrictions (#9500) --- spec/std/digest/md5_spec.cr | 3 +++ spec/std/digest/sha1_spec.cr | 3 +++ spec/std/digest/spec_helper.cr | 47 ++++++++++++++++++++++++++++++++++ src/digest/base.cr | 6 ++--- 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 spec/std/digest/spec_helper.cr diff --git a/spec/std/digest/md5_spec.cr b/spec/std/digest/md5_spec.cr index ac4cba08dc1c..7421f4625470 100644 --- a/spec/std/digest/md5_spec.cr +++ b/spec/std/digest/md5_spec.cr @@ -1,7 +1,10 @@ require "spec" +require "./spec_helper" require "digest/md5" describe Digest::MD5 do + it_acts_as_digest_algorithm Digest::MD5 + it "calculates digest from string" do Digest::MD5.digest("foo").to_slice.should eq Bytes[0xac, 0xbd, 0x18, 0xdb, 0x4c, 0xc2, 0xf8, 0x5c, 0xed, 0xef, 0x65, 0x4f, 0xcc, 0xc4, 0xa4, 0xd8] end diff --git a/spec/std/digest/sha1_spec.cr b/spec/std/digest/sha1_spec.cr index 484af50e2ec3..6f793c2cbde3 100644 --- a/spec/std/digest/sha1_spec.cr +++ b/spec/std/digest/sha1_spec.cr @@ -1,7 +1,10 @@ require "spec" +require "./spec_helper" require "digest/sha1" describe Digest::SHA1 do + it_acts_as_digest_algorithm Digest::SHA1 + [ {"", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "2jmj7l5rSw0yVb/vlWAYkK/YBwk="}, {"The quick brown fox jumps over the lazy dog", "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", "L9ThxnotKPzthJ7hu3bnORuT6xI="}, diff --git a/spec/std/digest/spec_helper.cr b/spec/std/digest/spec_helper.cr new file mode 100644 index 000000000000..d54db274605e --- /dev/null +++ b/spec/std/digest/spec_helper.cr @@ -0,0 +1,47 @@ +def it_acts_as_digest_algorithm(type : T.class) forall T + it "#hexdigest can update within a loop from explicit expr (#9483)" do + i = 0 + type.hexdigest do |digest| + while i < 3 + digest.update("") + i += 1 + end + end + end + + pending "#hexdigest can update within a loop by indirect expr (#9483)" do + algorithm = {} of String => Digest::Base.class + algorithm["me"] = type + i = 0 + algorithm["me"].hexdigest do |digest| + while i < 3 + digest.update("") + i += 1 + end + end + end + + it "context are independent" do + algorithm = type + res = algorithm.hexdigest do |digest| + digest.update("a") + digest.update("b") + end + + inner_res = nil + + outer_res = algorithm.hexdigest do |outer| + outer.update("a") + + inner_res = algorithm.hexdigest do |inner| + inner.update("a") + inner.update("b") + end + + outer.update("b") + end + + outer_res.should eq(res) + inner_res.should eq(res) + end +end diff --git a/src/digest/base.cr b/src/digest/base.cr index 53cfce8c8a8f..dd5e542c8970 100644 --- a/src/digest/base.cr +++ b/src/digest/base.cr @@ -24,7 +24,7 @@ abstract class Digest::Base # end # digest.to_slice.hexstring # => "acbd18db4cc2f85cedef654fccc4a4d8" # ``` - def self.digest(& : Digest::Base -> _) : Bytes + def self.digest(& : self ->) : Bytes context = new yield context context.final @@ -55,7 +55,7 @@ abstract class Digest::Base # end # # => "acbd18db4cc2f85cedef654fccc4a4d8" # ``` - def self.hexdigest(& : Digest::Base -> _) : String + def self.hexdigest(& : self ->) : String hashsum = digest do |ctx| yield ctx end @@ -87,7 +87,7 @@ abstract class Digest::Base # end # # => "C+7Hteo/D9vJXQ3UfzxbwnXaijM=" # ``` - def self.base64digest(& : Digest::Base -> _) : String + def self.base64digest(& : self -> _) : String hashsum = digest do |ctx| yield ctx end From 5999ae29beacf4cfd54e232ca83c1a46b79f26a5 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 19 Jun 2020 13:08:03 -0300 Subject: [PATCH 130/263] Release 0.35.1 (#9503) --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++++ src/VERSION | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a70cbc9dcd64..2be6a19098b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,37 @@ +# 0.35.1 (2020-06-??) + +## Standard library + +### Collections + +- Remove `Hash#each` type restriction to allow working with splats. ([#9456](https://github.com/crystal-lang/crystal/pull/9456), thanks @bcardiff) + +### Networking + +- Revert `IO#write` changes in 0.35.0 and let it return Nil. ([#9469](https://github.com/crystal-lang/crystal/pull/9469), thanks @bcardiff) +- Avoid leaking logging context in HTTP request handlers. ([#9494](https://github.com/crystal-lang/crystal/pull/9494), thanks @Blacksmoke16) + +### Crypto + +- Use less strict cipher compatibility for OpenSSL client context. ([#9459](https://github.com/crystal-lang/crystal/pull/9459), thanks @straight-shoota) +- Fix `Digest::Base` block argument type restrictions. ([#9500](https://github.com/crystal-lang/crystal/pull/9500), thanks @straight-shoota) + +### Logging + +- Fix `Log.context.set` docs for hash based data. ([#9470](https://github.com/crystal-lang/crystal/pull/9470), thanks @bcardiff) + +## Compiler + +- Show warnings even if there are errors. ([#9461](https://github.com/crystal-lang/crystal/pull/9461), thanks @asterite) +- Fix parsing of `{foo: X, typeof: Y}` type. ([#9453](https://github.com/crystal-lang/crystal/pull/9453), thanks @MakeNowJust) +- Fix parsing of proc in hash `of` key type. ([#9458](https://github.com/crystal-lang/crystal/pull/9458), thanks @MakeNowJust) +- Revert debug level information changes in specs to fix 32 bits builds. ([#9466](https://github.com/crystal-lang/crystal/pull/9466), thanks @bcardiff) + +## Others + +- CI improvements and housekeeping. ([#9455](https://github.com/crystal-lang/crystal/pull/9455), thanks @bcardiff) +- Code formatting. ([#9482](https://github.com/crystal-lang/crystal/pull/9482), thanks @MakeNowJust) + # 0.35.0 (2020-06-09) ## Language changes diff --git a/src/VERSION b/src/VERSION index ac40018a669e..731b95d7fc85 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -0.35.1-dev +0.35.1 From b52e1250078ab600e378303c7c1205af52d01592 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Fri, 19 Jun 2020 21:40:06 +0200 Subject: [PATCH 131/263] Win CI: Avoid looking for globally installed libs (#9507) They are not an expected dependency and they're linked differently, causing problems --- .github/workflows/win.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index c518e1885a1c..dc3a04a2856e 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -61,7 +61,7 @@ jobs: if: steps.cache-libs.outputs.cache-hit != 'true' working-directory: ./bdwgc run: | - cmake . -DBUILD_SHARED_LIBS=OFF -Denable_large_config=ON -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + cmake . -DBUILD_SHARED_LIBS=OFF -Denable_large_config=ON -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Download libpcre if: steps.cache-libs.outputs.cache-hit != 'true' @@ -73,7 +73,7 @@ jobs: if: steps.cache-libs.outputs.cache-hit != 'true' working-directory: ./pcre run: | - cmake . -DBUILD_SHARED_LIBS=OFF -DPCRE_SUPPORT_UNICODE_PROPERTIES=ON -DPCRE_SUPPORT_JIT=ON -DPCRE_STATIC_RUNTIME=ON + cmake . -DBUILD_SHARED_LIBS=OFF -DPCRE_SUPPORT_UNICODE_PROPERTIES=ON -DPCRE_SUPPORT_JIT=ON -DPCRE_STATIC_RUNTIME=ON -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Download zlib if: steps.cache-libs.outputs.cache-hit != 'true' @@ -85,7 +85,7 @@ jobs: if: steps.cache-libs.outputs.cache-hit != 'true' working-directory: ./zlib run: | - cmake . -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + cmake . -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Download libyaml if: steps.cache-libs.outputs.cache-hit != 'true' @@ -97,7 +97,7 @@ jobs: if: steps.cache-libs.outputs.cache-hit != 'true' working-directory: ./libyaml run: | - cmake . -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + cmake . -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Download libxml2 if: steps.cache-libs.outputs.cache-hit != 'true' @@ -110,7 +110,7 @@ jobs: if: steps.cache-libs.outputs.cache-hit != 'true' working-directory: ./libxml2 run: | - cmake . -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_TESTS=OFF -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded + cmake . -DBUILD_SHARED_LIBS=OFF -DLIBXML2_WITH_HTTP=OFF -DLIBXML2_WITH_FTP=OFF -DLIBXML2_WITH_TESTS=OFF -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Gather libraries if: steps.cache-libs.outputs.cache-hit != 'true' @@ -139,7 +139,7 @@ jobs: if: steps.cache-llvm.outputs.cache-hit != 'true' working-directory: ./llvm-src run: | - cmake . -Thost=x64 -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_USE_CRT_RELEASE=MT -DBUILD_SHARED_LIBS=OFF + cmake . -Thost=x64 -DLLVM_TARGETS_TO_BUILD="X86" -DLLVM_USE_CRT_RELEASE=MT -DBUILD_SHARED_LIBS=OFF -DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=OFF cmake --build . --config Release - name: Gather LLVM if: steps.cache-llvm.outputs.cache-hit != 'true' From 1b1382cb05e6c0be86d290ac6d5cca1677801d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Sat, 20 Jun 2020 00:00:30 +0200 Subject: [PATCH 132/263] Remove Travis IRC notifications (#9513) --- .travis.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index fffcb0697b41..c368c4123e39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,12 +41,5 @@ env: - secure: Zd/tZVmV2dRMao9z+ky5BywSKuWOF3MiKsZetwd1upZ+uj9qzfbOZMnWFW9dlA+Co4MyYqP/I6ADzRpoKLINUqEIPcAPNYQB1qG79SafrRAvTqcjtEHTn2wXh2ZGu3f1T+SCK0ZD3xx1ML8502ENzXjvq+dEmi4kknqmPudkb6k= notifications: - irc: - channels: - - secure: iarUM4VAMZxdZZQxDrQy+MpxygaFFXJJ1bxvafehnk2vjWbifmnQHZx5z3POaSfp76q+hdqSgBrK/QnGwd8NY70wD+bR0Vb5JQoHwEyHyGbLczHp606SXRl7c/ZMozfOk4bo0X5EPPtie+2mtkjSUB03TI4NYY/5LMFhCw79zsU= - use_notice: true - skip_join: true - template: - - "%{repository_slug}#%{commit} (%{branch} - %{commit_subject}): %{message} %{build_url}" slack: secure: Ng3nTqGWY+9p1pS6yjGqDhmRvdgbIZgTNpMWbO/ngwpCyicmD3jafZkShqqXbULZTJJr3OxIGzi6GHGusT0Ic/Pi9JCM3X3v/xuBruKIR+EnNyPo7IL4ZYAlwnXyJHlCHHDBq0gSHGvGJwsXn6IgZBPRfeIq+CCyQHVPyvc9EHE= From 02a2e6978e53f3bbaf363c32b9b511bbd6619ef2 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Sat, 20 Jun 2020 00:42:57 -0300 Subject: [PATCH 133/263] Update CI to use 0.35.1 (#9512) --- .circleci/config.yml | 4 ++-- .github/workflows/win.yml | 3 +-- CHANGELOG.md | 2 +- bin/ci | 6 +++--- src/VERSION | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 98b5dc397d62..ebd0fbaf9dc0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - run: | git clone https://github.com/crystal-lang/distribution-scripts.git ~/distribution-scripts cd ~/distribution-scripts - git checkout c8495fb1799c395b81923bd6d040fc8e75afbe7e + git checkout cb8f2c51a042609d7c45707ba8369b37d011037f # persist relevant information for build process - run: | cd ~/distribution-scripts @@ -183,7 +183,7 @@ jobs: echo "export CRYSTAL_SHA1=$CIRCLE_SHA1" >> build.env # Which previous version use - export PREVIOUS_CRYSTAL_BASE_URL="https://github.com/crystal-lang/crystal/releases/download/0.34.0/crystal-0.34.0-1" + export PREVIOUS_CRYSTAL_BASE_URL="https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1" echo "export PREVIOUS_CRYSTAL_RELEASE_LINUX64_TARGZ=${PREVIOUS_CRYSTAL_BASE_URL}-linux-x86_64.tar.gz" >> build.env echo "export PREVIOUS_CRYSTAL_RELEASE_LINUX32_TARGZ=${PREVIOUS_CRYSTAL_BASE_URL}-linux-i686.tar.gz" >> build.env echo "export PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ=${PREVIOUS_CRYSTAL_BASE_URL}-darwin-x86_64.tar.gz" >> build.env diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index dc3a04a2856e..571e7624acde 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: linux-job: runs-on: ubuntu-latest - container: crystallang/crystal:0.34.0-build + container: crystallang/crystal:0.35.1-build steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -16,7 +16,6 @@ jobs: - name: Cross-compile Crystal run: | LLVM_TARGETS=X86 bin/crystal build --cross-compile --target x86_64-pc-windows-msvc src/compiler/crystal.cr -Dwithout_playground - mv crystal.o crystal.obj || true # TODO: Remove this after 0.35.0 - name: Upload Crystal object file uses: actions/upload-artifact@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be6a19098b3..621699a07a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 0.35.1 (2020-06-??) +# 0.35.1 (2020-06-19) ## Standard library diff --git a/bin/ci b/bin/ci index f6870d7b5bc7..2c83c1b1b37a 100755 --- a/bin/ci +++ b/bin/ci @@ -108,8 +108,8 @@ format() { prepare_build() { on_linux verify_linux_environment - on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/0.34.0/crystal-0.34.0-1-darwin-x86_64.tar.gz -o ~/crystal.tar.gz - on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-0.34.0-1 crystal;popd' + on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-darwin-x86_64.tar.gz -o ~/crystal.tar.gz + on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-0.35.1-1 crystal;popd' on_osx 'brew unlink python@2 || true' on_osx brew install z3 llvm@10 gmp libevent pcre openssl pkg-config @@ -143,7 +143,7 @@ with_build_env() { on_linux verify_linux_environment - export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:0.34.0}" + export DOCKER_TEST_PREFIX="${DOCKER_TEST_PREFIX:=crystallang/crystal:0.35.1}" case $ARCH in x86_64) diff --git a/src/VERSION b/src/VERSION index 731b95d7fc85..05639a55677c 100644 --- a/src/VERSION +++ b/src/VERSION @@ -1 +1 @@ -0.35.1 +1.0.0-dev From ecea8f9493ddd614dfe7f663191d17f46023af22 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Mon, 22 Jun 2020 22:35:28 +0900 Subject: [PATCH 134/263] Remove unused assignment of `here` (#9468) This commit only contains removing a line to assign to `here`. In fact this `here` is unused in anywhere, then we should remove this. --- src/compiler/crystal/syntax/lexer.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index 40e33127d6c2..c7692ec49f51 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -202,7 +202,6 @@ module Crystal when '=' next_char :"<<=" when '-' - here = IO::Memory.new(20) has_single_quote = false found_closing_single_quote = false From 75ab8db1b841415fe89752b6573279f8ed69a8d8 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 22 Jun 2020 14:16:25 -0300 Subject: [PATCH 135/263] Document how to generate the SNAPCRAFT_TOKEN (#9515) Current token will expire at 2021-06-20 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ebd0fbaf9dc0..d4220e769a67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -434,6 +434,7 @@ jobs: - attach_workspace: at: /tmp/workspace - run: + # $SNAPCRAFT_TOKEN is generated using `snapcraft export-login --snaps crystal --channels edge,edge/* -` command: | cd /tmp/workspace/distribution-scripts source build.env From 3cdb9a93f188e1b1c3ca9ae3acb9726099b935be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 22 Jun 2020 19:22:47 +0200 Subject: [PATCH 136/263] Fix VaList and disable va_arg for AArch64 (#9422) va_arg is broken on most platforms, including AArch64 --- spec/compiler/codegen/primitives_spec.cr | 50 +++++++++---------- spec/spec_helper.cr | 6 +-- spec/std/spec_helper.cr | 11 ++++ spec/std/va_list_spec.cr | 40 +++++++++++++++ spec/support/tempfile.cr | 10 ++++ spec/win32_std_spec.cr | 1 + src/compiler/crystal/semantic/main_visitor.cr | 3 -- src/lib_c/aarch64-linux-gnu/c/stdarg.cr | 9 +++- src/lib_c/aarch64-linux-musl/c/stdarg.cr | 14 +++--- src/va_list.cr | 12 ++++- 10 files changed, 114 insertions(+), 42 deletions(-) create mode 100644 spec/std/va_list_spec.cr diff --git a/spec/compiler/codegen/primitives_spec.cr b/spec/compiler/codegen/primitives_spec.cr index 20e768e66086..7d6fba92ec37 100644 --- a/spec/compiler/codegen/primitives_spec.cr +++ b/spec/compiler/codegen/primitives_spec.cr @@ -239,8 +239,8 @@ describe "Code gen: primitives" do end describe "va_arg" do - # On Windows llvm's va_arg instruction works incorrectly. - {% unless flag?(:win32) %} + # On Windows and AArch64 llvm's va_arg instruction works incorrectly. + {% unless flag?(:win32) || flag?(:aarch64) %} it "uses llvm's va_arg instruction" do mod = codegen(%( struct VaList @@ -255,33 +255,33 @@ describe "Code gen: primitives" do str = mod.to_s str.should contain("va_arg %VaList* %list") end - {% end %} - pending_win32 "works with C code" do - test_c( - %( - extern int foo_f(int,...); - int foo() { - return foo_f(3,1,2,3); - } - ), - %( - lib LibFoo - fun foo() : LibC::Int - end + it "works with C code" do + test_c( + %( + extern int foo_f(int,...); + int foo() { + return foo_f(3,1,2,3); + } + ), + %( + lib LibFoo + fun foo() : LibC::Int + end - fun foo_f(count : Int32, ...) : LibC::Int - sum = 0 - VaList.open do |list| - count.times do |i| - sum += list.next(Int32) + fun foo_f(count : Int32, ...) : LibC::Int + sum = 0 + VaList.open do |list| + count.times do |i| + sum += list.next(Int32) + end end + sum end - sum - end - LibFoo.foo - ), &.to_i.should eq(6)) - end + LibFoo.foo + ), &.to_i.should eq(6)) + end + {% end %} end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 0a67b2f9f802..ee121a3a6038 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -257,11 +257,7 @@ def run(code, filename = nil, inject_primitives = true, debug = Crystal::Debug:: end def test_c(c_code, crystal_code, *, file = __FILE__) - with_tempfile("temp_abi.c", "temp_abi.o", file: file) do |c_filename, o_filename| - File.write(c_filename, c_code) - - `#{Crystal::Compiler::CC} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy - + with_temp_c_object_file(c_code, file: file) do |o_filename| yield run(%( require "prelude" diff --git a/spec/std/spec_helper.cr b/spec/std/spec_helper.cr index c06dd485768b..9b67c2b4b9a7 100644 --- a/spec/std/spec_helper.cr +++ b/spec/std/spec_helper.cr @@ -108,3 +108,14 @@ def compile_and_run_source(source, flags = %w(), file = __FILE__) compile_and_run_file(source_file, flags, file: file) end end + +def compile_and_run_source_with_c(c_code, crystal_code, flags = %w(--debug), file = __FILE__) + with_temp_c_object_file(c_code, file: file) do |o_filename| + yield compile_and_run_source(%( + require "prelude" + + @[Link(ldflags: #{o_filename.inspect})] + #{crystal_code} + )) + end +end diff --git a/spec/std/va_list_spec.cr b/spec/std/va_list_spec.cr new file mode 100644 index 000000000000..79347c0638fe --- /dev/null +++ b/spec/std/va_list_spec.cr @@ -0,0 +1,40 @@ +require "./spec_helper" + +describe VaList do + it "works with C code" do + compile_and_run_source_with_c( + %( + #include + extern int foo_f(int,...); + int foo() { + return foo_f(3,1,2,3); + } + + int read_arg(va_list *ap) { + return va_arg(*ap, int); + } + ), + %( + lib LibFoo + fun foo : LibC::Int + fun read_arg(ap : LibC::VaList*) : LibC::Int + end + + fun foo_f(count : LibC::Int, ...) : LibC::Int + sum = 0 + VaList.open do |list| + ap = list.to_unsafe + count.times do |i| + sum += LibFoo.read_arg(pointerof(ap)) + end + end + sum + end + + puts LibFoo.foo + )) do |status, output| + status.success?.should be_true + output.to_i.should eq(6) + end + end +end diff --git a/spec/support/tempfile.cr b/spec/support/tempfile.cr index 0c47ee765e90..42001785d1c8 100644 --- a/spec/support/tempfile.cr +++ b/spec/support/tempfile.cr @@ -43,6 +43,16 @@ def with_temp_executable(name, file = __FILE__) end end +def with_temp_c_object_file(c_code, file = __FILE__) + with_tempfile("temp_c.c", "temp_c.o", file: file) do |c_filename, o_filename| + File.write(c_filename, c_code) + + `#{ENV["CC"]? || "cc"} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy + + yield o_filename + end +end + if SPEC_TEMPFILE_CLEANUP at_exit do FileUtils.rm_r(SPEC_TEMPFILE_PATH) if Dir.exists?(SPEC_TEMPFILE_PATH) diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index dbf3f13c8a08..2bc3be978063 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -221,6 +221,7 @@ require "./std/uint_spec.cr" require "./std/uri/punycode_spec.cr" require "./std/uri_spec.cr" require "./std/uuid_spec.cr" +# require "./std/va_list_spec.cr" require "./std/weak_ref_spec.cr" require "./std/xml/builder_spec.cr" require "./std/xml/html_spec.cr" diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index a4cff837c390..c6b4ec4009c4 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -2431,9 +2431,6 @@ module Crystal end def visit_va_arg(node) - if program.has_flag? "windows" - node.raise "va_arg is not yet supported on Windows" - end arg = call.not_nil!.args[0]? || node.raise("requires type argument") node.type = arg.type.instance_type end diff --git a/src/lib_c/aarch64-linux-gnu/c/stdarg.cr b/src/lib_c/aarch64-linux-gnu/c/stdarg.cr index 882d4f51d35c..965355556d71 100644 --- a/src/lib_c/aarch64-linux-gnu/c/stdarg.cr +++ b/src/lib_c/aarch64-linux-gnu/c/stdarg.cr @@ -1,3 +1,10 @@ lib LibC - type VaList = Void* + # based on https://github.com/llvm/llvm-project/blob/bf1cdc2c6c0460b7121ac653c796ef4995b1dfa9/clang/lib/AST/ASTContext.cpp#L7678-L7739 + struct VaList + __stack : Void* + __gr_top : Void* + __vr_top : Void* + __gr_offs : Int32 + __vr_offs : Int32 + end end diff --git a/src/lib_c/aarch64-linux-musl/c/stdarg.cr b/src/lib_c/aarch64-linux-musl/c/stdarg.cr index fcad7714f16a..965355556d71 100644 --- a/src/lib_c/aarch64-linux-musl/c/stdarg.cr +++ b/src/lib_c/aarch64-linux-musl/c/stdarg.cr @@ -1,10 +1,10 @@ lib LibC - struct VaListTag - gp_offset : UInt - fp_offset : UInt - overflow_arg_area : Void* - reg_save_area : Void* + # based on https://github.com/llvm/llvm-project/blob/bf1cdc2c6c0460b7121ac653c796ef4995b1dfa9/clang/lib/AST/ASTContext.cpp#L7678-L7739 + struct VaList + __stack : Void* + __gr_top : Void* + __vr_top : Void* + __gr_offs : Int32 + __vr_offs : Int32 end - - type VaList = VaListTag[1] end diff --git a/src/va_list.cr b/src/va_list.cr index eb0d0121f612..b24304c32e4c 100644 --- a/src/va_list.cr +++ b/src/va_list.cr @@ -18,7 +18,17 @@ struct VaList end end - {% if compare_versions(Crystal::VERSION, "0.33.0-0") > 0 %} + {% if flag?(:aarch64) || flag?(:win32) %} + {% platform = flag?(:aarch64) ? "AArch64" : "Windows" %} + {% clang_impl = flag?(:aarch64) ? "https://github.com/llvm/llvm-project/blob/a574edbba2b24fcfb733aa2d82308131f5b7d2d6/clang/lib/CodeGen/TargetInfo.cpp#L5677-L5921" : "https://github.com/llvm/llvm-project/blob/a574edbba2b24fcfb733aa2d82308131f5b7d2d6/clang/lib/CodeGen/TargetInfo.cpp#L5958-L5964" %} + # Do not call this, instead use C wrappers calling the va_arg macro for the types you need. + # + # Clang implements va_arg on {{platform.id}} like this: {{clang_impl.id}} + # If somebody wants to fix the LLVM IR va_arg instruction on {{platform}} upstream, or port the above here, that would be welcome. + def next(type) + \{% raise "Cannot get variadic argument on {{platform.id}}. As a workaround implement wrappers in C calling the va_arg macro for the types you need and bind to those." %} + end + {% else %} @[Primitive(:va_arg)] def next(type) end From 224ba61bdf3b2e3a803384c14159ec9aaefe3863 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Mon, 22 Jun 2020 19:25:08 +0200 Subject: [PATCH 137/263] Fix bug with passing many args then a struct in Win64 C lib ABI (#9520) Also add a compilation spec for it -- which actually fails on x86_64! --- spec/compiler/codegen/c_abi/c_abi_spec.cr | 30 +++++++++++++++++++++++ src/llvm/abi/x86_win64.cr | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/spec/compiler/codegen/c_abi/c_abi_spec.cr b/spec/compiler/codegen/c_abi/c_abi_spec.cr index 1a772b897ae2..2099c597d770 100644 --- a/spec/compiler/codegen/c_abi/c_abi_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_spec.cr @@ -84,6 +84,36 @@ describe "Code gen: C ABI" do ), &.to_i.should eq(6)) end + {% if flag?(:x86_64) && !flag?(:win32) %} + pending "passes struct after many other args (for real) (#9519)" + {% else %} + it "passes struct after many other args (for real)" do + test_c( + %( + struct s { + long long x, y; + }; + + long long foo(long long a, long long b, long long c, long long d, long long e, struct s v) { + return a + b + c + d + e + v.x + v.y; + } + ), + %( + lib LibFoo + struct S + x : Int64 + y : Int64 + end + + fun foo(a : Int64, b : Int64, c : Int64, d : Int64, e : Int64, v : S) : Int64 + end + + v = LibFoo::S.new(x: 6, y: 7) + LibFoo.foo(1, 2, 3, 4, 5, v) + ), &.to_string.should eq("28")) + end + {% end %} + it "returns struct less than 64 bits (for real)" do test_c( %( diff --git a/src/llvm/abi/x86_win64.cr b/src/llvm/abi/x86_win64.cr index 5a9a07bdb31e..69e9b0823ba9 100644 --- a/src/llvm/abi/x86_win64.cr +++ b/src/llvm/abi/x86_win64.cr @@ -12,7 +12,7 @@ class LLVM::ABI::X86_Win64 < LLVM::ABI::X86 when 2 then ArgType.direct(t, context.int16) when 4 then ArgType.direct(t, context.int32) when 8 then ArgType.direct(t, context.int64) - else ArgType.indirect(t, LLVM::Attribute::ByVal) + else ArgType.indirect(t, nil) end else non_struct(t, context) From bab17be529a40a6b3175dfd07b629cc0752df7b5 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 22 Jun 2020 14:33:24 -0300 Subject: [PATCH 138/263] GitHub Actions (#9514) * GitHub Actions for Linux build * Added macOS workflow * Break spec output lines * Group output on GitHub Actions * Alpine, MT and 32bit jobs for Linux CI * Check format job * Fix interpolation for GitHub log grouping --- .github/workflows/linux.yml | 101 ++++++++++++++++++++++++++++++++++++ .github/workflows/macos.yml | 25 +++++++++ bin/ci | 15 +++++- src/spec/formatter.cr | 20 +++++++ 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/linux.yml create mode 100644 .github/workflows/macos.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 000000000000..2d36da97594c --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,101 @@ +name: Linux CI + +on: [push, pull_request] + +env: + TRAVIS_OS_NAME: linux + SPEC_SPLIT_DOTS: 160 + +jobs: + test_linux: + env: + ARCH: x86_64 + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Test + run: bin/ci build + + test_linux_32: + env: + ARCH: i386 + ARCH_CMD: linux32 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Test + run: bin/ci with_build_env 'make std_spec threads=1' + + test_alpine: + env: + ARCH: x86_64-musl + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Test + run: bin/ci build + + test_preview_mt: + env: + ARCH: x86_64 + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Make Crystal + run: bin/ci with_build_env 'make crystal' + + - name: Test + run: bin/ci with_build_env 'CRYSTAL_WORKERS=4 make std_spec threads=1 FLAGS="-D preview_mt"' + + check_format: + env: + ARCH: x86_64 + ARCH_CMD: linux64 + runs-on: ubuntu-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Check Format + run: bin/ci format diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000000..9d29a7d243a3 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,25 @@ +name: macOS CI + +on: [push, pull_request] + +env: + TRAVIS_OS_NAME: osx + LLVM_CONFIG: /usr/local/opt/llvm/bin/llvm-config + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + SPEC_SPLIT_DOTS: 160 + +jobs: + test_macos: + runs-on: macos-latest + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + + - name: Prepare System + run: bin/ci prepare_system + + - name: Prepare Build + run: bin/ci prepare_build + + - name: Test + run: bin/ci build diff --git a/bin/ci b/bin/ci index 2c83c1b1b37a..5bff8a0a6797 100755 --- a/bin/ci +++ b/bin/ci @@ -65,6 +65,15 @@ on_osx() { fail_on_error on_os "osx" "${@}" } +on_github() { + if [ "$GITHUB_ACTIONS" = "true" ]; then + eval "${@}" + return $? + else + return 0 + fi +} + prepare_system() { on_linux 'echo '"'"'{"ipv6":true, "fixed-cidr-v6":"2001:db8:1::/64"}'"'"' | sudo tee /etc/docker/daemon.json' on_linux sudo service docker restart @@ -112,7 +121,8 @@ prepare_build() { on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-0.35.1-1 crystal;popd' on_osx 'brew unlink python@2 || true' - on_osx brew install z3 llvm@10 gmp libevent pcre openssl pkg-config + on_osx brew install z3 llvm@10 gmp libevent pcre pkg-config + on_osx brew reinstall openssl on_osx brew link --force llvm@10 # Note: brew link --force might show: # Warning: Refusing to link macOS-provided software: llvm @@ -137,6 +147,7 @@ verify_version() { with_build_env() { command="$1" + on_github "echo '::group::$1'" # Ensure non GMT timezone export TZ="America/New_York" @@ -165,6 +176,7 @@ with_build_env() { -v /etc/group:/etc/group \ -w /mnt \ -e CRYSTAL_CACHE_DIR="/tmp/crystal" \ + -e SPEC_SPLIT_DOTS \ "$DOCKER_TEST_IMAGE" \ "$ARCH_CMD" /bin/sh -c "'$command'" @@ -174,6 +186,7 @@ with_build_env() { CRYSTAL_CACHE_DIR="/tmp/crystal" \ /bin/sh -c "'$command'" + on_github echo "::endgroup::" } usage() { diff --git a/src/spec/formatter.cr b/src/spec/formatter.cr index e031204e109e..51b355ec075f 100644 --- a/src/spec/formatter.cr +++ b/src/spec/formatter.cr @@ -25,11 +25,31 @@ module Spec # :nodoc: class DotFormatter < Formatter + @count = 0 + @split = 0 + + def initialize(*args) + super + + if split = ENV["SPEC_SPLIT_DOTS"]? + @split = split.to_i + end + end + def report(result) @io << Spec.color(LETTERS[result.kind], result.kind) + split_lines @io.flush end + private def split_lines + return unless @split > 0 + if (@count += 1) >= @split + @io.puts + @count = 0 + end + end + def finish(elapsed_time, aborted) @io.puts end From 560581bede5cf6fbf2fd70d2d7ffa381b14a27f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 22 Jun 2020 21:22:20 +0200 Subject: [PATCH 139/263] Fix C ABI for AArch64 (#9430) I don't claim to understand why this is more correct, I just compared the LLVM IR from one failing spec to what Clang would generate for the equivalent code. It doesn't seemm to break any other specs --- spec/compiler/codegen/c_abi/c_abi_spec.cr | 2 +- spec/std/llvm/aarch64_spec.cr | 2 +- src/llvm/abi/aarch64.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/compiler/codegen/c_abi/c_abi_spec.cr b/spec/compiler/codegen/c_abi/c_abi_spec.cr index 2099c597d770..17e885628c32 100644 --- a/spec/compiler/codegen/c_abi/c_abi_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_spec.cr @@ -55,7 +55,7 @@ describe "Code gen: C ABI" do ), &.to_i.should eq(3)) end - it "passes struct bigger than128 bits (for real)" do + it "passes struct bigger than 128 bits (for real)" do test_c( %( struct s { diff --git a/spec/std/llvm/aarch64_spec.cr b/spec/std/llvm/aarch64_spec.cr index ee30468f0d26..caaa9d8e0ff1 100644 --- a/spec/std/llvm/aarch64_spec.cr +++ b/spec/std/llvm/aarch64_spec.cr @@ -140,7 +140,7 @@ class LLVM::ABI info = abi.abi_info(arg_types, return_type, true, ctx) info.arg_types.size.should eq(1) - info.arg_types[0].should eq(ArgType.indirect(str, Attribute::ByVal)) + info.arg_types[0].should eq(ArgType.indirect(str, nil)) info.return_type.should eq(ArgType.indirect(str, Attribute::StructRet)) end end diff --git a/src/llvm/abi/aarch64.cr b/src/llvm/abi/aarch64.cr index 52cf84ae4b82..14a576d415e8 100644 --- a/src/llvm/abi/aarch64.cr +++ b/src/llvm/abi/aarch64.cr @@ -136,7 +136,7 @@ class LLVM::ABI::AArch64 < LLVM::ABI end ArgType.direct(aty, cast) else - ArgType.indirect(aty, LLVM::Attribute::ByVal) + ArgType.indirect(aty, nil) end end end From b0b891a9828a9cad64b9f2630d7692a012397878 Mon Sep 17 00:00:00 2001 From: Caspian Baska Date: Tue, 23 Jun 2020 06:42:30 +1000 Subject: [PATCH 140/263] Add a note regarding Log.setup (#9497) --- src/log.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/log.cr b/src/log.cr index 5b5f7acb8913..b40c089e5a12 100644 --- a/src/log.cr +++ b/src/log.cr @@ -80,6 +80,8 @@ # # If you need to change the default level, backend or sources call `Log.setup` upon startup. # +# NOTE: Calling `setup` will override previous `setup` calls. +# # ``` # Log.setup(:debug) # Log debug and above for all sources to STDOUT # Log.setup("myapp.*, http.*", :notice) # Log notice and above for myapp.* and http.* sources only, and log nothing for any other source. From 71b97fbe222865b657461b486c693cd985d3a094 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 22 Jun 2020 15:43:17 -0500 Subject: [PATCH 141/263] Update installation link in CONTRIBUTING.md (#9448) fixes https://github.com/crystal-lang/crystal/issues/9447 --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd56c948b535..f2e8cece4925 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,7 +91,7 @@ the docs execute `make docs`. Please follow the guidelines described in our Be sure to execute `make libcrystal` inside the cloned repository. -Once in the cloned directory, and once you [installed Crystal](http://crystal-lang.org/docs/installation/index.html), +Once in the cloned directory, and once you [installed Crystal](https://crystal-lang.org/install/), you can execute `bin/crystal` instead of `crystal`. This is a wrapper that will use the cloned repository as the standard library. Otherwise the barebones `crystal` executable uses the standard library that comes in your installation. @@ -111,7 +111,7 @@ Then push your changes and create a pull request. ### The compiler itself If you want to add/change something in the compiler, -the first thing you will need to do is to [install the compiler](https://crystal-lang.org/docs/installation/index.html). +the first thing you will need to do is to [install the compiler](https://crystal-lang.org/install/). Once you have a compiler up and running, check that executing `crystal` on the command line prints its usage. Now you can setup your environment to compile Crystal itself, which is itself written in Crystal. Check out From af2a7c196360e812eb98baad439a0eae1fd557a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 22 Jun 2020 22:50:13 +0200 Subject: [PATCH 142/263] Prevent recursive call in bin/crystal wrapper (#9505) Apply the existing PATH technique removal to the new call to crystal env, allowing to put bin/ into one's PATH again --- bin/crystal | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/crystal b/bin/crystal index 65f39e07643f..d12f72f5aca7 100755 --- a/bin/crystal +++ b/bin/crystal @@ -142,7 +142,10 @@ export CRYSTAL_PATH=lib:$CRYSTAL_ROOT/src export CRYSTAL_HAS_WRAPPER=true if [ -z "$CRYSTAL_CONFIG_LIBRARY_PATH" ]; then - export CRYSTAL_CONFIG_LIBRARY_PATH="$(crystal env CRYSTAL_LIBRARY_PATH || echo "")" + export CRYSTAL_CONFIG_LIBRARY_PATH="$( + export PATH="$(remove_path_item "$(remove_path_item "$PATH" "$SCRIPT_ROOT")" "bin")" + crystal env CRYSTAL_LIBRARY_PATH || echo "" + )" fi if [ -x "$CRYSTAL_DIR/crystal" ]; then From 6768950bccd016ef1b8c10433df40304c6e7e307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 22 Jun 2020 22:53:05 +0200 Subject: [PATCH 143/263] Extract helper methods in call_error.cr (#9376) * Extract helper methods in call_error.cr * Update src/compiler/crystal/semantic/call_error.cr Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/semantic/call_error.cr | 233 ++++++++++---------- 1 file changed, 116 insertions(+), 117 deletions(-) diff --git a/src/compiler/crystal/semantic/call_error.cr b/src/compiler/crystal/semantic/call_error.cr index 6309874069ce..c10bfb5681d7 100644 --- a/src/compiler/crystal/semantic/call_error.cr +++ b/src/compiler/crystal/semantic/call_error.cr @@ -92,77 +92,7 @@ class Crystal::Call end if defs.empty? - check_macro_wrong_number_of_arguments(def_name) - - owner_trace = obj.try &.find_owner_trace(owner.program, owner) - similar_name = owner.lookup_similar_def_name(def_name, self.args.size, block) - - error_msg = String.build do |msg| - if obj - could_be_local_variable = false - elsif logical_op = convert_to_logical_operator(def_name) - similar_name = logical_op - could_be_local_variable = false - elsif args.size > 0 || has_parentheses? - could_be_local_variable = false - else - # This check is for the case `a if a = 1` - similar_name = parent_visitor.lookup_similar_var_name(def_name) unless similar_name - if similar_name == def_name - could_be_local_variable = false - else - could_be_local_variable = true - end - end - - if could_be_local_variable - msg << "undefined local variable or method '#{def_name}'" - else - msg << "undefined method '#{def_name}'" - end - - owner_name = owner.is_a?(Program) ? "top-level" : owner.to_s - - if with_scope && !obj && with_scope != owner - msg << " for #{with_scope} (with ... yield) and #{owner_name} (current scope)" - else - msg << " for #{owner_name}" - end - - if def_name == "allocate" && owner.is_a?(MetaclassType) && owner.instance_type.module? - msg << colorize(" (modules cannot be instantiated)").yellow.bold - end - - if obj && obj.type != owner - msg << colorize(" (compile-time type is #{obj.type})").yellow.bold - end - - if similar_name - msg << '\n' - if similar_name == def_name - # This check is for the case `a if a = 1` - msg << "If you declared '#{def_name}' in a suffix if, declare it in a regular if for this to work. If the variable was declared in a macro it's not visible outside it)" - else - msg << "Did you mean '#{similar_name}'?" - end - end - - # Check if it's an instance variable that was never assigned a value - if obj.is_a?(InstanceVar) - scope = self.scope - ivar = scope.lookup_instance_var(obj.name) - deps = ivar.dependencies? - if deps && deps.size == 1 && deps.first.same?(program.nil_var) - similar_name = scope.lookup_similar_instance_var_name(ivar.name) - if similar_name - msg << colorize(" (#{ivar.name} was never assigned a value, did you mean #{similar_name}?)").yellow.bold - else - msg << colorize(" (#{ivar.name} was never assigned a value)").yellow.bold - end - end - end - end - raise error_msg, owner_trace + raise_undefined_method(owner, def_name, obj) end real_args_size = arg_types.size @@ -173,13 +103,6 @@ class Crystal::Call inner_exception = TypeException.for_node(similar_def, inner_msg) end - if owner_trace - owner_trace.inner = inner_exception - inner_exception = nil - else - owner_trace = inner_exception - end - defs_matching_args_size = defs.select do |a_def| min_size, max_size = a_def.min_max_args_sizes min_size <= real_args_size <= max_size @@ -187,44 +110,7 @@ class Crystal::Call # Don't say "wrong number of arguments" when there are named args in this call if defs_matching_args_size.empty? && !named_args_types - all_arguments_sizes = [] of Int32 - min_splat = Int32::MAX - defs.each do |a_def| - min_size, max_size = a_def.min_max_args_sizes - if max_size == Int32::MAX - min_splat = Math.min(min_size, min_splat) - all_arguments_sizes.push min_splat - else - min_size.upto(max_size) do |size| - all_arguments_sizes.push size - end - end - end - all_arguments_sizes.uniq!.sort! - - raise(String.build do |str| - unless check_single_def_error_message(defs, named_args_types, str) - str << "wrong number of arguments for '" - str << full_name(owner, def_name) - str << "' (given " - str << real_args_size - str << ", expected " - - # If we have 2, 3, 4, show it as 2..4 - if all_arguments_sizes.size > 1 && all_arguments_sizes.last - all_arguments_sizes.first == all_arguments_sizes.size - 1 - str << all_arguments_sizes.first - str << ".." - str << all_arguments_sizes.last - else - all_arguments_sizes.join str, ", " - end - - str << '+' if min_splat != Int32::MAX - str << ")\n" - end - str << "Overloads are:" - append_matches(defs, arg_types, str) - end, inner: inner_exception) + raise_matches_not_found_named_args(owner, def_name, defs, real_args_size, arg_types, named_args_types, inner_exception) end if defs_matching_args_size.size > 0 @@ -257,6 +143,8 @@ class Crystal::Call if args.size == 1 && args.first.type.includes_type?(program.nil) owner_trace = args.first.find_owner_trace(program, program.nil) + else + owner_trace = inner_exception end arg_names = [] of Array(String) @@ -283,7 +171,7 @@ class Crystal::Call msg << '\n' defs.each do |a_def| - arg_names.try &.push a_def.args.map(&.name) + arg_names << a_def.args.map(&.name) end end @@ -323,6 +211,117 @@ class Crystal::Call raise message, owner_trace end + private def raise_undefined_method(owner, def_name, obj) + check_macro_wrong_number_of_arguments(def_name) + + owner_trace = obj.try &.find_owner_trace(owner.program, owner) + similar_name = owner.lookup_similar_def_name(def_name, self.args.size, block) + + error_msg = String.build do |msg| + if obj + could_be_local_variable = false + elsif logical_op = convert_to_logical_operator(def_name) + similar_name = logical_op + could_be_local_variable = false + elsif args.size > 0 || has_parentheses? + could_be_local_variable = false + else + # This check is for the case `a if a = 1` + similar_name = parent_visitor.lookup_similar_var_name(def_name) unless similar_name + could_be_local_variable = (similar_name != def_name) + end + + if could_be_local_variable + msg << "undefined local variable or method '#{def_name}'" + else + msg << "undefined method '#{def_name}'" + end + + owner_name = owner.is_a?(Program) ? "top-level" : owner.to_s + + if with_scope && !obj && with_scope != owner + msg << " for #{with_scope} (with ... yield) and #{owner_name} (current scope)" + else + msg << " for #{owner_name}" + end + + if def_name == "allocate" && owner.is_a?(MetaclassType) && owner.instance_type.module? + msg << colorize(" (modules cannot be instantiated)").yellow.bold + end + + if obj && obj.type != owner + msg << colorize(" (compile-time type is #{obj.type})").yellow.bold + end + + if similar_name + msg << '\n' + if similar_name == def_name + # This check is for the case `a if a = 1` + msg << "If you declared '#{def_name}' in a suffix if, declare it in a regular if for this to work. If the variable was declared in a macro it's not visible outside it)" + else + msg << "Did you mean '#{similar_name}'?" + end + end + + # Check if it's an instance variable that was never assigned a value + if obj.is_a?(InstanceVar) + scope = self.scope + ivar = scope.lookup_instance_var(obj.name) + deps = ivar.dependencies? + if deps && deps.size == 1 && deps.first.same?(program.nil_var) + similar_name = scope.lookup_similar_instance_var_name(ivar.name) + if similar_name + msg << colorize(" (#{ivar.name} was never assigned a value, did you mean #{similar_name}?)").yellow.bold + else + msg << colorize(" (#{ivar.name} was never assigned a value)").yellow.bold + end + end + end + end + raise error_msg, owner_trace + end + + private def raise_matches_not_found_named_args(owner, def_name, defs, real_args_size, arg_types, named_args_types, inner_exception) + all_arguments_sizes = [] of Int32 + min_splat = Int32::MAX + defs.each do |a_def| + min_size, max_size = a_def.min_max_args_sizes + if max_size == Int32::MAX + min_splat = Math.min(min_size, min_splat) + all_arguments_sizes.push min_splat + else + min_size.upto(max_size) do |size| + all_arguments_sizes.push size + end + end + end + all_arguments_sizes.uniq!.sort! + + raise(String.build do |str| + unless check_single_def_error_message(defs, named_args_types, str) + str << "wrong number of arguments for '" + str << full_name(owner, def_name) + str << "' (given " + str << real_args_size + str << ", expected " + + # If we have 2, 3, 4, show it as 2..4 + if all_arguments_sizes.size > 1 && all_arguments_sizes.last - all_arguments_sizes.first == all_arguments_sizes.size - 1 + str << all_arguments_sizes.first + str << ".." + str << all_arguments_sizes.last + else + all_arguments_sizes.join str, ", " + end + + str << '+' if min_splat != Int32::MAX + str << ")\n" + end + str << "Overloads are:" + append_matches(defs, arg_types, str) + end, inner: inner_exception) + end + def convert_to_logical_operator(def_name) case def_name when "and"; "&&" From 778ffaceaeb82f5a6306a300da47b311f8435339 Mon Sep 17 00:00:00 2001 From: Hiroki Noda Date: Tue, 23 Jun 2020 05:54:38 +0900 Subject: [PATCH 144/263] Use function attribute "frame-pointer" (#9361) LLVM 8([D56351](https://reviews.llvm.org/D56351)) introduced frame-pointer which was intended to replace no-frame-pointer-elim and no-frame-pointer-elim-non-leaf. --- src/compiler/crystal/codegen/fun.cr | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/codegen/fun.cr b/src/compiler/crystal/codegen/fun.cr index e8e8c548eaed..8562cacb4528 100644 --- a/src/compiler/crystal/codegen/fun.cr +++ b/src/compiler/crystal/codegen/fun.cr @@ -88,8 +88,12 @@ class Crystal::CodeGenVisitor context.fun.add_attribute LLVM::Attribute::UWTable if @program.has_flag?("darwin") # Disable frame pointer elimination in Darwin, as it causes issues during stack unwind - context.fun.add_target_dependent_attribute "no-frame-pointer-elim", "true" - context.fun.add_target_dependent_attribute "no-frame-pointer-elim-non-leaf", "true" + {% if compare_versions(Crystal::LLVM_VERSION, "8.0.0") < 0 %} + context.fun.add_target_dependent_attribute "no-frame-pointer-elim", "true" + context.fun.add_target_dependent_attribute "no-frame-pointer-elim-non-leaf", "true" + {% else %} + context.fun.add_target_dependent_attribute "frame-pointer", "all" + {% end %} end new_entry_block From 8fd246c8ded2446a3f6cb5a4ed8c1accb0eec98b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 22 Jun 2020 22:55:53 +0200 Subject: [PATCH 145/263] Add Crystal::Path#name_size implementation (#9380) --- src/compiler/crystal/syntax/ast.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/syntax/ast.cr b/src/compiler/crystal/syntax/ast.cr index 271a8320b9b5..7b1b489ad9c7 100644 --- a/src/compiler/crystal/syntax/ast.cr +++ b/src/compiler/crystal/syntax/ast.cr @@ -1273,7 +1273,6 @@ module Crystal class Path < ASTNode property names : Array(String) property? global : Bool - property name_size = 0 property visibility = Visibility::Public def initialize(@names : Array, @global = false) @@ -1287,6 +1286,10 @@ module Crystal new names, true end + def name_size + names.sum(&.size) + (names.size + (global? ? 0 : -1)) * 2 + end + # Returns true if this path has a single component # with the given name def single?(name) @@ -1295,7 +1298,6 @@ module Crystal def clone_without_location ident = Path.new(@names.clone, @global) - ident.name_size = name_size ident end From 649236042cf19fc5b48e5a9ce586628f2e99fbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 22 Jun 2020 22:58:23 +0200 Subject: [PATCH 146/263] Make specs pass in non-IPv6 environments (#9438) * Make specs pass in non-IPv6 environments It's 2020 and sadly IPv6 is not yet truly ubiquitous. While supported by all relevant operating systems, enabled, even just locally, in all relevant execution environments, not so much. Prominently Docker does not enable it in its default configuration. * fixup! Make specs pass in non-IPv6 environments --- spec/std/http/client/client_spec.cr | 11 +++++----- spec/std/oauth2/client_spec.cr | 32 ++++++++++++++--------------- spec/std/openssl/ssl/server_spec.cr | 8 ++++---- spec/std/openssl/ssl/socket_spec.cr | 15 +++++++------- spec/std/socket/socket_spec.cr | 8 ++++---- spec/std/socket/spec_helper.cr | 31 +++++++++++++++++++++++----- 6 files changed, 64 insertions(+), 41 deletions(-) diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 6b9453f47b32..1a80c07941cc 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -1,4 +1,5 @@ require "../spec_helper" +require "../../socket/spec_helper" require "openssl" require "http/client" require "http/server" @@ -142,7 +143,7 @@ module HTTP end end - it "sends the host header ipv6 with brackets" do + pending_ipv6 "sends the host header ipv6 with brackets" do server = HTTP::Server.new do |context| context.response.print context.request.headers["Host"] end @@ -157,10 +158,10 @@ module HTTP server = HTTP::Server.new do |context| context.response.print context.request.headers["connection"] end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - HTTP::Client.get("http://[::1]:#{address.port}/").body.should eq("close") + HTTP::Client.get("http://127.0.0.1:#{address.port}/").body.should eq("close") end end @@ -168,10 +169,10 @@ module HTTP server = HTTP::Server.new do |context| context.response.print context.request.headers["connection"] end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - HTTP::Client.get("http://[::1]:#{address.port}/") do |response| + HTTP::Client.get("http://127.0.0.1:#{address.port}/") do |response| response.body_io.gets_to_end end.should eq("close") end diff --git a/spec/std/oauth2/client_spec.cr b/spec/std/oauth2/client_spec.cr index c5506cbbede8..e1224574efbe 100644 --- a/spec/std/oauth2/client_spec.cr +++ b/spec/std/oauth2/client_spec.cr @@ -48,10 +48,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http" token = client.get_access_token_using_authorization_code(authorization_code: "SDFhw39fwfg23flSfpawbef") token.extra.not_nil!["body"].should eq %("redirect_uri=&grant_type=authorization_code&code=SDFhw39fwfg23flSfpawbef") @@ -66,10 +66,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http" token = client.get_access_token_using_resource_owner_credentials(username: "user123", password: "monkey", scope: "read_posts") token.extra.not_nil!["body"].should eq %("grant_type=password&username=user123&password=monkey&scope=read_posts") @@ -84,10 +84,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http" token = client.get_access_token_using_client_credentials(scope: "read_posts") token.extra.not_nil!["body"].should eq %("grant_type=client_credentials&scope=read_posts") @@ -102,10 +102,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http" + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http" token = client.get_access_token_using_refresh_token(scope: "read_posts", refresh_token: "some_refresh_token") token.extra.not_nil!["body"].should eq %("grant_type=refresh_token&refresh_token=some_refresh_token&scope=read_posts") @@ -121,10 +121,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody token = client.get_access_token_using_authorization_code(authorization_code: "SDFhw39fwfg23flSfpawbef") token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&redirect_uri=&grant_type=authorization_code&code=SDFhw39fwfg23flSfpawbef") @@ -139,10 +139,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody token = client.get_access_token_using_resource_owner_credentials(username: "user123", password: "monkey", scope: "read_posts") token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=password&username=user123&password=monkey&scope=read_posts") @@ -157,10 +157,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody token = client.get_access_token_using_client_credentials(scope: "read_posts") token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=client_credentials&scope=read_posts") @@ -175,10 +175,10 @@ describe OAuth2::Client do context.response.print response.to_json end - address = server.bind_unused_port "::1" + address = server.bind_unused_port "127.0.0.1" run_server(server) do - client = OAuth2::Client.new "[::1]", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody + client = OAuth2::Client.new "127.0.0.1", "client_id", "client_secret", port: address.port, scheme: "http", auth_scheme: OAuth2::AuthScheme::RequestBody token = client.get_access_token_using_refresh_token(scope: "read_posts", refresh_token: "some_refresh_token") token.extra.not_nil!["body"].should eq %("client_id=client_id&client_secret=client_secret&grant_type=refresh_token&refresh_token=some_refresh_token&scope=read_posts") diff --git a/spec/std/openssl/ssl/server_spec.cr b/spec/std/openssl/ssl/server_spec.cr index 0fd82271ad0d..ff5e578a8ed0 100644 --- a/spec/std/openssl/ssl/server_spec.cr +++ b/spec/std/openssl/ssl/server_spec.cr @@ -54,7 +54,7 @@ describe OpenSSL::SSL::Server do describe "#accept?" do it "accepts" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair @@ -79,7 +79,7 @@ describe OpenSSL::SSL::Server do describe "#accept" do it "accepts and do handshake" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair @@ -100,7 +100,7 @@ describe OpenSSL::SSL::Server do end it "doesn't to SSL handshake with start_immediately = false" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair @@ -125,7 +125,7 @@ describe OpenSSL::SSL::Server do end it "detects SNI hostname" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair OpenSSL::SSL::Server.open tcp_server, server_context do |server| diff --git a/spec/std/openssl/ssl/socket_spec.cr b/spec/std/openssl/ssl/socket_spec.cr index 40e10d5c31b9..d56d744a3514 100644 --- a/spec/std/openssl/ssl/socket_spec.cr +++ b/spec/std/openssl/ssl/socket_spec.cr @@ -1,12 +1,13 @@ require "spec" require "socket" require "../../spec_helper" +require "../../socket/spec_helper" require "../../../support/ssl" describe OpenSSL::SSL::Socket do describe OpenSSL::SSL::Socket::Server do it "auto accept client by default" do - TCPServer.open(0) do |tcp_server| + TCPServer.open("127.0.0.1", 0) do |tcp_server| server_context, client_context = ssl_context_pair spawn do @@ -23,7 +24,7 @@ describe OpenSSL::SSL::Socket do end it "doesn't accept client when specified" do - TCPServer.open(0) do |tcp_server| + TCPServer.open("127.0.0.1", 0) do |tcp_server| server_context, client_context = ssl_context_pair spawn do @@ -42,7 +43,7 @@ describe OpenSSL::SSL::Socket do end it "returns the cipher that is currently in use" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair OpenSSL::SSL::Server.open(tcp_server, server_context) do |server| @@ -58,7 +59,7 @@ describe OpenSSL::SSL::Socket do end it "returns the TLS version" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair OpenSSL::SSL::Server.open(tcp_server, server_context) do |server| @@ -74,7 +75,7 @@ describe OpenSSL::SSL::Socket do end it "accepts clients that only write then close the connection" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair # in tls 1.3, if clients don't read anything and close the connection # the server still try and write to it a ticket, resulting in a "pipe failure" @@ -96,7 +97,7 @@ describe OpenSSL::SSL::Socket do end it "closes connection to server that doesn't properly terminate SSL session" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair server_context.disable_session_resume_tickets # avoid Broken pipe @@ -116,7 +117,7 @@ describe OpenSSL::SSL::Socket do end it "interprets graceful EOF of underlying socket as SSL termination" do - tcp_server = TCPServer.new(0) + tcp_server = TCPServer.new("127.0.0.1", 0) server_context, client_context = ssl_context_pair server_context.disable_session_resume_tickets # avoid Broken pipe diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index 9b9ecb146c60..c0ae54291d84 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -44,10 +44,10 @@ describe Socket do it "sends messages" do port = unused_local_port - server = Socket.tcp(Socket::Family::INET6) - server.bind("::1", port) + server = Socket.tcp(Socket::Family::INET) + server.bind("127.0.0.1", port) server.listen - address = Socket::IPAddress.new("::1", port) + address = Socket::IPAddress.new("127.0.0.1", port) spawn do client = server.not_nil!.accept client.gets.should eq "foo" @@ -55,7 +55,7 @@ describe Socket do ensure client.try &.close end - socket = Socket.tcp(Socket::Family::INET6) + socket = Socket.tcp(Socket::Family::INET) socket.connect(address) socket.puts "foo" socket.gets.should eq "bar" diff --git a/spec/std/socket/spec_helper.cr b/spec/std/socket/spec_helper.cr index bfd016f4129a..486e4a142ee7 100644 --- a/spec/std/socket/spec_helper.cr +++ b/spec/std/socket/spec_helper.cr @@ -1,9 +1,20 @@ require "spec" require "socket" -def unused_local_port - TCPServer.open("::", 0) do |server| - server.local_address.port +module SocketSpecHelper + class_getter?(supports_ipv6 : Bool) do + TCPServer.open("::1", 0) { return true } + false + rescue Socket::BindError + false + end +end + +def pending_ipv6(description = "assert", file = __FILE__, line = __LINE__, end_line = __END_LINE__, &block) + if SocketSpecHelper.supports_ipv6? + it(description, file: file, line: line, end_line: end_line, &block) + else + pending(description, file: file, line: line, end_line: end_line) end end @@ -12,7 +23,17 @@ def each_ip_family(&block : Socket::Family, String, String ->) block.call Socket::Family::INET, "127.0.0.1", "0.0.0.0" end - describe "using IPv6" do - block.call Socket::Family::INET6, "::1", "::" + if SocketSpecHelper.supports_ipv6? + describe "using IPv6" do + block.call Socket::Family::INET6, "::1", "::" + end + else + pending "using IPv6" + end +end + +def unused_local_port + TCPServer.open("::", 0) do |server| + server.local_address.port end end From 956f40164ae5e2c8fa9a2e31e54e5cde10ee5f0d Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 23 Jun 2020 09:06:09 +0200 Subject: [PATCH 147/263] Win CI: Fix and enable compiler specs (#9348) * Win CI: Fix and enable compiler specs * Just move crystal.exe under bin/ to not need the env override --- .github/workflows/win.yml | 18 +++++++--- spec/compiler/codegen/extern_spec.cr | 2 +- spec/compiler/codegen/lib_spec.cr | 4 +-- spec/compiler/codegen/special_vars_spec.cr | 10 +++--- spec/compiler/codegen/thread_local_spec.cr | 2 +- spec/compiler/compiler_spec.cr | 2 +- .../crystal/tools/doc/project_info_spec.cr | 12 +++---- spec/compiler/crystal/tools/format_spec.cr | 34 +++++++++---------- spec/compiler/crystal/tools/init_spec.cr | 2 +- .../compiler/crystal/tools/playground_spec.cr | 2 ++ .../crystal_path/crystal_path_spec.cr | 4 +-- spec/compiler/semantic/generic_class_spec.cr | 6 ++-- spec/compiler/semantic/primitives_spec.cr | 2 +- spec/std/exception/call_stack_spec.cr | 4 +-- spec/std/io/file_descriptor_spec.cr | 2 +- spec/support/tempfile.cr | 31 ++++++++++++++--- spec/win32_std_spec.cr | 4 +-- src/empty.cr | 3 ++ 18 files changed, 90 insertions(+), 54 deletions(-) diff --git a/.github/workflows/win.yml b/.github/workflows/win.yml index 571e7624acde..3edaa3f89364 100644 --- a/.github/workflows/win.yml +++ b/.github/workflows/win.yml @@ -148,7 +148,7 @@ jobs: - name: Set up environment run: | - echo "::set-env name=CRYSTAL_PATH::src" + echo "::set-env name=CRYSTAL_PATH::$(pwd)\src" echo "::set-env name=LIB::${env:LIB};$(pwd)\libs" echo "::set-env name=TERM::dumb" echo "::set-env name=LLVM_CONFIG::$(pwd)\llvm\bin\llvm-config.exe" @@ -168,13 +168,14 @@ jobs: - name: Re-build Crystal run: | .\crystal-cross.exe build src/compiler/crystal.cr -Di_know_what_im_doing -Dwithout_playground --link-flags=/F10000000 + mv crystal.exe bin/ - - name: Gather Crystal executable + - name: Gather Crystal binaries run: | mkdir crystal - cp crystal.exe crystal/ + cp bin/crystal.exe crystal/ cp libs/* crystal/ - - name: Upload Crystal executable + - name: Upload Crystal binaries uses: actions/upload-artifact@v1 with: name: crystal @@ -182,7 +183,14 @@ jobs: - name: Build stdlib specs executable run: | - .\crystal.exe build spec/std_spec.cr --exclude-warnings spec/std -Dwithout_openssl -Di_know_what_im_doing + bin\crystal.exe build spec/std_spec.cr --exclude-warnings spec/std --exclude-warnings spec/compiler -Dwithout_openssl -Di_know_what_im_doing - name: Run stdlib specs run: | .\std_spec.exe + + - name: Build compiler specs executable + run: | + bin\crystal.exe build spec/compiler_spec.cr --exclude-warnings spec/std --exclude-warnings spec/compiler -Dwithout_playground -Di_know_what_im_doing + - name: Run compiler specs + run: | + .\compiler_spec.exe diff --git a/spec/compiler/codegen/extern_spec.cr b/spec/compiler/codegen/extern_spec.cr index e1f0e401e18a..6efd7a44c16b 100644 --- a/spec/compiler/codegen/extern_spec.cr +++ b/spec/compiler/codegen/extern_spec.cr @@ -427,7 +427,7 @@ describe "Codegen: extern struct" do ), &.to_i.should eq(30)) end - it "codegens proc that takes and returns an extern struct with sret" do + pending_win32 "codegens proc that takes and returns an extern struct with sret" do test_c( %( struct Struct { diff --git a/spec/compiler/codegen/lib_spec.cr b/spec/compiler/codegen/lib_spec.cr index e899bcf6d2cf..695a02d6fd6b 100644 --- a/spec/compiler/codegen/lib_spec.cr +++ b/spec/compiler/codegen/lib_spec.cr @@ -15,11 +15,11 @@ describe "Code gen: lib" do it "call to void function" do run(" lib LibC - fun srandom(x : UInt32) : Void + fun srand(x : UInt32) : Void end def foo - LibC.srandom(0_u32) + LibC.srand(0_u32) end foo diff --git a/spec/compiler/codegen/special_vars_spec.cr b/spec/compiler/codegen/special_vars_spec.cr index 280da6caaa98..14323c96d28e 100644 --- a/spec/compiler/codegen/special_vars_spec.cr +++ b/spec/compiler/codegen/special_vars_spec.cr @@ -2,7 +2,7 @@ require "../../spec_helper" describe "Codegen: special vars" do ["$~", "$?"].each do |name| - it "codegens #{name}" do + pending_win32 "codegens #{name}" do run(%( class Object; def not_nil!; self; end; end @@ -15,7 +15,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - it "codegens #{name} with nilable (1)" do + pending_win32 "codegens #{name} with nilable (1)" do run(%( require "prelude" @@ -35,7 +35,7 @@ describe "Codegen: special vars" do )).to_string.should eq("ouch") end - it "codegens #{name} with nilable (2)" do + pending_win32 "codegens #{name} with nilable (2)" do run(%( require "prelude" @@ -74,7 +74,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - it "works lazily" do + pending_win32 "works lazily" do run(%( require "prelude" @@ -145,7 +145,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - it "codegens after block" do + pending_win32 "codegens after block" do run(%( require "prelude" diff --git a/spec/compiler/codegen/thread_local_spec.cr b/spec/compiler/codegen/thread_local_spec.cr index 09e40b159bf4..ea26bfe50fcc 100644 --- a/spec/compiler/codegen/thread_local_spec.cr +++ b/spec/compiler/codegen/thread_local_spec.cr @@ -2,7 +2,7 @@ require "../../spec_helper" {% if !flag?(:openbsd) %} describe "Codegen: thread local" do - it "works with class variables" do + pending_win32 "works with class variables" do run(%( require "prelude" diff --git a/spec/compiler/compiler_spec.cr b/spec/compiler/compiler_spec.cr index ce7c086f99ec..66d2ad7871aa 100644 --- a/spec/compiler/compiler_spec.cr +++ b/spec/compiler/compiler_spec.cr @@ -30,7 +30,7 @@ describe "Compiler" do it "treats all arguments post-filename as program arguments" do with_tempfile "args_test" do |path| - `bin/crystal '#{compiler_datapath}/args_test' -Dother_flag -- bar '#{path}'` + `bin/crystal #{Process.quote(File.join(compiler_datapath, "args_test"))} -Dother_flag -- bar #{Process.quote(path)}` File.read(path).should eq(<<-FILE) ["-Dother_flag", "--", "bar"] diff --git a/spec/compiler/crystal/tools/doc/project_info_spec.cr b/spec/compiler/crystal/tools/doc/project_info_spec.cr index f255d8ae72de..42191829968a 100644 --- a/spec/compiler/crystal/tools/doc/project_info_spec.cr +++ b/spec/compiler/crystal/tools/doc/project_info_spec.cr @@ -50,7 +50,7 @@ describe Crystal::Doc::ProjectInfo do it "git tagged version" do run_git "init" run_git "add shard.yml" - run_git "commit -m 'Initial commit' --no-gpg-sign" + run_git "commit -m \"Initial commit\" --no-gpg-sign" run_git "tag v3.0" assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "3.0", refname: "v3.0")) @@ -61,7 +61,7 @@ describe Crystal::Doc::ProjectInfo do it "git tagged version dirty" do run_git "init" run_git "add shard.yml" - run_git "commit -m 'Initial commit' --no-gpg-sign" + run_git "commit -m \"Initial commit\" --no-gpg-sign" run_git "tag v3.0" File.write("foo.txt", "bar") @@ -73,7 +73,7 @@ describe Crystal::Doc::ProjectInfo do it "git non-tagged commit" do run_git "init" run_git "add shard.yml" - run_git "commit -m 'Initial commit' --no-gpg-sign" + run_git "commit -m \"Initial commit\" --no-gpg-sign" commit_sha = `git rev-parse HEAD`.chomp assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master", refname: commit_sha)) @@ -85,7 +85,7 @@ describe Crystal::Doc::ProjectInfo do it "git non-tagged commit dirty" do run_git "init" run_git "add shard.yml" - run_git "commit -m 'Initial commit' --no-gpg-sign" + run_git "commit -m \"Initial commit\" --no-gpg-sign" File.write("foo.txt", "bar") assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new("foo", "master-dev", refname: nil)) @@ -109,7 +109,7 @@ describe Crystal::Doc::ProjectInfo do File.write("foo.txt", "bar") run_git "init" run_git "add foo.txt" - run_git "commit -m 'Remove shard.yml' --no-gpg-sign" + run_git "commit -m \"Remove shard.yml\" --no-gpg-sign" run_git "tag v4.0" assert_with_defaults(ProjectInfo.new(nil, nil), ProjectInfo.new(nil, "4.0", refname: "v4.0")) @@ -130,7 +130,7 @@ describe Crystal::Doc::ProjectInfo do # Non-tagged commit File.write("file.txt", "foo") run_git "add file.txt" - run_git "commit -m 'Initial commit' --no-gpg-sign" + run_git "commit -m \"Initial commit\" --no-gpg-sign" ProjectInfo.find_git_version.should eq "master" # Other branch diff --git a/spec/compiler/crystal/tools/format_spec.cr b/spec/compiler/crystal/tools/format_spec.cr index b8a1a8c753a8..d0b37e866e29 100644 --- a/spec/compiler/crystal/tools/format_spec.cr +++ b/spec/compiler/crystal/tools/format_spec.cr @@ -99,8 +99,8 @@ describe Crystal::Command::FormatCommand do format_command = Crystal::Command::FormatCommand.new([] of String, color: false, stdin: stdin, stdout: stdout, stderr: stderr) format_command.run format_command.status_code.should eq(0) - stdout.to_s.should contain("Format ./format.cr") - stdout.to_s.should_not contain("Format ./not_format.cr") + stdout.to_s.should contain("Format #{Path[".", "format.cr"]}") + stdout.to_s.should_not contain("Format #{Path[".", "not_format.cr"]}") stderr.to_s.empty?.should be_true File.read(File.join(path, "format.cr")).should eq("if true\n 1\nend\n") @@ -124,8 +124,8 @@ describe Crystal::Command::FormatCommand do format_command = Crystal::Command::FormatCommand.new(["dir"], color: false, stdin: stdin, stdout: stdout, stderr: stderr) format_command.run format_command.status_code.should eq(0) - stdout.to_s.should contain("Format ./dir/format.cr") - stdout.to_s.should_not contain("Format ./dir/not_format.cr") + stdout.to_s.should contain("Format #{Path[".", "dir", "format.cr"]}") + stdout.to_s.should_not contain("Format #{Path[".", "dir", "not_format.cr"]}") stderr.to_s.empty?.should be_true {stdout, stderr}.each &.clear @@ -133,10 +133,10 @@ describe Crystal::Command::FormatCommand do format_command = Crystal::Command::FormatCommand.new([] of String, color: false, stdin: stdin, stdout: stdout, stderr: stderr) format_command.run format_command.status_code.should eq(0) - stdout.to_s.should contain("Format ./format.cr") - stdout.to_s.should_not contain("Format ./not_format.cr") - stdout.to_s.should_not contain("Format ./dir/format.cr") - stdout.to_s.should_not contain("Format ./dir/not_format.cr") + stdout.to_s.should contain("Format #{Path[".", "format.cr"]}") + stdout.to_s.should_not contain("Format #{Path[".", "not_format.cr"]}") + stdout.to_s.should_not contain("Format #{Path[".", "dir", "format.cr"]}") + stdout.to_s.should_not contain("Format #{Path[".", "dir", "not_format.cr"]}") stderr.to_s.empty?.should be_true File.read(File.join(path, "format.cr")).should eq("if true\n 1\nend\n") @@ -160,9 +160,9 @@ describe Crystal::Command::FormatCommand do format_command = Crystal::Command::FormatCommand.new([] of String, color: false, stdin: stdin, stdout: stdout, stderr: stderr) format_command.run format_command.status_code.should eq(1) - stdout.to_s.should contain("Format ./format.cr") - stderr.to_s.should contain("syntax error in './syntax_error.cr:1:3': unexpected token: EOF") - stderr.to_s.should contain("file './invalid_byte_sequence_error.cr' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") + stdout.to_s.should contain("Format #{Path[".", "format.cr"]}") + stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF") + stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") File.read(File.join(path, "format.cr")).should eq("if true\n 1\nend\n") end @@ -182,7 +182,7 @@ describe Crystal::Command::FormatCommand do format_command = BuggyFormatCommand.new([] of String, color: false, stdin: stdin, stdout: stdout, stderr: stderr) format_command.run format_command.status_code.should eq(1) - stderr.to_s.should contain("there's a bug formatting './empty.cr', to show more information, please run:\n\n $ crystal tool format --show-backtrace './empty.cr'") + stderr.to_s.should contain("there's a bug formatting '#{Path[".", "empty.cr"]}', to show more information, please run:\n\n $ crystal tool format --show-backtrace '#{Path[".", "empty.cr"]}'") end end end @@ -201,7 +201,7 @@ describe Crystal::Command::FormatCommand do format_command.run format_command.status_code.should eq(1) stderr.to_s.should contain("format command test") - stderr.to_s.should contain("couldn't format './empty.cr', please report a bug including the contents of it: https://github.com/crystal-lang/crystal/issues") + stderr.to_s.should contain("couldn't format '#{Path[".", "empty.cr"]}', please report a bug including the contents of it: https://github.com/crystal-lang/crystal/issues") end end end @@ -224,9 +224,9 @@ describe Crystal::Command::FormatCommand do format_command.status_code.should eq(1) stdout.to_s.empty?.should be_true stderr.to_s.should_not contain("not_format.cr") - stderr.to_s.should contain("formatting './format.cr' produced changes") - stderr.to_s.should contain("syntax error in './syntax_error.cr:1:3': unexpected token: EOF") - stderr.to_s.should contain("file './invalid_byte_sequence_error.cr' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") + stderr.to_s.should contain("formatting '#{Path[".", "format.cr"]}' produced changes") + stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF") + stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8") end end end @@ -286,7 +286,7 @@ describe Crystal::Command::FormatCommand do format_command.run format_command.status_code.should eq(1) stdout.to_s.empty?.should be_true - stderr.to_s.should contain("formatting './format.cr' produced changes") + stderr.to_s.should contain("formatting '#{Path[".", "format.cr"]}' produced changes") end end end diff --git a/spec/compiler/crystal/tools/init_spec.cr b/spec/compiler/crystal/tools/init_spec.cr index c16409e35dd1..7d59a6586282 100644 --- a/spec/compiler/crystal/tools/init_spec.cr +++ b/spec/compiler/crystal/tools/init_spec.cr @@ -280,7 +280,7 @@ module Crystal config.expanded_dir.should eq ::Path[Dir.current, "foo", "bar"] end - it "DIR (relative to home)" do + pending_win32 "DIR (relative to home)" do path = ::Path["~", "foo"].to_s config = Crystal::Init.parse_args(["lib", path]) config.name.should eq "foo" diff --git a/spec/compiler/crystal/tools/playground_spec.cr b/spec/compiler/crystal/tools/playground_spec.cr index 2e122ab8912c..4710f4fb0521 100644 --- a/spec/compiler/crystal/tools/playground_spec.cr +++ b/spec/compiler/crystal/tools/playground_spec.cr @@ -1,3 +1,5 @@ +{% skip_file if flag?(:without_playground) %} + require "../../../spec_helper" private def instrument(source) diff --git a/spec/compiler/crystal_path/crystal_path_spec.cr b/spec/compiler/crystal_path/crystal_path_spec.cr index cdcf5ba47256..390f91cd7c03 100644 --- a/spec/compiler/crystal_path/crystal_path_spec.cr +++ b/spec/compiler/crystal_path/crystal_path_spec.cr @@ -4,7 +4,7 @@ require "../../support/env" private def assert_finds(search, results, relative_to = nil, path = __DIR__, file = __FILE__, line = __LINE__) it "finds #{search.inspect}", file, line do crystal_path = Crystal::CrystalPath.new(path) - results = results.map { |result| File.join(__DIR__, result) } + results = results.map { |result| ::Path[__DIR__, result].normalize.to_s } Dir.cd(__DIR__) do matches = crystal_path.find search, relative_to: relative_to matches.should eq(results), file: file, line: line @@ -112,7 +112,7 @@ describe Crystal::CrystalPath do end it "overrides path with environment variable" do - with_env("CRYSTAL_PATH": "foo:bar") do + with_env("CRYSTAL_PATH": "foo#{Process::PATH_DELIMITER}bar") do crystal_path = Crystal::CrystalPath.new crystal_path.entries.should eq(%w(foo bar)) end diff --git a/spec/compiler/semantic/generic_class_spec.cr b/spec/compiler/semantic/generic_class_spec.cr index 93aac06adf11..3725488a727e 100644 --- a/spec/compiler/semantic/generic_class_spec.cr +++ b/spec/compiler/semantic/generic_class_spec.cr @@ -479,7 +479,7 @@ describe "Semantic: generic class" do "use a more specific type" end - it "errors on too nested generic instance" do + pending_win32 "errors on too nested generic instance" do assert_error %( class Foo(T) end @@ -493,7 +493,7 @@ describe "Semantic: generic class" do "generic type too nested" end - it "errors on too nested generic instance, with union type" do + pending_win32 "errors on too nested generic instance, with union type" do assert_error %( class Foo(T) end @@ -507,7 +507,7 @@ describe "Semantic: generic class" do "generic type too nested" end - it "errors on too nested tuple instance" do + pending_win32 "errors on too nested tuple instance" do assert_error %( def foo {typeof(foo)} diff --git a/spec/compiler/semantic/primitives_spec.cr b/spec/compiler/semantic/primitives_spec.cr index 9b9555b73363..fcf826540b96 100644 --- a/spec/compiler/semantic/primitives_spec.cr +++ b/spec/compiler/semantic/primitives_spec.cr @@ -224,7 +224,7 @@ describe "Semantic: primitives" do "method marked as Primitive must have an empty body" end - it "types va_arg primitive" do + pending_win32 "types va_arg primitive" do assert_type(%( struct VaList @[Primitive(:va_arg)] diff --git a/spec/std/exception/call_stack_spec.cr b/spec/std/exception/call_stack_spec.cr index ee42e4d77b5c..81ef86ad7333 100644 --- a/spec/std/exception/call_stack_spec.cr +++ b/spec/std/exception/call_stack_spec.cr @@ -1,7 +1,7 @@ require "../spec_helper" describe "Backtrace" do - it "prints file line:colunm" do + pending_win32 "prints file line:colunm" do source_file = datapath("backtrace_sample") # CallStack tries to make files relative to the current dir, @@ -34,7 +34,7 @@ describe "Backtrace" do error.to_s.should contain("IndexError") end - it "prints crash backtrace to stderr" do + pending_win32 "prints crash backtrace to stderr" do sample = datapath("crash_backtrace_sample") _, output, error = compile_and_run_file(sample) diff --git a/spec/std/io/file_descriptor_spec.cr b/spec/std/io/file_descriptor_spec.cr index 718a47acf5f6..b1ce0c33cdfd 100644 --- a/spec/std/io/file_descriptor_spec.cr +++ b/spec/std/io/file_descriptor_spec.cr @@ -1,7 +1,7 @@ require "../spec_helper" describe IO::FileDescriptor do - it "reopen STDIN with the right mode" do + pending_win32 "reopen STDIN with the right mode" do code = %q(puts "#{STDIN.blocking} #{STDIN.info.type}") compile_source(code) do |binpath| `#{Process.quote(binpath)} < #{Process.quote(binpath)}`.chomp.should eq("true File") diff --git a/spec/support/tempfile.cr b/spec/support/tempfile.cr index 42001785d1c8..036c161fcae4 100644 --- a/spec/support/tempfile.cr +++ b/spec/support/tempfile.cr @@ -28,7 +28,7 @@ def with_tempfile(*paths, file = __FILE__) ensure if SPEC_TEMPFILE_CLEANUP paths.each do |path| - FileUtils.rm_r(path) if File.exists?(path) + rm_rf(path) if File.exists?(path) end end end @@ -44,10 +44,15 @@ def with_temp_executable(name, file = __FILE__) end def with_temp_c_object_file(c_code, file = __FILE__) - with_tempfile("temp_c.c", "temp_c.o", file: file) do |c_filename, o_filename| + obj_ext = {{ flag?(:win32) ? ".obj" : ".o" }} + with_tempfile("temp_c.c", "temp_c#{obj_ext}", file: file) do |c_filename, o_filename| File.write(c_filename, c_code) - `#{ENV["CC"]? || "cc"} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy + {% if flag?(:win32) %} + `cl.exe /nologo /c #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_filename}")}`.should be_truthy + {% else %} + `#{ENV["CC"]? || "cc"} #{Process.quote(c_filename)} -c -o #{Process.quote(o_filename)}`.should be_truthy + {% end %} yield o_filename end @@ -55,6 +60,24 @@ end if SPEC_TEMPFILE_CLEANUP at_exit do - FileUtils.rm_r(SPEC_TEMPFILE_PATH) if Dir.exists?(SPEC_TEMPFILE_PATH) + rm_rf(SPEC_TEMPFILE_PATH) if Dir.exists?(SPEC_TEMPFILE_PATH) + end +end + +private def rm_rf(path : String) : Nil + if Dir.exists?(path) && !File.symlink?(path) + Dir.each_child(path) do |entry| + src = File.join(path, entry) + rm_rf(src) + end + Dir.delete(path) + else + begin + File.delete(path) + rescue File::AccessDeniedError + # To be able to delete read-only files (e.g. ones under .git/) on Windows. + File.chmod(path, 0o666) + File.delete(path) + end end end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index 2bc3be978063..e8b420a6dbf6 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -55,7 +55,7 @@ require "./std/ecr/ecr_spec.cr" require "./std/enum_spec.cr" require "./std/enumerable_spec.cr" require "./std/env_spec.cr" -# require "./std/exception/call_stack_spec.cr" +require "./std/exception/call_stack_spec.cr" require "./std/exception_spec.cr" require "./std/file/tempfile_spec.cr" require "./std/file_spec.cr" @@ -100,7 +100,7 @@ require "./std/io/argf_spec.cr" require "./std/io/buffered_spec.cr" require "./std/io/byte_format_spec.cr" require "./std/io/delimited_spec.cr" -# require "./std/io/file_descriptor_spec.cr" (failed codegen) +require "./std/io/file_descriptor_spec.cr" require "./std/io/hexdump_spec.cr" require "./std/io/io_spec.cr" require "./std/io/memory_spec.cr" diff --git a/src/empty.cr b/src/empty.cr index 8a78751e0f85..35f988afb583 100644 --- a/src/empty.cr +++ b/src/empty.cr @@ -1,5 +1,8 @@ require "primitives" +{% if flag?(:win32) %} + @[Link("libcmt")] # For `mainCRTStartup` +{% end %} lib LibCrystalMain @[Raises] fun __crystal_main(argc : Int32, argv : UInt8**) From e75e42920162e17a6de7c820796f87f343a1742d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Tue, 23 Jun 2020 10:20:48 +0200 Subject: [PATCH 148/263] Fix type of HTTP::Server::Response#closed? to Bool (#9489) --- src/http/server/response.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http/server/response.cr b/src/http/server/response.cr index c0e0a18d3b42..5662656746de 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -175,6 +175,7 @@ class HTTP::Server def initialize(@io) @chunked = false + @closed = false end def reset @@ -215,7 +216,7 @@ class HTTP::Server raise ClientError.new("Error while writing data to the client", ex) end - def closed? + def closed? : Bool @closed end From 8b6e62efebebcb88a7e7b7dd8515ca286b1e207b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 23 Jun 2020 11:12:45 -0300 Subject: [PATCH 149/263] Drop deprecated CRC32, Adler32 top-level (#9530) Use Digest::CRC32, Digest::Adler32 --- spec/std/adler32_spec.cr | 16 ---------------- spec/std/crc32_spec.cr | 16 ---------------- spec/win32_std_spec.cr | 2 -- src/adler32/adler32.cr | 23 ----------------------- src/crc32/crc32.cr | 23 ----------------------- src/docs_main.cr | 2 -- 6 files changed, 82 deletions(-) delete mode 100644 spec/std/adler32_spec.cr delete mode 100644 spec/std/crc32_spec.cr delete mode 100644 src/adler32/adler32.cr delete mode 100644 src/crc32/crc32.cr diff --git a/spec/std/adler32_spec.cr b/spec/std/adler32_spec.cr deleted file mode 100644 index 450ee398d3d8..000000000000 --- a/spec/std/adler32_spec.cr +++ /dev/null @@ -1,16 +0,0 @@ -require "spec" -require "adler32" - -describe Adler32 do - it "should be able to calculate adler32" do - adler = Adler32.checksum("foo").to_s(16) - adler.should eq("2820145") - end - - it "should be able to calculate adler32 combined" do - adler1 = Adler32.checksum("hello") - adler2 = Adler32.checksum(" world!") - combined = Adler32.combine(adler1, adler2, " world!".size) - Adler32.checksum("hello world!").should eq(combined) - end -end diff --git a/spec/std/crc32_spec.cr b/spec/std/crc32_spec.cr deleted file mode 100644 index cfc5ef1b0853..000000000000 --- a/spec/std/crc32_spec.cr +++ /dev/null @@ -1,16 +0,0 @@ -require "spec" -require "crc32" - -describe CRC32 do - it "should be able to calculate crc32" do - crc = CRC32.checksum("foo").to_s(16) - crc.should eq("8c736521") - end - - it "should be able to calculate crc32 combined" do - crc1 = CRC32.checksum("hello") - crc2 = CRC32.checksum(" world!") - combined = CRC32.combine(crc1, crc2, " world!".size) - CRC32.checksum("hello world!").should eq(combined) - end -end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index e8b420a6dbf6..af72b2e6bfc3 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -1,7 +1,6 @@ # This file is autogenerated by `scripts/generate_windows_spec.sh` # 2020-04-20 22:36:41+02:00 -require "./std/adler32_spec.cr" require "./std/array_spec.cr" require "./std/atomic_spec.cr" require "./std/base64_spec.cr" @@ -30,7 +29,6 @@ require "./std/compress/zlib/stress_spec.cr" require "./std/compress/zlib/writer_spec.cr" # require "./std/concurrent/select_spec.cr" (failed to run) require "./std/concurrent_spec.cr" -require "./std/crc32_spec.cr" require "./std/crypto/bcrypt/base64_spec.cr" require "./std/crypto/bcrypt/password_spec.cr" require "./std/crypto/bcrypt_spec.cr" diff --git a/src/adler32/adler32.cr b/src/adler32/adler32.cr deleted file mode 100644 index 43a9737d332d..000000000000 --- a/src/adler32/adler32.cr +++ /dev/null @@ -1,23 +0,0 @@ -require "digest" - -module Adler32 - @[Deprecated("Use `Digest::Adler32.initial` instead")] - def self.initial : UInt32 - Digest::Adler32.initial - end - - @[Deprecated("Use `Digest::Adler32.checksum` instead")] - def self.checksum(data) : UInt32 - Digest::Adler32.checksum(data) - end - - @[Deprecated("Use `Digest::Adler32.update` instead")] - def self.update(data, adler32 : UInt32) : UInt32 - Digest::Adler32.update(data, adler32) - end - - @[Deprecated("Use `Digest::Adler32.combine` instead")] - def self.combine(adler1 : UInt32, adler2 : UInt32, len) : UInt32 - Digest::Adler32.combine(adler1, adler2, len) - end -end diff --git a/src/crc32/crc32.cr b/src/crc32/crc32.cr deleted file mode 100644 index 7af25e8a54ea..000000000000 --- a/src/crc32/crc32.cr +++ /dev/null @@ -1,23 +0,0 @@ -require "digest" - -module CRC32 - @[Deprecated("Use `Digest::CRC32.initial` instead")] - def self.initial : UInt32 - Digest::CRC32.initial - end - - @[Deprecated("Use `Digest::CRC32.checksum` instead")] - def self.checksum(data) : UInt32 - Digest::CRC32.checksum(data) - end - - @[Deprecated("Use `Digest::CRC32.update` instead")] - def self.update(data, crc32 : UInt32) : UInt32 - Digest::CRC32.update(data, crc32) - end - - @[Deprecated("Use `Digest::CRC32.combine` instead")] - def self.combine(crc1 : UInt32, crc2 : UInt32, len) : UInt32 - Digest::CRC32.combine(crc1, crc2, len) - end -end diff --git a/src/docs_main.cr b/src/docs_main.cr index d65a12d47046..d8645ca03e4a 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -32,13 +32,11 @@ require "./weak_ref" require "./xml" require "./yaml" require "./benchmark" -require "./adler32" require "./array" require "./bit_array" require "./box" require "./colorize" require "./complex" -require "./crc32" require "./deque" require "./file_utils" require "./flate" From 4d7e5851657907704cea9ae4714a2e3d5afa5230 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 23 Jun 2020 11:13:20 -0300 Subject: [PATCH 150/263] Drop deprecated Flate, Gzip, Zip, Zlib top-level (#9529) Use Compress::Deflate, Compress::Gzip, Comperss::Zip, Compress::Zlib --- spec/std/flate/flate_spec.cr | 106 ---------------- spec/std/gzip/gzip_spec.cr | 60 ---------- spec/std/zip/zip_file_spec.cr | 132 -------------------- spec/std/zip/zip_spec.cr | 219 ---------------------------------- spec/std/zlib/reader_spec.cr | 88 -------------- spec/std/zlib/stress_spec.cr | 36 ------ spec/std/zlib/writer_spec.cr | 71 ----------- spec/win32_std_spec.cr | 7 -- src/docs_main.cr | 4 - src/flate.cr | 6 - src/gzip.cr | 6 - src/zip.cr | 6 - src/zlib.cr | 6 - 13 files changed, 747 deletions(-) delete mode 100644 spec/std/flate/flate_spec.cr delete mode 100644 spec/std/gzip/gzip_spec.cr delete mode 100644 spec/std/zip/zip_file_spec.cr delete mode 100644 spec/std/zip/zip_spec.cr delete mode 100644 spec/std/zlib/reader_spec.cr delete mode 100644 spec/std/zlib/stress_spec.cr delete mode 100644 spec/std/zlib/writer_spec.cr delete mode 100644 src/flate.cr delete mode 100644 src/gzip.cr delete mode 100644 src/zip.cr delete mode 100644 src/zlib.cr diff --git a/spec/std/flate/flate_spec.cr b/spec/std/flate/flate_spec.cr deleted file mode 100644 index f2071b571d76..000000000000 --- a/spec/std/flate/flate_spec.cr +++ /dev/null @@ -1,106 +0,0 @@ -require "spec" -require "flate" - -private def new_sample_io - io = IO::Memory.new - "cbc9cc4b350402ae1c20c30808b800".scan(/../).each do |match| - io.write_byte match[0].to_u8(16) - end - io.rewind -end - -module Flate - describe Reader do - it "should read byte by byte (#4192)" do - io = new_sample_io - - reader = Reader.new(io) - - str = String::Builder.build do |builder| - while b = reader.read_byte - builder.write_byte b - end - end - - str.should eq("line1111\nline2222\n") - end - - it "should rewind" do - io = new_sample_io - - reader = Reader.new(io) - reader.gets.should eq("line1111") - reader.rewind - reader.gets_to_end.should eq("line1111\nline2222\n") - end - - describe ".open" do - it "yields itself to block" do - # Hello Crystal! - message = Bytes[243, 72, 205, 201, 201, 87, 112, 46, 170, 44, 46, 73, - 204, 81, 4, 0] - - io = IO::Memory.new(message) - Reader.open(io) do |reader| - reader.gets_to_end.should eq("Hello Crystal!") - end - end - end - end - - describe Writer do - it "should be able to write" do - message = "this is a test string !!!!\n" - io = IO::Memory.new - writer = Writer.new(io) - writer.print message - writer.close - - io.rewind - reader = Reader.new(io) - reader.gets_to_end.should eq(message) - end - - it "can be closed without sync" do - io = IO::Memory.new - writer = Writer.new(io) - writer.close - writer.closed?.should be_true - io.closed?.should be_false - - expect_raises IO::Error, "Closed stream" do - writer.print "a" - end - end - - it "can be closed with sync (1)" do - io = IO::Memory.new - writer = Writer.new(io, sync_close: true) - writer.close - writer.closed?.should be_true - io.closed?.should be_true - end - - it "can be closed with sync (2)" do - io = IO::Memory.new - writer = Writer.new(io) - writer.sync_close = true - writer.close - writer.closed?.should be_true - io.closed?.should be_true - end - - describe ".open" do - it "yields itself to block" do - io = IO::Memory.new - Writer.open(io) do |writer| - writer.write "Hello Crystal!".to_slice - end - - io.rewind - io.to_slice.should eq(Bytes[243, 72, 205, 201, 201, 87, 112, 46, 170, 44, 46, 73, - 204, 81, 4, 0]) - end - end - end -end diff --git a/spec/std/gzip/gzip_spec.cr b/spec/std/gzip/gzip_spec.cr deleted file mode 100644 index 249855d8c899..000000000000 --- a/spec/std/gzip/gzip_spec.cr +++ /dev/null @@ -1,60 +0,0 @@ -require "spec" -require "gzip" - -private SAMPLE_TIME = Time.utc(2016, 1, 2) -private SAMPLE_OS = 4_u8 -private SAMPLE_EXTRA = Bytes[1, 2, 3] -private SAMPLE_NAME = "foo.txt" -private SAMPLE_COMMENT = "some comment" -private SAMPLE_CONTENTS = "hello world\nfoo bar" - -private def new_sample_io - io = IO::Memory.new - - Gzip::Writer.open(io) do |gzip| - header = gzip.header - header.modification_time = SAMPLE_TIME - header.os = SAMPLE_OS - header.extra = SAMPLE_EXTRA - header.name = SAMPLE_NAME - header.comment = SAMPLE_COMMENT - - io.bytesize.should eq(0) - gzip.flush - io.bytesize.should_not eq(0) - - gzip.print SAMPLE_CONTENTS - end - - io.rewind -end - -describe Gzip do - it "writes and reads to memory" do - io = new_sample_io - - Gzip::Reader.open(io) do |gzip| - header = gzip.header.not_nil! - header.modification_time.should eq(SAMPLE_TIME) - header.os.should eq(SAMPLE_OS) - header.extra.should eq(SAMPLE_EXTRA) - header.name.should eq(SAMPLE_NAME) - header.comment.should eq(SAMPLE_COMMENT) - - # Reading zero bytes is OK - gzip.read(Bytes.empty).should eq(0) - - gzip.gets_to_end.should eq(SAMPLE_CONTENTS) - end - end - - it "rewinds" do - io = new_sample_io - - gzip = Gzip::Reader.new(io) - gzip.gets.should eq(SAMPLE_CONTENTS.lines.first) - - gzip.rewind - gzip.gets_to_end.should eq(SAMPLE_CONTENTS) - end -end diff --git a/spec/std/zip/zip_file_spec.cr b/spec/std/zip/zip_file_spec.cr deleted file mode 100644 index 7558d101479a..000000000000 --- a/spec/std/zip/zip_file_spec.cr +++ /dev/null @@ -1,132 +0,0 @@ -require "../spec_helper" -require "zip" - -describe Zip do - it "reads file from memory" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add "foo.txt", "contents of foo" - zip.add "bar.txt", "contents of bar" - end - - io.rewind - - Zip::File.open(io) do |zip| - entries = zip.entries - entries.size.should eq(2) - - foo = entries[0] - foo.filename.should eq("foo.txt") - - bar = entries[1] - bar.filename.should eq("bar.txt") - - zip["foo.txt"].filename.should eq("foo.txt") - zip["bar.txt"].filename.should eq("bar.txt") - zip["baz.txt"]?.should be_nil - - foo.open do |foo_io| - bar.open do |bar_io| - foo_io.gets_to_end.should eq("contents of foo") - bar_io.gets_to_end.should eq("contents of bar") - end - end - end - end - - it "reads file from file system" do - filename = datapath("file.zip") - - begin - File.open(filename, "w") do |file| - Zip::Writer.open(file) do |zip| - zip.add "foo.txt", "contents of foo" - zip.add "bar.txt", "contents of bar" - end - end - - File.open(filename, "r") do |file| - Zip::File.open(file) do |zip| - entries = zip.entries - entries.size.should eq(2) - - foo = entries[0] - foo.filename.should eq("foo.txt") - - bar = entries[1] - bar.filename.should eq("bar.txt") - - zip["foo.txt"].filename.should eq("foo.txt") - zip["bar.txt"].filename.should eq("bar.txt") - zip["baz.txt"]?.should be_nil - - foo.open do |foo_io| - bar.open do |bar_io| - foo_io.gets_to_end.should eq("contents of foo") - bar_io.gets_to_end.should eq("contents of bar") - end - end - end - end - ensure - File.delete(filename) - end - end - - it "writes comment" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add Zip::Writer::Entry.new("foo.txt", comment: "some comment"), - "contents of foo" - end - - io.rewind - - Zip::File.open(io) do |zip| - zip["foo.txt"].comment.should eq("some comment") - end - end - - it "reads big file" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - 100.times do |i| - zip.add "foo#{i}.txt", "some contents #{i}" - end - end - - io.rewind - - Zip::File.open(io) do |zip| - zip.entries.size.should eq(100) - end - end - - it "reads zip file with different extra in local file header and central directory header" do - Zip::File.open(datapath("test.zip")) do |zip| - zip.entries.size.should eq(2) - zip["one.txt"].open(&.gets_to_end).should eq("One") - zip["two.txt"].open(&.gets_to_end).should eq("Two") - end - end - - it "reads zip comment" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.comment = "zip comment" - end - - io.rewind - - Zip::File.open(io) do |zip| - zip.comment.should eq("zip comment") - end - end - - typeof(Zip::File.new("file.zip")) - typeof(Zip::File.open("file.zip") { }) -end diff --git a/spec/std/zip/zip_spec.cr b/spec/std/zip/zip_spec.cr deleted file mode 100644 index fdb88c7ad5fa..000000000000 --- a/spec/std/zip/zip_spec.cr +++ /dev/null @@ -1,219 +0,0 @@ -require "../spec_helper" -require "zip" - -describe Zip do - it "writes and reads to memory" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add "foo.txt", &.print("contents of foo") - zip.add "bar.txt", &.print("contents of bar") - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.file?.should be_true - entry.dir?.should be_false - entry.filename.should eq("foo.txt") - entry.compression_method.should eq(Zip::CompressionMethod::DEFLATED) - entry.crc32.should eq(0) - entry.compressed_size.should eq(0) - entry.uncompressed_size.should eq(0) - entry.extra.empty?.should be_true - entry.io.gets_to_end.should eq("contents of foo") - - entry = zip.next_entry.not_nil! - entry.filename.should eq("bar.txt") - entry.io.gets_to_end.should eq("contents of bar") - - zip.next_entry.should be_nil - end - end - - it "writes entry" do - io = IO::Memory.new - - time = Time.utc(2017, 1, 14, 2, 3, 4) - extra = Bytes[1, 2, 3, 4] - - Zip::Writer.open(io) do |zip| - zip.add(Zip::Writer::Entry.new("foo.txt", time: time, extra: extra)) do |io| - io.print("contents of foo") - end - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.time.should eq(time) - entry.extra.should eq(extra) - entry.io.gets_to_end.should eq("contents of foo") - end - end - - it "writes entry uncompressed" do - io = IO::Memory.new - - text = "contents of foo" - crc32 = Digest::CRC32.checksum(text) - - Zip::Writer.open(io) do |zip| - entry = Zip::Writer::Entry.new("foo.txt") - entry.compression_method = Zip::CompressionMethod::STORED - entry.crc32 = crc32 - entry.compressed_size = text.bytesize.to_u32 - entry.uncompressed_size = text.bytesize.to_u32 - zip.add entry, &.print(text) - - entry = Zip::Writer::Entry.new("bar.txt") - entry.compression_method = Zip::CompressionMethod::STORED - entry.crc32 = crc32 - entry.compressed_size = text.bytesize.to_u32 - entry.uncompressed_size = text.bytesize.to_u32 - zip.add entry, &.print(text) - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.compression_method.should eq(Zip::CompressionMethod::STORED) - entry.crc32.should eq(crc32) - entry.compressed_size.should eq(text.bytesize) - entry.uncompressed_size.should eq(text.bytesize) - entry.io.gets_to_end.should eq(text) - - entry = zip.next_entry.not_nil! - entry.filename.should eq("bar.txt") - entry.io.gets_to_end.should eq(text) - end - end - - it "writes entry uncompressed and reads with Zip::File" do - io = IO::Memory.new - - text = "contents of foo" - crc32 = Digest::CRC32.checksum(text) - - Zip::Writer.open(io) do |zip| - entry = Zip::Writer::Entry.new("foo.txt") - entry.compression_method = Zip::CompressionMethod::STORED - entry.crc32 = crc32 - entry.compressed_size = text.bytesize.to_u32 - entry.uncompressed_size = text.bytesize.to_u32 - zip.add entry, &.print(text) - end - - io.rewind - - Zip::File.open(io) do |zip| - zip.entries.size.should eq(1) - entry = zip.entries.first - entry.filename.should eq("foo.txt") - entry.open(&.gets_to_end).should eq(text) - end - end - - it "adds a directory" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add_dir "one" - zip.add_dir "two/" - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("one/") - entry.file?.should be_false - entry.dir?.should be_true - entry.io.gets_to_end.should eq("") - - entry = zip.next_entry.not_nil! - entry.filename.should eq("two/") - entry.dir?.should be_true - entry.io.gets_to_end.should eq("") - end - end - - it "writes string" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add "foo.txt", "contents of foo" - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.io.gets_to_end.should eq("contents of foo") - end - end - - it "writes bytes" do - io = IO::Memory.new - - Zip::Writer.open(io) do |zip| - zip.add "foo.txt", "contents of foo".to_slice - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.io.gets_to_end.should eq("contents of foo") - end - end - - it "writes io" do - io = IO::Memory.new - data = IO::Memory.new("contents of foo") - - Zip::Writer.open(io) do |zip| - zip.add "foo.txt", data - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.io.gets_to_end.should eq("contents of foo") - end - end - - it "writes file" do - io = IO::Memory.new - filename = datapath("test_file.txt") - - Zip::Writer.open(io) do |zip| - file = File.open(filename) - zip.add "foo.txt", file - file.closed?.should be_true - end - - io.rewind - - Zip::Reader.open(io) do |zip| - entry = zip.next_entry.not_nil! - entry.filename.should eq("foo.txt") - entry.io.gets_to_end.should eq(File.read(filename)) - end - end - - typeof(Zip::Reader.new("file.zip")) - typeof(Zip::Reader.open("file.zip") { }) - - typeof(Zip::Writer.new("file.zip")) - typeof(Zip::Writer.open("file.zip") { }) -end diff --git a/spec/std/zlib/reader_spec.cr b/spec/std/zlib/reader_spec.cr deleted file mode 100644 index 23a1953ff169..000000000000 --- a/spec/std/zlib/reader_spec.cr +++ /dev/null @@ -1,88 +0,0 @@ -require "spec" -require "zlib" - -private def new_sample_io - io = IO::Memory.new - "789c2bc9c82c5600a2448592d4e21285e292a2ccbc74054520e00200854f087b".scan(/../).each do |match| - io.write_byte match[0].to_u8(16) - end - io.rewind -end - -module Zlib - describe Reader do - it "should be able to read" do - io = new_sample_io - - reader = Reader.new(io) - - str = String::Builder.build do |builder| - IO.copy(reader, builder) - end - - str.should eq("this is a test string !!!!\n") - reader.read(Bytes.new(10)).should eq(0) - end - - it "rewinds" do - io = new_sample_io - - reader = Reader.new(io) - reader.gets(3).should eq("thi") - reader.rewind - reader.gets_to_end.should eq("this is a test string !!!!\n") - end - - it "can be closed without sync" do - io = IO::Memory.new(Bytes[120, 156, 3, 0, 0, 0, 0, 1]) - reader = Reader.new(io) - reader.close - reader.closed?.should be_true - io.closed?.should be_false - - expect_raises IO::Error, "Closed stream" do - reader.gets - end - end - - it "can be closed with sync (1)" do - io = IO::Memory.new(Bytes[120, 156, 3, 0, 0, 0, 0, 1]) - reader = Reader.new(io, sync_close: true) - reader.close - reader.closed?.should be_true - io.closed?.should be_true - end - - it "can be closed with sync (2)" do - io = IO::Memory.new(Bytes[120, 156, 3, 0, 0, 0, 0, 1]) - reader = Reader.new(io) - reader.sync_close = true - reader.close - reader.closed?.should be_true - io.closed?.should be_true - end - - it "should not read from empty stream" do - io = IO::Memory.new(Bytes[120, 156, 3, 0, 0, 0, 0, 1]) - reader = Reader.new(io) - reader.read_byte.should be_nil - end - - it "should not freeze when reading empty slice" do - io = new_sample_io - reader = Reader.new(io) - slice = Bytes.empty - reader.read(slice).should eq(0) - end - - it "should raise buffer error on error (#6575)" do - io = IO::Memory.new("x\x9C4\xC9\xD1\n@@\u0010\u0005\xD0\u007F\xB9ϻeEj~E\xD2`B\xAD\xA55H\x9B\u007F\xE7\xC5۩\x93\xA0\xA0pxo\xB0\xFFX7\x90\xCB\f\u0006P\xC2$\u001C\xB5\u0013\xD6v\u000E*\xF1d\u000F*\\^~\xDFj\xE4^@5FV\xB9\xF8\xB6[\u001C\xEC\xC2s\xB0\x99\xD3\n\xCD\xF3\xBC\u0000\u0000\u0000\xFF\xFF") - - reader = Reader.new(io) - - expect_raises(Compress::Deflate::Error, "deflate: buffer error") do - reader.gets_to_end - end - end - end -end diff --git a/spec/std/zlib/stress_spec.cr b/spec/std/zlib/stress_spec.cr deleted file mode 100644 index dd0005c8b55e..000000000000 --- a/spec/std/zlib/stress_spec.cr +++ /dev/null @@ -1,36 +0,0 @@ -require "spec" -require "zlib" - -module Zlib - describe Zlib do - it "write read should be inverse with random string" do - expected = String.build do |io| - 1_000_000.times { rand(2000).to_i.to_s(32, io) } - end - - io = IO::Memory.new - - writer = Writer.new(io) - writer.print expected - writer.close - - io.rewind - reader = Reader.new(io) - reader.gets_to_end.should eq(expected) - end - - it "write read should be inverse (utf-8)" do - expected = "日本さん語日本さん語" - - io = IO::Memory.new - - writer = Writer.new(io) - writer.print expected - writer.close - - io.rewind - reader = Reader.new(io) - reader.gets_to_end.should eq(expected) - end - end -end diff --git a/spec/std/zlib/writer_spec.cr b/spec/std/zlib/writer_spec.cr deleted file mode 100644 index 3966a3425678..000000000000 --- a/spec/std/zlib/writer_spec.cr +++ /dev/null @@ -1,71 +0,0 @@ -require "spec" -require "zlib" - -module Zlib - describe Writer do - it "should be able to write" do - message = "this is a test string !!!!\n" - io = IO::Memory.new - - writer = Writer.new(io) - - io.bytesize.should eq(0) - writer.flush - io.bytesize.should_not eq(0) - - writer.print message - writer.close - - io.rewind - reader = Reader.new(io) - reader.gets_to_end.should eq(message) - end - - it "can be closed without sync" do - io = IO::Memory.new - writer = Writer.new(io) - writer.close - writer.closed?.should be_true - io.closed?.should be_false - - expect_raises IO::Error, "Closed stream" do - writer.print "a" - end - end - - it "can be closed with sync (1)" do - io = IO::Memory.new - writer = Writer.new(io, sync_close: true) - writer.close - writer.closed?.should be_true - io.closed?.should be_true - end - - it "can be closed with sync (2)" do - io = IO::Memory.new - writer = Writer.new(io) - writer.sync_close = true - writer.close - writer.closed?.should be_true - io.closed?.should be_true - end - - it "can be flushed" do - io = IO::Memory.new - writer = Writer.new(io) - - writer.print "this" - io.to_slice.hexstring.should eq("789c") - - writer.flush - (io.to_slice.hexstring.size > 4).should be_true - - writer.print " is a test string !!!!\n" - writer.close - - io.rewind - reader = Reader.new(io) - reader.gets_to_end.should eq("this is a test string !!!!\n") - end - end -end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index af72b2e6bfc3..16e912becd47 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -58,14 +58,12 @@ require "./std/exception_spec.cr" require "./std/file/tempfile_spec.cr" require "./std/file_spec.cr" # require "./std/file_utils_spec.cr" -require "./std/flate/flate_spec.cr" require "./std/float_printer/diy_fp_spec.cr" require "./std/float_printer/grisu3_spec.cr" require "./std/float_printer/ieee_spec.cr" require "./std/float_printer_spec.cr" require "./std/float_spec.cr" require "./std/gc_spec.cr" -require "./std/gzip/gzip_spec.cr" require "./std/hash_spec.cr" require "./std/html_spec.cr" require "./std/http/chunked_content_spec.cr" @@ -236,8 +234,3 @@ require "./std/yaml/serializable_spec.cr" require "./std/yaml/serialization_spec.cr" require "./std/yaml/yaml_pull_parser_spec.cr" require "./std/yaml/yaml_spec.cr" -require "./std/zip/zip_file_spec.cr" -require "./std/zip/zip_spec.cr" -require "./std/zlib/reader_spec.cr" -require "./std/zlib/stress_spec.cr" -require "./std/zlib/writer_spec.cr" diff --git a/src/docs_main.cr b/src/docs_main.cr index d8645ca03e4a..57d80d4fbc85 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -39,8 +39,6 @@ require "./colorize" require "./complex" require "./deque" require "./file_utils" -require "./flate" -require "./gzip" require "./ini" require "./levenshtein" require "./log" @@ -56,7 +54,5 @@ require "./unicode/unicode" require "./uri" require "./uuid" require "./uuid/json" -require "./zip" -require "./zlib" require "./system/*" require "./docs_pseudo_methods" diff --git a/src/flate.cr b/src/flate.cr deleted file mode 100644 index 1f69693c26cd..000000000000 --- a/src/flate.cr +++ /dev/null @@ -1,6 +0,0 @@ -require "compress/deflate" - -{% puts "Warning: Flate is deprecated, use Compress::Deflate" %} - -# DEPRECATED: Use `Compress::Deflate` -alias Flate = Compress::Deflate diff --git a/src/gzip.cr b/src/gzip.cr deleted file mode 100644 index 74e6210f0060..000000000000 --- a/src/gzip.cr +++ /dev/null @@ -1,6 +0,0 @@ -require "compress/gzip" - -{% puts "Warning: Gzip is deprecated, use Compress::Gzip" %} - -# DEPRECATED: Use `Compress::Gzip` -alias Gzip = Compress::Gzip diff --git a/src/zip.cr b/src/zip.cr deleted file mode 100644 index a8c3459f3fba..000000000000 --- a/src/zip.cr +++ /dev/null @@ -1,6 +0,0 @@ -require "compress/zip" - -{% puts "Warning: Zip is deprecated, use Compress::Zip" %} - -# DEPRECATED: Use `Compress::Zip` -alias Zip = Compress::Zip diff --git a/src/zlib.cr b/src/zlib.cr deleted file mode 100644 index d8e10e0c58fe..000000000000 --- a/src/zlib.cr +++ /dev/null @@ -1,6 +0,0 @@ -require "compress/zlib" - -{% puts "Warning: Zlib is deprecated, use Compress::Zlib" %} - -# DEPRECATED: Use `Compress::Zlib` -alias Zlib = Compress::Zlib From 0618737fed29b16ae9220febd0a8f7a26f4f35f0 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 23 Jun 2020 11:32:47 -0300 Subject: [PATCH 151/263] Makefile: support changing the current crystal via env var and argument (#9471) * Makefile: support changing the current crystal via env var and argument Allow bin/crystal and makefile to not require installed crystal in path by reading CRYSTAL env var or makefile argument make CRYSTAL=path/to/previous/compiler * Propagate CRYSTAL env variable NB: .build/crystal still has precedence --- Makefile | 3 ++- bin/crystal | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a764d6dba47f..76b95cb1d032 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,8 @@ ## Run all specs in verbose mode ## $ make spec verbose=1 -LLVM_CONFIG ?= ## llvm-config command path to use +CRYSTAL ?= crystal ## which previous crystal compiler use +LLVM_CONFIG ?= ## llvm-config command path to use release ?= ## Compile in release mode stats ?= ## Enable statistics output diff --git a/bin/crystal b/bin/crystal index d12f72f5aca7..7d2e3ebf1048 100755 --- a/bin/crystal +++ b/bin/crystal @@ -141,6 +141,8 @@ CRYSTAL_DIR="$CRYSTAL_ROOT/.build" export CRYSTAL_PATH=lib:$CRYSTAL_ROOT/src export CRYSTAL_HAS_WRAPPER=true +export CRYSTAL="${CRYSTAL:-"crystal"}" + if [ -z "$CRYSTAL_CONFIG_LIBRARY_PATH" ]; then export CRYSTAL_CONFIG_LIBRARY_PATH="$( export PATH="$(remove_path_item "$(remove_path_item "$PATH" "$SCRIPT_ROOT")" "bin")" @@ -151,12 +153,12 @@ fi if [ -x "$CRYSTAL_DIR/crystal" ]; then __warning_msg "Using compiled compiler at ${CRYSTAL_DIR#"$PWD/"}/crystal" exec "$CRYSTAL_DIR/crystal" "$@" -elif ! command -v crystal > /dev/null; then - __error_msg 'You need to have a crystal executable in your path!' +elif ! command -v $CRYSTAL > /dev/null; then + __error_msg 'You need to have a crystal executable in your path! or set CRYSTAL env variable' exit 1 -elif [ "$(command -v crystal)" = "$SCRIPT_PATH" ] || [ "$(command -v crystal)" = "bin/crystal" ]; then +elif [ "$(command -v $CRYSTAL)" = "$SCRIPT_PATH" ] || [ "$(command -v $CRYSTAL)" = "bin/crystal" ]; then export PATH="$(remove_path_item "$(remove_path_item "$PATH" "$SCRIPT_ROOT")" "bin")" exec "$SCRIPT_PATH" "$@" else - exec crystal "$@" + exec $CRYSTAL "$@" fi From 4482942a1c000516a3521714a18d1422d637a9fd Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 23 Jun 2020 23:34:17 +0900 Subject: [PATCH 152/263] Fix `CRYSTAL_OPTS` parsing. Add `Process.parse_arguments` (#9518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix `CRYSTAL_OPTS` parsing Fixed #9509 Ref https://github.com/crystal-lang/crystal/pull/8900#discussion_r391925011 This commit adds `Process.split`, which works as like `Shellwords.split` in Ruby. And, this is used to parse `CRYSTAL_OPTS`. I'm not sure `Process.split` is good naming. However I have no idea of better name. And also, I'm not sure Windows version of `Process.split` is needed. * Rename `Process.split` to `parse_arguments` * Use `in?` instead of `includes?` Co-authored-by: Sijawusz Pur Rahnama * Let `Process.parse_arguments` raise an error against unclosed quote This also fix backslash behavior in double quote as POSIX. * Handle error on `CRYSTAL_OPTS` parsing Co-authored-by: Jonne Haß * Remove debug output from example Co-authored-by: Brian J. Cardiff * Fix method name in example Co-authored-by: Sijawusz Pur Rahnama Co-authored-by: Jonne Haß Co-authored-by: Brian J. Cardiff --- spec/std/process_spec.cr | 26 +++++++++++++++ src/compiler/crystal/command.cr | 4 ++- src/process/shell.cr | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 22934b119975..968fc2559aa1 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -460,4 +460,30 @@ describe Process do it { Process.quote_windows(["foo ", "", " ", " bar"]).should eq %("foo " "" " " " bar") } end end + + describe "parse_arguments" do + it { Process.parse_arguments("").should eq(%w[]) } + it { Process.parse_arguments(" ").should eq(%w[]) } + it { Process.parse_arguments("foo").should eq(%w[foo]) } + it { Process.parse_arguments("foo bar").should eq(%w[foo bar]) } + it { Process.parse_arguments(%q("foo bar" 'foo bar' baz)).should eq(["foo bar", "foo bar", "baz"]) } + it { Process.parse_arguments(%q("foo bar"'foo bar'baz)).should eq(["foo barfoo barbaz"]) } + it { Process.parse_arguments(%q(foo\ bar)).should eq(["foo bar"]) } + it { Process.parse_arguments(%q("foo\ bar")).should eq(["foo\\ bar"]) } + it { Process.parse_arguments(%q('foo\ bar')).should eq(["foo\\ bar"]) } + it { Process.parse_arguments("\\").should eq(["\\"]) } + it { Process.parse_arguments(%q["foo bar" '\hello/' Fizz\ Buzz]).should eq(["foo bar", "\\hello/", "Fizz Buzz"]) } + + it "raises an error when double quote is unclosed" do + expect_raises ArgumentError, "Unmatched quote" do + Process.parse_arguments(%q["foo]) + end + end + + it "raises an error if single quote is unclosed" do + expect_raises ArgumentError, "Unmatched quote" do + Process.parse_arguments(%q['foo]) + end + end + end end diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 9f722944e12f..11b687f8dc2f 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -620,7 +620,9 @@ class Crystal::Command end private def use_crystal_opts - @options = ENV.fetch("CRYSTAL_OPTS", "").split.concat(options) + @options = Process.parse_arguments(ENV.fetch("CRYSTAL_OPTS", "")).concat(options) + rescue ex + raise LocationlessException.new("Failed to parse CRYSTAL_OPTS: #{ex.message}") end private def new_compiler diff --git a/src/process/shell.cr b/src/process/shell.cr index ee69eb636f78..a356302fe430 100644 --- a/src/process/shell.cr +++ b/src/process/shell.cr @@ -102,4 +102,60 @@ class Process def self.quote_windows(arg : String) : String quote_windows({arg}) end + + # Split a *line* string into the array of tokens in the same way the POSIX shell. + # + # ``` + # Process.parse_arguments(%q["foo bar" '\hello/' Fizz\ Buzz]) # => ["foo bar", "\\hello/", "Fizz Buzz"] + # ``` + # + # See https://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_03 + def self.parse_arguments(line : String) : Array(String) + tokens = [] of String + + reader = Char::Reader.new(line) + + while reader.has_next? + # skip whitespace + while reader.current_char.ascii_whitespace? + reader.next_char + end + break unless reader.has_next? + + token = String.build do |str| + while reader.has_next? && !reader.current_char.ascii_whitespace? + quote = nil + if reader.current_char.in?('\'', '"') + quote = reader.current_char + reader.next_char + end + + until (char = reader.current_char) == quote || (!quote && char.ascii_whitespace?) + break unless reader.has_next? + reader.next_char + if char == '\\' && quote != '\'' + str << char if quote == '"' + char = reader.current_char + if reader.has_next? + reader.next_char + else + break if quote == '"' + char = '\\' + end + end + str << char + end + + if quote + raise ArgumentError.new("Unmatched quote") unless reader.has_next? + reader.next_char + end + end + end + + tokens << token + end + + tokens + end end From 19c0adbcd805c134bf5710745f54ecc86be9b19e Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 23 Jun 2020 17:13:25 -0300 Subject: [PATCH 153/263] Drop deprecated with_color top-level method (#9531) --- samples/2048.cr | 2 +- samples/pretty_json.cr | 12 ++++++------ spec/std/colorize_spec.cr | 26 -------------------------- src/colorize.cr | 10 ---------- 4 files changed, 7 insertions(+), 43 deletions(-) diff --git a/samples/2048.cr b/samples/2048.cr index 91668f3d5553..a76eccf9a59b 100644 --- a/samples/2048.cr +++ b/samples/2048.cr @@ -25,7 +25,7 @@ module Screen def self.colorize_for(tile) fg_color, bg_color = TILES[tile] - color = with_color(fg_color) + color = Colorize.with.fore(fg_color) color = color.on(bg_color) if bg_color color.surround do yield diff --git a/samples/pretty_json.cr b/samples/pretty_json.cr index 71d0d8d80971..0fb30599ef65 100644 --- a/samples/pretty_json.cr +++ b/samples/pretty_json.cr @@ -21,23 +21,23 @@ class PrettyPrinter def read_any case @pull.kind when .null? - with_color.bold.surround(@output) do + Colorize.with.bold.surround(@output) do @pull.read_null.to_json(@output) end when .bool? - with_color.light_blue.surround(@output) do + Colorize.with.light_blue.surround(@output) do @pull.read_bool.to_json(@output) end when .int? - with_color.red.surround(@output) do + Colorize.with.red.surround(@output) do @pull.read_int.to_json(@output) end when .float? - with_color.red.surround(@output) do + Colorize.with.red.surround(@output) do @pull.read_float.to_json(@output) end when .string? - with_color.yellow.surround(@output) do + Colorize.with.yellow.surround(@output) do @pull.read_string.to_json(@output) end when .begin_array? @@ -80,7 +80,7 @@ class PrettyPrinter print '\n' if @indent > 0 end print_indent - with_color.cyan.surround(@output) do + Colorize.with.cyan.surround(@output) do key.to_json(@output) end print ": " diff --git a/spec/std/colorize_spec.cr b/spec/std/colorize_spec.cr index 772eae3f4d76..bfe39f9613da 100644 --- a/spec/std/colorize_spec.cr +++ b/spec/std/colorize_spec.cr @@ -129,32 +129,6 @@ describe "colorize" do colorize("hello", :red).inspect.should eq("\e[31m\"hello\"\e[0m") end - describe "with_color deprecated top-level method" do - it "without args" do - io = IO::Memory.new - with_color.red.toggle(true).surround(io) do - io << "hello" - with_color.green.toggle(true).surround(io) do - io << "world" - end - io << "bye" - end - io.to_s.should eq("\e[31mhello\e[0;32mworld\e[0;31mbye\e[0m") - end - - it "with args" do - io = IO::Memory.new - with_color(:red).toggle(true).surround(io) do - io << "hello" - with_color(:green).toggle(true).surround(io) do - io << "world" - end - io << "bye" - end - io.to_s.should eq("\e[31mhello\e[0;32mworld\e[0;31mbye\e[0m") - end - end - it "colorizes with surround" do io = IO::Memory.new with_color_wrap.red.surround(io) do diff --git a/src/colorize.cr b/src/colorize.cr index d7a7bd16f550..792faa55e229 100644 --- a/src/colorize.cr +++ b/src/colorize.cr @@ -150,16 +150,6 @@ module Colorize end end -@[Deprecated("Use `Colorize.with`")] -def with_color - Colorize.with -end - -@[Deprecated("Use `Colorize.with.fore(color)`")] -def with_color(color : Symbol) - Colorize.with.fore(color) -end - module Colorize::ObjectExtensions def colorize Colorize::Object.new(self) From 476486e0a0424fe9a02f82a6ac597ede0122d957 Mon Sep 17 00:00:00 2001 From: Alexandre Morignot Date: Tue, 23 Jun 2020 22:39:21 +0200 Subject: [PATCH 154/263] Fix OptionParser to handle sub-commands with hyphen (#9465) It was matching regexs anywhere on the string, but we want the regexs to match the full string. In particular this behavior made that a flag `sub-command` was interpeted as `su`, because it matches the `-(.)\S+` regex. Co-authored-by: Alexandre Morignot --- spec/std/option_parser_spec.cr | 10 ++++++++++ src/option_parser.cr | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/spec/std/option_parser_spec.cr b/spec/std/option_parser_spec.cr index 05653a597686..531552a6ee23 100644 --- a/spec/std/option_parser_spec.cr +++ b/spec/std/option_parser_spec.cr @@ -572,6 +572,16 @@ describe "OptionParser" do USAGE end + it "handles subcommands with hyphen" do + subcommand = false + OptionParser.parse(%w(sub-command)) do |opts| + opts.banner = "Usage: foo" + opts.on("sub-command", "Subcommand description") { subcommand = true } + end + + subcommand.should be_true + end + it "stops when asked" do args = %w(--foo --stop --bar) foo = false diff --git a/src/option_parser.cr b/src/option_parser.cr index 7962e776df51..95dd75178ed2 100644 --- a/src/option_parser.cr +++ b/src/option_parser.cr @@ -203,16 +203,16 @@ class OptionParser private def parse_flag_definition(flag : String) case flag - when /--(\S+)\s+\[\S+\]/ + when /\A--(\S+)\s+\[\S+\]\z/ {"--#{$1}", FlagValue::Optional} - when /--(\S+)(\s+|\=)(\S+)?/ + when /\A--(\S+)(\s+|\=)(\S+)?\z/ {"--#{$1}", FlagValue::Required} - when /--\S+/ + when /\A--\S+\z/ # This can't be merged with `else` otherwise /-(.)/ matches {flag, FlagValue::None} - when /-(.)\s*\[\S+\]/ + when /\A-(.)\s*\[\S+\]\z/ {flag[0..1], FlagValue::Optional} - when /-(.)\s+\S+/, /-(.)\s+/, /-(.)\S+/ + when /\A-(.)\s+\S+\z/, /\A-(.)\s+\z/, /\A-(.)\S+\z/ {flag[0..1], FlagValue::Required} else # This happens for -f without argument From bb233eba9e3bd9225b27883a96d46fff99858e5d Mon Sep 17 00:00:00 2001 From: jjlorenzo Date: Mon, 29 Jun 2020 08:20:43 -0500 Subject: [PATCH 155/263] Fix Log docs (#9559) add missing keyword `do` --- src/log.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log.cr b/src/log.cr index b40c089e5a12..23b32ec51558 100644 --- a/src/log.cr +++ b/src/log.cr @@ -107,7 +107,7 @@ # sources errors (or higher) to an elasticsearch backend. # # ``` -# Log.setup |c| +# Log.setup do |c| # backend = Log::IOBackend.new # # c.bind "*", :warn, backend From c608b03d7ab68f92b0b1634f15112d89b1f3030a Mon Sep 17 00:00:00 2001 From: Scott Boggs Date: Mon, 29 Jun 2020 10:15:45 -0400 Subject: [PATCH 156/263] Update YAML::Field(converter) argument docs (#9557) --- src/yaml/serialization.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index c0d3b23e7d26..bd1b086d16d4 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -61,7 +61,7 @@ module YAML # `YAML::Field` properties: # * **ignore**: if `true` skip this field in seriazation and deserialization (by default false) # * **key**: the value of the key in the yaml object (by default the name of the instance variable) - # * **converter**: specify an alternate type for parsing and generation. The converter must define `from_yaml(YAML::PullParser)` and `to_yaml(value, YAML::Builder)` as class methods. Examples of converters are `Time::Format` and `Time::EpochConverter` for `Time`. + # * **converter**: specify an alternate type for parsing and generation. The converter must define `from_yaml(YAML::ParseContext, YAML::Nodes::Node)` and `to_yaml(value, YAML::Nodes::Builder)` as class methods. Examples of converters are `Time::Format` and `Time::EpochConverter` for `Time`. # * **presence**: if `true`, a `@{{key}}_present` instance variable will be generated when the key was present (even if it has a `null` value), `false` by default # * **emit_null**: if `true`, emits a `null` value for nilable property (by default nulls are not emitted) # From 4f4ec97fd297727e10f99571a3706261078f5d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Medeiros?= Date: Mon, 29 Jun 2020 10:17:11 -0400 Subject: [PATCH 157/263] Add `dup` to BitArray (#9550) --- spec/std/bit_array_spec.cr | 9 +++++++++ src/bit_array.cr | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/spec/std/bit_array_spec.cr b/spec/std/bit_array_spec.cr index 6383adeb1ad1..f4ed5f6c880c 100644 --- a/spec/std/bit_array_spec.cr +++ b/spec/std/bit_array_spec.cr @@ -337,4 +337,13 @@ describe "BitArray" do iter.next.should be_true iter.next.should be_a(Iterator::Stop) end + + it "provides dup" do + a = BitArray.new(2) + b = a.dup + + b[0] = true + a[0].should be_false + b[0].should be_true + end end diff --git a/src/bit_array.cr b/src/bit_array.cr index ef779b458583..efb9ba9b6766 100644 --- a/src/bit_array.cr +++ b/src/bit_array.cr @@ -243,6 +243,13 @@ struct BitArray hasher end + # Returns a new `BitArray` with all of the same elements. + def dup + bit_array = BitArray.new(@size) + @bits.copy_to(bit_array.@bits, malloc_size) + bit_array + end + private def bit_index_and_sub_index(index) bit_index_and_sub_index(index) { raise IndexError.new } end From 9d15c538719a1d65b0249bb169013c57644909be Mon Sep 17 00:00:00 2001 From: Todd Sundsted Date: Mon, 29 Jun 2020 10:21:20 -0400 Subject: [PATCH 158/263] Remove Reference to `Kernel`. (#9549) * Remove reference to `Kernel`. * Fix doc. * Prevent doc linking to local method. --- src/io.cr | 2 +- src/string.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io.cr b/src/io.cr index 96e036e24908..d17f9c670651 100644 --- a/src/io.cr +++ b/src/io.cr @@ -259,7 +259,7 @@ abstract class IO end # Writes a formatted string to this IO. - # For details on the format string, see `Kernel::sprintf`. + # For details on the format string, see top-level `::printf`. def printf(format_string, *args) : Nil printf format_string, args end diff --git a/src/string.cr b/src/string.cr index cc45f7e41a89..2e2314590d66 100644 --- a/src/string.cr +++ b/src/string.cr @@ -4634,7 +4634,7 @@ class String !!($~ = /#{re}\z/.match(self)) end - # Interpolates *other* into the string using `Kernel#sprintf`. + # Interpolates *other* into the string using top-level `::sprintf`. # # ``` # "I have %d apples" % 5 # => "I have 5 apples" From 56ab086218ea4b504e2c5bdc717929b6de8d51d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 29 Jun 2020 17:00:54 +0200 Subject: [PATCH 159/263] Disable LLVM Global Isel (#9401) It's not enabled by default on x86, but on other targets it is. At least until https://reviews.llvm.org/D80898 is released, it doesn't like us generating a value of a zero sized type We may need to fix doing that, but until then this workarounds the issue. See https://github.com/crystal-lang/crystal/issues/9297#issuecomment-636512270 for more background. --- spec/std/llvm/aarch64_spec.cr | 1 + spec/std/llvm/arm_abi_spec.cr | 1 + spec/std/llvm/x86_64_abi_spec.cr | 1 + spec/std/llvm/x86_abi_spec.cr | 1 + src/compiler/crystal/codegen/target.cr | 9 ++- src/llvm/ext/llvm_ext.cc | 84 +++++++++++++++++++++++++- src/llvm/jit_compiler.cr | 2 +- src/llvm/lib_llvm_ext.cr | 3 + src/llvm/target_machine.cr | 4 ++ 9 files changed, 102 insertions(+), 4 deletions(-) diff --git a/spec/std/llvm/aarch64_spec.cr b/spec/std/llvm/aarch64_spec.cr index caaa9d8e0ff1..afb682c0c43f 100644 --- a/spec/std/llvm/aarch64_spec.cr +++ b/spec/std/llvm/aarch64_spec.cr @@ -9,6 +9,7 @@ private def abi triple = "aarch64-unknown-linux-gnu" target = LLVM::Target.from_triple(triple) machine = target.create_target_machine(triple) + machine.enable_global_isel = false LLVM::ABI::AArch64.new(machine) end diff --git a/spec/std/llvm/arm_abi_spec.cr b/spec/std/llvm/arm_abi_spec.cr index 1bc7d32d5599..98ae9b588a41 100644 --- a/spec/std/llvm/arm_abi_spec.cr +++ b/spec/std/llvm/arm_abi_spec.cr @@ -9,6 +9,7 @@ private def abi triple = "arm-unknown-linux-gnueabihf" target = LLVM::Target.from_triple(triple) machine = target.create_target_machine(triple) + machine.enable_global_isel = false LLVM::ABI::ARM.new(machine) end diff --git a/spec/std/llvm/x86_64_abi_spec.cr b/spec/std/llvm/x86_64_abi_spec.cr index e9dcc70423ce..2e2514e209d2 100644 --- a/spec/std/llvm/x86_64_abi_spec.cr +++ b/spec/std/llvm/x86_64_abi_spec.cr @@ -9,6 +9,7 @@ private def abi triple = LLVM.default_target_triple.gsub(/^(.+?)-/, "x86_64-") target = LLVM::Target.from_triple(triple) machine = target.create_target_machine(triple) + machine.enable_global_isel = false LLVM::ABI::X86_64.new(machine) end diff --git a/spec/std/llvm/x86_abi_spec.cr b/spec/std/llvm/x86_abi_spec.cr index e3ae341756b3..5f69be12df87 100644 --- a/spec/std/llvm/x86_abi_spec.cr +++ b/spec/std/llvm/x86_abi_spec.cr @@ -13,6 +13,7 @@ private def abi {% end %} target = LLVM::Target.from_triple(triple) machine = target.create_target_machine(triple) + machine.enable_global_isel = false LLVM::ABI::X86.new(machine) end diff --git a/src/compiler/crystal/codegen/target.cr b/src/compiler/crystal/codegen/target.cr index 4e469fef9d4a..1a9c7727571c 100644 --- a/src/compiler/crystal/codegen/target.cr +++ b/src/compiler/crystal/codegen/target.cr @@ -143,7 +143,14 @@ class Crystal::Codegen::Target opt_level = release ? LLVM::CodeGenOptLevel::Aggressive : LLVM::CodeGenOptLevel::None target = LLVM::Target.from_triple(self.to_s) - target.create_target_machine(self.to_s, cpu: cpu, features: features, opt_level: opt_level, code_model: code_model).not_nil! + machine = target.create_target_machine(self.to_s, cpu: cpu, features: features, opt_level: opt_level, code_model: code_model).not_nil! + # We need to disable global isel until https://reviews.llvm.org/D80898 is released, + # or we fixed generating values for 0 sized types. + # When removing this, also remove it from the ABI specs and jit compiler. + # See https://github.com/crystal-lang/crystal/issues/9297#issuecomment-636512270 + # for background info + machine.enable_global_isel = false + machine end def to_s(io : IO) : Nil diff --git a/src/llvm/ext/llvm_ext.cc b/src/llvm/ext/llvm_ext.cc index b418dabb5716..760653d57172 100644 --- a/src/llvm/ext/llvm_ext.cc +++ b/src/llvm/ext/llvm_ext.cc @@ -9,6 +9,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include using namespace llvm; @@ -475,10 +481,84 @@ char *LLVMExtBasicBlockName(LLVMBasicBlockRef BB) { #endif } +static TargetMachine *unwrap(LLVMTargetMachineRef P) { + return reinterpret_cast(P); +} + +void LLVMExtTargetMachineEnableGlobalIsel(LLVMTargetMachineRef T, LLVMBool Enable) { + unwrap(T)->setGlobalISel(Enable); +} + +// Copy paste of https://github.com/llvm/llvm-project/blob/dace8224f38a31636a02fe9c2af742222831f70c/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp#L160-L214 +// but with a parameter to set global isel state +LLVMBool LLVMExtCreateMCJITCompilerForModule( + LLVMExecutionEngineRef *OutJIT, LLVMModuleRef M, + LLVMMCJITCompilerOptions *PassedOptions, size_t SizeOfPassedOptions, + LLVMBool EnableGlobalISel, + char **OutError) { + LLVMMCJITCompilerOptions options; + // If the user passed a larger sized options struct, then they were compiled + // against a newer LLVM. Tell them that something is wrong. + if (SizeOfPassedOptions > sizeof(options)) { + *OutError = strdup( + "Refusing to use options struct that is larger than my own; assuming " + "LLVM library mismatch."); + return 1; + } + + + // Defend against the user having an old version of the API by ensuring that + // any fields they didn't see are cleared. We must defend against fields being + // set to the bitwise equivalent of zero, and assume that this means "do the + // default" as if that option hadn't been available. + LLVMInitializeMCJITCompilerOptions(&options, sizeof(options)); + memcpy(&options, PassedOptions, SizeOfPassedOptions); + + + TargetOptions targetOptions; + targetOptions.EnableFastISel = options.EnableFastISel; + targetOptions.EnableGlobalISel = EnableGlobalISel; + std::unique_ptr Mod(unwrap(M)); + + if (Mod) + // Set function attribute "frame-pointer" based on + // NoFramePointerElim. + for (auto &F : *Mod) { + auto Attrs = F.getAttributes(); + StringRef Value = options.NoFramePointerElim ? "all" : "none"; + Attrs = Attrs.addAttribute(F.getContext(), AttributeList::FunctionIndex, + "frame-pointer", Value); + F.setAttributes(Attrs); + } + + + std::string Error; + EngineBuilder builder(std::move(Mod)); + builder.setEngineKind(EngineKind::JIT) + .setErrorStr(&Error) + .setOptLevel((CodeGenOpt::Level)options.OptLevel) + .setTargetOptions(targetOptions); + bool JIT; + if (Optional CM = unwrap(options.CodeModel, JIT)) + builder.setCodeModel(*CM); + if (options.MCJMM) + builder.setMCJITMemoryManager( + std::unique_ptr(unwrap(options.MCJMM))); + + TargetMachine* tm = builder.selectTarget(); + tm->setGlobalISel(EnableGlobalISel); + + if (ExecutionEngine *JIT = builder.create(tm)) { + *OutJIT = wrap(JIT); + return 0; + } + *OutError = strdup(Error.c_str()); + return 1; +} + LLVMMetadataRef LLVMExtDIBuilderGetOrCreateArraySubrange( DIBuilderRef Dref, uint64_t Lo, uint64_t Count) { return wrap(Dref->getOrCreateSubrange(Lo, Count)); -} - + } } diff --git a/src/llvm/jit_compiler.cr b/src/llvm/jit_compiler.cr index 8fb45068b993..b881e06ee8d7 100644 --- a/src/llvm/jit_compiler.cr +++ b/src/llvm/jit_compiler.cr @@ -5,7 +5,7 @@ class LLVM::JITCompiler mod.take_ownership { raise "Can't create two JIT compilers for the same module" } # if LibLLVM.create_jit_compiler_for_module(out @unwrap, mod, 3, out error) != 0 - if LibLLVM.create_mc_jit_compiler_for_module(out @unwrap, mod, nil, 0, out error) != 0 + if LibLLVMExt.create_mc_jit_compiler_for_module(out @unwrap, mod, nil, 0, false, out error) != 0 raise LLVM.string_and_dispose(error) end diff --git a/src/llvm/lib_llvm_ext.cr b/src/llvm/lib_llvm_ext.cr index b7bae30dcd4d..7c29cf3f6c2a 100644 --- a/src/llvm/lib_llvm_ext.cr +++ b/src/llvm/lib_llvm_ext.cr @@ -162,4 +162,7 @@ lib LibLLVMExt fun normalize_target_triple = LLVMExtNormalizeTargetTriple(triple : Char*) : Char* fun basic_block_name = LLVMExtBasicBlockName(basic_block : LibLLVM::BasicBlockRef) : Char* fun di_builder_get_or_create_array_subrange = LLVMExtDIBuilderGetOrCreateArraySubrange(builder : DIBuilder, lo : UInt64, count : UInt64) : Metadata + + fun target_machine_enable_global_isel = LLVMExtTargetMachineEnableGlobalIsel(machine : LibLLVM::TargetMachineRef, enable : Bool) + fun create_mc_jit_compiler_for_module = LLVMExtCreateMCJITCompilerForModule(jit : LibLLVM::ExecutionEngineRef*, m : LibLLVM::ModuleRef, options : LibLLVM::JITCompilerOptions*, options_length : UInt32, enable_global_isel : Bool, error : UInt8**) : Int32 end diff --git a/src/llvm/target_machine.cr b/src/llvm/target_machine.cr index d5c0fc2c8db6..b2b214f53892 100644 --- a/src/llvm/target_machine.cr +++ b/src/llvm/target_machine.cr @@ -32,6 +32,10 @@ class LLVM::TargetMachine emit_to_file llvm_mod, filename, LLVM::CodeGenFileType::AssemblyFile end + def enable_global_isel=(enable : Bool) + LibLLVMExt.target_machine_enable_global_isel(self, enable) + end + private def emit_to_file(llvm_mod, filename, type) status = LibLLVM.target_machine_emit_to_file(self, llvm_mod, filename, type, out error_msg) unless status == 0 From b32d3b4f82f10e4a0afe2260d1065942de2f8908 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 29 Jun 2020 15:57:37 -0300 Subject: [PATCH 160/263] Compiler: add a few missing `expanded.transform self` (#9506) --- src/compiler/crystal/semantic/cleanup_transformer.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/semantic/cleanup_transformer.cr b/src/compiler/crystal/semantic/cleanup_transformer.cr index c35d4030e8df..d7e03c97d29e 100644 --- a/src/compiler/crystal/semantic/cleanup_transformer.cr +++ b/src/compiler/crystal/semantic/cleanup_transformer.cr @@ -315,7 +315,7 @@ module Crystal def transform(node : Global) if expanded = node.expanded - return expanded + return expanded.transform self end node @@ -792,7 +792,7 @@ module Crystal end if expanded = node.expanded - return expanded + return expanded.transform self end node From 489251c08e8cad934bce08c647f92b223630e64f Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 30 Jun 2020 05:08:05 -0300 Subject: [PATCH 161/263] Make `Socket::IpAddress` a bit more type-friendly (#9528) * Socket::IpAddress: use union instead of two nilable types * Socket::IpAddress: use lazy getter * Replace uses of old `@addr4` --- src/socket/address.cr | 58 ++++++++++++++++------------------------ src/socket/udp_socket.cr | 9 ++++--- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/socket/address.cr b/src/socket/address.cr index 550605e1ab5a..b3dcaa5103ae 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -83,15 +83,15 @@ class Socket getter port : Int32 - @address : String? - @addr6 : LibC::In6Addr? - @addr4 : LibC::InAddr? + @addr : LibC::In6Addr | LibC::InAddr def initialize(@address : String, @port : Int32) - if @addr6 = ip6?(address) + if addr = ip6?(address) + @addr = addr @family = Family::INET6 @size = sizeof(LibC::SockaddrIn6) - elsif @addr4 = ip4?(address) + elsif addr = ip4?(address) + @addr = addr @family = Family::INET @size = sizeof(LibC::SockaddrIn) else @@ -148,7 +148,7 @@ class Socket protected def initialize(sockaddr : LibC::SockaddrIn6*, @size) @family = Family::INET6 - @addr6 = sockaddr.value.sin6_addr + @addr = sockaddr.value.sin6_addr @port = {% if flag?(:dragonfly) %} Intrinsics.bswap16(sockaddr.value.sin6_port).to_i @@ -159,7 +159,7 @@ class Socket protected def initialize(sockaddr : LibC::SockaddrIn*, @size) @family = Family::INET - @addr4 = sockaddr.value.sin_addr + @addr = sockaddr.value.sin_addr @port = {% if flag?(:dragonfly) %} Intrinsics.bswap16(sockaddr.value.sin_port).to_i @@ -185,15 +185,7 @@ class Socket # ip_address = socket.remote_address # ip_address.address # => "127.0.0.1" # ``` - def address - @address ||= begin - case family - when Family::INET6 then address(@addr6.not_nil!) - when Family::INET then address(@addr4.not_nil!) - else raise "Unsupported IP address family: #{family}" - end - end - end + getter(address : String) { address(@addr) } private def address(addr : LibC::In6Addr) String.new(46) do |buffer| @@ -218,23 +210,21 @@ class Socket # In the IPv4 family, loopback addresses are all addresses in the subnet # `127.0.0.0/24`. In IPv6 `::1` is the loopback address. def loopback? : Bool - if addr = @addr4 + case addr = @addr + in LibC::InAddr addr.s_addr & 0x00000000ff_u32 == 0x0000007f_u32 - elsif addr = @addr6 + in LibC::In6Addr ipv6_addr8(addr) == StaticArray[0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 1_u8] - else - raise "unreachable!" end end # Returns `true` if this IP is an unspecified address, either the IPv4 address `0.0.0.0` or the IPv6 address `::`. def unspecified? : Bool - if addr = @addr4 + case addr = @addr + in LibC::InAddr addr.s_addr == 0_u32 - elsif addr = @addr6 + in LibC::In6Addr ipv6_addr8(addr) == StaticArray[0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8] - else - raise "unreachable!" end end @@ -275,17 +265,15 @@ class Socket end def to_unsafe : LibC::Sockaddr* - case family - when Family::INET6 - to_sockaddr_in6 - when Family::INET - to_sockaddr_in - else - raise "Unsupported IP address family: #{family}" + case addr = @addr + in LibC::InAddr + to_sockaddr_in(addr) + in LibC::In6Addr + to_sockaddr_in6(addr) end end - private def to_sockaddr_in6 + private def to_sockaddr_in6(addr) sockaddr = Pointer(LibC::SockaddrIn6).malloc sockaddr.value.sin6_family = family {% if flag?(:dragonfly) %} @@ -293,11 +281,11 @@ class Socket {% else %} sockaddr.value.sin6_port = LibC.htons(port) {% end %} - sockaddr.value.sin6_addr = @addr6.not_nil! + sockaddr.value.sin6_addr = addr sockaddr.as(LibC::Sockaddr*) end - private def to_sockaddr_in + private def to_sockaddr_in(addr) sockaddr = Pointer(LibC::SockaddrIn).malloc sockaddr.value.sin_family = family {% if flag?(:dragonfly) %} @@ -305,7 +293,7 @@ class Socket {% else %} sockaddr.value.sin_port = LibC.htons(port) {% end %} - sockaddr.value.sin_addr = @addr4.not_nil! + sockaddr.value.sin_addr = addr sockaddr.as(LibC::Sockaddr*) end end diff --git a/src/socket/udp_socket.cr b/src/socket/udp_socket.cr index 6887d5cc9706..5f25c6b1ce30 100644 --- a/src/socket/udp_socket.cr +++ b/src/socket/udp_socket.cr @@ -160,7 +160,8 @@ class UDPSocket < IPSocket # Raises `Socket::Error` unless the socket is IPv4 and an IPv4 address is provided. def multicast_interface(address : IPAddress) if @family == Family::INET - if addr = address.@addr4 + addr = address.@addr + if addr.is_a?(LibC::InAddr) setsockopt LibC::IP_MULTICAST_IF, addr, LibC::IPPROTO_IP else raise Socket::Error.new "Expecting an IPv4 interface address. Address provided: #{address.address}" @@ -211,9 +212,11 @@ class UDPSocket < IPSocket end private def group_modify(ip, operation) + ip_addr = ip.@addr + case @family when Family::INET - if ip_addr = ip.@addr4 + if ip_addr.is_a?(LibC::InAddr) req = LibC::IpMreq.new req.imr_multiaddr = ip_addr @@ -222,7 +225,7 @@ class UDPSocket < IPSocket raise Socket::Error.new "Expecting an IPv4 multicast address. Address provided: #{ip.address}" end when Family::INET6 - if ip_addr = ip.@addr6 + if ip_addr.is_a?(LibC::In6Addr) req = LibC::Ipv6Mreq.new req.ipv6mr_multiaddr = ip_addr From 5564c06f165b42926f42d165fefa521c2b41f945 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Tue, 30 Jun 2020 10:15:43 +0200 Subject: [PATCH 162/263] Enable newly passing special var specs (#9560) Followup to #9506 --- spec/compiler/codegen/special_vars_spec.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/compiler/codegen/special_vars_spec.cr b/spec/compiler/codegen/special_vars_spec.cr index 14323c96d28e..280da6caaa98 100644 --- a/spec/compiler/codegen/special_vars_spec.cr +++ b/spec/compiler/codegen/special_vars_spec.cr @@ -2,7 +2,7 @@ require "../../spec_helper" describe "Codegen: special vars" do ["$~", "$?"].each do |name| - pending_win32 "codegens #{name}" do + it "codegens #{name}" do run(%( class Object; def not_nil!; self; end; end @@ -15,7 +15,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - pending_win32 "codegens #{name} with nilable (1)" do + it "codegens #{name} with nilable (1)" do run(%( require "prelude" @@ -35,7 +35,7 @@ describe "Codegen: special vars" do )).to_string.should eq("ouch") end - pending_win32 "codegens #{name} with nilable (2)" do + it "codegens #{name} with nilable (2)" do run(%( require "prelude" @@ -74,7 +74,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - pending_win32 "works lazily" do + it "works lazily" do run(%( require "prelude" @@ -145,7 +145,7 @@ describe "Codegen: special vars" do )).to_string.should eq("hey") end - pending_win32 "codegens after block" do + it "codegens after block" do run(%( require "prelude" From 8219529e91a0e52067a79f846b1b30039bf3345e Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Tue, 30 Jun 2020 12:43:21 -0300 Subject: [PATCH 163/263] Channel#close returns boolean (#9443) --- spec/std/channel_spec.cr | 3 ++- src/channel.cr | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 0a82deaf0806..097c2945a579 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -633,8 +633,9 @@ describe "unbuffered" do it "can be closed" do ch = Channel(Int32).new ch.closed?.should be_false - ch.close.should be_nil + ch.close.should be_true ch.closed?.should be_true + ch.close.should be_false expect_raises(Channel::ClosedError) { ch.receive } end diff --git a/src/channel.cr b/src/channel.cr index 83ea9d6c238f..5f4c77e77fa7 100644 --- a/src/channel.cr +++ b/src/channel.cr @@ -183,11 +183,14 @@ class Channel(T) # Both awaiting and subsequent calls to `#send` will consider the channel closed. # All items successfully sent to the channel can be received, before `#receive` considers the channel closed. # Calling `#close` on a closed channel does not have any effect. - def close : Nil + # + # It returns `true` when the channel was successfuly closed, or `false` if it was already closed. + def close : Bool sender_list = Crystal::PointerLinkedList(Sender(T)).new receiver_list = Crystal::PointerLinkedList(Receiver(T)).new @lock.sync do + return false if @closed @closed = true @senders, sender_list = sender_list, @senders @@ -196,6 +199,7 @@ class Channel(T) sender_list.each(&.value.close) receiver_list.each(&.value.close) + true end def closed? From ccab86d97fa8d4a7086dec50fa57a550e1e72b2e Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Tue, 30 Jun 2020 12:44:34 -0300 Subject: [PATCH 164/263] Use unchecked arithmetics in `Int#times` (#9511) --- src/int.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/int.cr b/src/int.cr index 2bdb2dbeaf2d..87e868995f89 100644 --- a/src/int.cr +++ b/src/int.cr @@ -508,7 +508,7 @@ struct Int i = self ^ self while i < self yield i - i += 1 + i &+= 1 end end @@ -707,7 +707,7 @@ struct Int def next if @index < @n value = @index - @index += 1 + @index &+= 1 value else stop From aad908ef597038bc407c1a5ac94b34f553418949 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Tue, 30 Jun 2020 12:48:47 -0300 Subject: [PATCH 165/263] Delete existing object file if bc file changes (#9558) --- src/compiler/crystal/compiler.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index 57bdb9d597ea..305e0a1122e6 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -698,6 +698,8 @@ module Crystal # If there's a memory buffer, it means we must create a .o from it if memory_buffer + # Delete existing .o file. It cannot be used anymore. + File.delete(object_name) if File.exists?(object_name) # Create the .bc file (for next compilations) File.write(bc_name, memory_buffer.to_slice) memory_buffer.dispose From e04f5089cabed78e7f2b1dc36221bc7ebb971c14 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 1 Jul 2020 09:45:24 -0300 Subject: [PATCH 166/263] Fix build on llvm 6.0 (#9562) * Fix build for llvm 6.0 * Ignore setGlobalISel LLVM < 7.0 --- src/llvm/ext/llvm_ext.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/llvm/ext/llvm_ext.cc b/src/llvm/ext/llvm_ext.cc index 760653d57172..e67716e0fc46 100644 --- a/src/llvm/ext/llvm_ext.cc +++ b/src/llvm/ext/llvm_ext.cc @@ -14,7 +14,6 @@ #include #include #include -#include using namespace llvm; @@ -27,6 +26,12 @@ using namespace llvm; #define LLVM_VERSION_LE(major, minor) \ (LLVM_VERSION_MAJOR < (major) || LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR <= (minor)) +#if LLVM_VERSION_GE(7, 0) +#include +#else +#include +#endif + #if LLVM_VERSION_GE(6, 0) #include #endif @@ -486,7 +491,9 @@ static TargetMachine *unwrap(LLVMTargetMachineRef P) { } void LLVMExtTargetMachineEnableGlobalIsel(LLVMTargetMachineRef T, LLVMBool Enable) { +#if LLVM_VERSION_GE(7, 0) unwrap(T)->setGlobalISel(Enable); +#endif } // Copy paste of https://github.com/llvm/llvm-project/blob/dace8224f38a31636a02fe9c2af742222831f70c/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp#L160-L214 @@ -517,7 +524,9 @@ LLVMBool LLVMExtCreateMCJITCompilerForModule( TargetOptions targetOptions; targetOptions.EnableFastISel = options.EnableFastISel; - targetOptions.EnableGlobalISel = EnableGlobalISel; + #if LLVM_VERSION_GE(7, 0) + targetOptions.EnableGlobalISel = EnableGlobalISel; + #endif std::unique_ptr Mod(unwrap(M)); if (Mod) @@ -546,7 +555,9 @@ LLVMBool LLVMExtCreateMCJITCompilerForModule( std::unique_ptr(unwrap(options.MCJMM))); TargetMachine* tm = builder.selectTarget(); - tm->setGlobalISel(EnableGlobalISel); + #if LLVM_VERSION_GE(7, 0) + tm->setGlobalISel(EnableGlobalISel); + #endif if (ExecutionEngine *JIT = builder.create(tm)) { *OutJIT = wrap(JIT); From 2f8092cea778509a5d4a35b7d4f6797e37110786 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 1 Jul 2020 09:50:24 -0300 Subject: [PATCH 167/263] Allow HTTP::Client to work with any IO (#9543) * Allow HTTP socket to be any IO * Update src/http/client.cr Co-authored-by: Sijawusz Pur Rahnama * Disallow HTTP::Client reconnection when initialized with IO * HTTP::Client send empty `Host` header by default when IO is used * Rename @socket to @io Co-authored-by: Sijawusz Pur Rahnama --- spec/std/http/client/client_spec.cr | 35 +++++++++++++++++++++ spec/std/http/server/server_spec.cr | 4 +-- src/http/client.cr | 49 ++++++++++++++++++----------- 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index 1a80c07941cc..e7dc6443b0b7 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -265,5 +265,40 @@ module HTTP request.host.should eq "other.example.com" end end + + it "works with IO" do + io_response = IO::Memory.new <<-RESPONSE.gsub('\n', "\r\n") + HTTP/1.1 200 OK + Content-Type: text/plain + Content-Length: 3 + + Hi! + RESPONSE + io_request = IO::Memory.new + io = IO::Stapled.new(io_response, io_request) + client = Client.new(io) + response = client.get("/") + response.body.should eq("Hi!") + + io_request.rewind + request = HTTP::Request.from_io(io_request).as(HTTP::Request) + request.host.should eq("") + end + + it "can specify host and port when initialized with IO" do + client = Client.new(IO::Memory.new, "host", 1234) + client.host.should eq("host") + client.port.should eq(1234) + end + + it "cannot reconnect when initialized with IO" do + io = IO::Memory.new + client = Client.new(io) + client.close + io.closed?.should be_true + expect_raises(Exception, "This HTTP::Client cannot be reconnected") do + client.get("/") + end + end end end diff --git a/spec/std/http/server/server_spec.cr b/spec/std/http/server/server_spec.cr index cc932a7f7b4d..6c39a8e21aff 100644 --- a/spec/std/http/server/server_spec.cr +++ b/spec/std/http/server/server_spec.cr @@ -493,7 +493,7 @@ describe "#remote_address" do HTTP::Client.new(URI.parse("http://#{address1}/")) do |client| client.get("/") - remote_address.should eq(client.@socket.as(IPSocket).local_address) + remote_address.should eq(client.@io.as(IPSocket).local_address) end end end @@ -516,7 +516,7 @@ describe "#remote_address" do uri: URI.parse("https://#{ip_address1}"), tls: client_context) do |client| client.get("/") - remote_address.should eq(client.@socket.as(OpenSSL::SSL::Socket).local_address) + remote_address.should eq(client.@io.as(OpenSSL::SSL::Socket).local_address) end end end diff --git a/src/http/client.cr b/src/http/client.cr index 4bcf8f4e0526..bc032ac0b986 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -105,21 +105,21 @@ class HTTP::Client # ``` {% if flag?(:without_openssl) %} getter! tls : Nil - @socket : TCPSocket | Nil alias TLSContext = Bool | Nil {% else %} getter! tls : OpenSSL::SSL::Context::Client - @socket : TCPSocket | OpenSSL::SSL::Socket | Nil alias TLSContext = OpenSSL::SSL::Context::Client | Bool | Nil {% end %} # Whether automatic compression/decompression is enabled. property? compress : Bool = true + @io : IO? @dns_timeout : Float64? @connect_timeout : Float64? @read_timeout : Float64? @write_timeout : Float64? + @reconnect = true # Creates a new HTTP client with the given *host*, *port* and *tls* # configurations. If no port is given, the default one will @@ -148,6 +148,16 @@ class HTTP::Client @port = (port || (@tls ? 443 : 80)).to_i end + # Creates a new HTTP client bound to an existing `IO`. + # *host* and *port* can be specified and they will be used + # to conform the `Host` header on each request. + # Instances created with this constructor cannot be reconnected. Once + # `close` is called explicitly or if the connection doesn't support keep-alive, + # the next call to make a request will raise an exception. + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + end + private def check_host_only(string : String) # When parsing a URI with just a host # we end up with a URI with just a path @@ -585,7 +595,7 @@ class HTTP::Client private def exec_internal_single(request) decompress = send_request(request) - HTTP::Client::Response.from_io?(socket, ignore_body: request.ignore_body?, decompress: decompress) + HTTP::Client::Response.from_io?(io, ignore_body: request.ignore_body?, decompress: decompress) end private def handle_response(response) @@ -632,7 +642,7 @@ class HTTP::Client private def exec_internal_single(request) decompress = send_request(request) - HTTP::Client::Response.from_io?(socket, ignore_body: request.ignore_body?, decompress: decompress) do |response| + HTTP::Client::Response.from_io?(io, ignore_body: request.ignore_body?, decompress: decompress) do |response| yield response end end @@ -647,8 +657,8 @@ class HTTP::Client private def send_request(request) decompress = set_defaults request run_before_request_callbacks(request) - request.to_io(socket) - socket.flush + request.to_io(io) + io.flush decompress end @@ -746,29 +756,32 @@ class HTTP::Client # Closes this client. If used again, a new connection will be opened. def close - @socket.try &.close - @socket = nil + @io.try &.close + @io = nil end private def new_request(method, path, headers, body : BodyType) HTTP::Request.new(method, path, headers, body) end - private def socket - socket = @socket - return socket if socket + private def io + io = @io + return io if io + unless @reconnect + raise "This HTTP::Client cannot be reconnected" + end hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - socket = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout - socket.read_timeout = @read_timeout if @read_timeout - socket.write_timeout = @write_timeout if @write_timeout - socket.sync = false + io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout + io.read_timeout = @read_timeout if @read_timeout + io.write_timeout = @write_timeout if @write_timeout + io.sync = false {% if !flag?(:without_openssl) %} if tls = @tls - tcp_socket = socket + tcp_socket = io begin - socket = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) + io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) rescue exc # don't leak the TCP socket when the SSL connection failed tcp_socket.close @@ -777,7 +790,7 @@ class HTTP::Client end {% end %} - @socket = socket + @io = io end private def host_header From cdb4b31fe5f139b4df4d2ca915c456e728558692 Mon Sep 17 00:00:00 2001 From: nia <29542929+niacat@users.noreply.github.com> Date: Thu, 2 Jul 2020 13:38:12 +0000 Subject: [PATCH 168/263] Port to NetBSD (#9360) * Add netbsd and bsd compiler flags * Port to NetBSD * Build only X86 target * More accurate NetBSD libc definitions * netbsd: More accurate pthreads * Fix whitespace * correct typo * netbsd: Correct stat definitions * netbsd: More accurate sys/types * netbsd: Avoid using libc compat functions * netbsd: more accurate sys/socket * netbsd: More libc RENAMEs * netbsd: avoid compat function for unsetenv * netbsd: Don't try to link -liconv * Build only X86 target * libc elf bits * more pthread bits * netbsd: correct tcp socket option definitions * netbsd: avoid pwnam compat functions in libc * netbsd: slightly different LibC::KERN_PROC_PATHNAME than freebsd * libm: list targets the code is intended for rather than reverse * netbsd: run crystal format * netbsd: define struct DIR * netbsd: correct socket definitions * netbsd: correct size of fds_bits * netbsd: correct order of struct addrinfo * netbsd: use actual types used by libc. should not be a functional change * netbsd: grr struct addrinfo * netbsd: fix tests relying on iconv * fix io_spec on netbsd * Revert "fix io_spec on netbsd" This reverts commit db81090a2c045b94a21d24f40f4773ca094d9f07. * arc4random: remove incorrect comment Co-authored-by: RX14 --- src/compiler/crystal/codegen/target.cr | 8 +- src/compiler/crystal/semantic/flags.cr | 1 + src/crystal/iconv.cr | 2 +- src/crystal/system.cr | 2 +- src/crystal/system/random.cr | 2 +- src/crystal/system/unix/arc4random.cr | 4 +- src/crystal/system/unix/pthread.cr | 2 +- src/crystal/system/unix/sysconf_cpucount.cr | 2 +- src/crystal/system/unix/sysctl_cpucount.cr | 2 +- src/crystal/system/unix/urandom.cr | 2 +- src/errno.cr | 6 +- src/lib_c/x86_64-netbsd/c/arpa/inet.cr | 9 ++ src/lib_c/x86_64-netbsd/c/dirent.cr | 30 +++++++ src/lib_c/x86_64-netbsd/c/dlfcn.cr | 21 +++++ src/lib_c/x86_64-netbsd/c/elf.cr | 24 +++++ src/lib_c/x86_64-netbsd/c/errno.cr | 99 +++++++++++++++++++++ src/lib_c/x86_64-netbsd/c/fcntl.cr | 32 +++++++ src/lib_c/x86_64-netbsd/c/grp.cr | 11 +++ src/lib_c/x86_64-netbsd/c/iconv.cr | 9 ++ src/lib_c/x86_64-netbsd/c/link.cr | 17 ++++ src/lib_c/x86_64-netbsd/c/netdb.cr | 37 ++++++++ src/lib_c/x86_64-netbsd/c/netinet/in.cr | 70 +++++++++++++++ src/lib_c/x86_64-netbsd/c/netinet/tcp.cr | 5 ++ src/lib_c/x86_64-netbsd/c/pthread.cr | 39 ++++++++ src/lib_c/x86_64-netbsd/c/pwd.cr | 17 ++++ src/lib_c/x86_64-netbsd/c/sched.cr | 3 + src/lib_c/x86_64-netbsd/c/signal.cr | 55 ++++++++++++ src/lib_c/x86_64-netbsd/c/stdarg.cr | 10 +++ src/lib_c/x86_64-netbsd/c/stddef.cr | 3 + src/lib_c/x86_64-netbsd/c/stdint.cr | 10 +++ src/lib_c/x86_64-netbsd/c/stdio.cr | 9 ++ src/lib_c/x86_64-netbsd/c/stdlib.cr | 27 ++++++ src/lib_c/x86_64-netbsd/c/string.cr | 9 ++ src/lib_c/x86_64-netbsd/c/sys/file.cr | 11 +++ src/lib_c/x86_64-netbsd/c/sys/mman.cr | 30 +++++++ src/lib_c/x86_64-netbsd/c/sys/resource.cr | 25 ++++++ src/lib_c/x86_64-netbsd/c/sys/select.cr | 12 +++ src/lib_c/x86_64-netbsd/c/sys/socket.cr | 68 ++++++++++++++ src/lib_c/x86_64-netbsd/c/sys/stat.cr | 57 ++++++++++++ src/lib_c/x86_64-netbsd/c/sys/time.cr | 16 ++++ src/lib_c/x86_64-netbsd/c/sys/types.cr | 67 ++++++++++++++ src/lib_c/x86_64-netbsd/c/sys/un.cr | 9 ++ src/lib_c/x86_64-netbsd/c/sys/wait.cr | 8 ++ src/lib_c/x86_64-netbsd/c/sysctl.cr | 13 +++ src/lib_c/x86_64-netbsd/c/termios.cr | 95 ++++++++++++++++++++ src/lib_c/x86_64-netbsd/c/time.cr | 37 ++++++++ src/lib_c/x86_64-netbsd/c/unistd.cr | 46 ++++++++++ src/math/libm.cr | 2 +- src/process/executable_path.cr | 14 +++ src/socket/address.cr | 2 +- src/socket/tcp_socket.cr | 4 + src/termios.cr | 2 +- 52 files changed, 1080 insertions(+), 17 deletions(-) create mode 100644 src/lib_c/x86_64-netbsd/c/arpa/inet.cr create mode 100644 src/lib_c/x86_64-netbsd/c/dirent.cr create mode 100644 src/lib_c/x86_64-netbsd/c/dlfcn.cr create mode 100644 src/lib_c/x86_64-netbsd/c/elf.cr create mode 100644 src/lib_c/x86_64-netbsd/c/errno.cr create mode 100644 src/lib_c/x86_64-netbsd/c/fcntl.cr create mode 100644 src/lib_c/x86_64-netbsd/c/grp.cr create mode 100644 src/lib_c/x86_64-netbsd/c/iconv.cr create mode 100644 src/lib_c/x86_64-netbsd/c/link.cr create mode 100644 src/lib_c/x86_64-netbsd/c/netdb.cr create mode 100644 src/lib_c/x86_64-netbsd/c/netinet/in.cr create mode 100644 src/lib_c/x86_64-netbsd/c/netinet/tcp.cr create mode 100644 src/lib_c/x86_64-netbsd/c/pthread.cr create mode 100644 src/lib_c/x86_64-netbsd/c/pwd.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sched.cr create mode 100644 src/lib_c/x86_64-netbsd/c/signal.cr create mode 100644 src/lib_c/x86_64-netbsd/c/stdarg.cr create mode 100644 src/lib_c/x86_64-netbsd/c/stddef.cr create mode 100644 src/lib_c/x86_64-netbsd/c/stdint.cr create mode 100644 src/lib_c/x86_64-netbsd/c/stdio.cr create mode 100644 src/lib_c/x86_64-netbsd/c/stdlib.cr create mode 100644 src/lib_c/x86_64-netbsd/c/string.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/file.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/mman.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/resource.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/select.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/socket.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/stat.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/time.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/types.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/un.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sys/wait.cr create mode 100644 src/lib_c/x86_64-netbsd/c/sysctl.cr create mode 100644 src/lib_c/x86_64-netbsd/c/termios.cr create mode 100644 src/lib_c/x86_64-netbsd/c/time.cr create mode 100644 src/lib_c/x86_64-netbsd/c/unistd.cr diff --git a/src/compiler/crystal/codegen/target.cr b/src/compiler/crystal/codegen/target.cr index 1a9c7727571c..e6db9532c24f 100644 --- a/src/compiler/crystal/codegen/target.cr +++ b/src/compiler/crystal/codegen/target.cr @@ -55,6 +55,8 @@ class Crystal::Codegen::Target "dragonfly" when .openbsd? "openbsd" + when .netbsd? + "netbsd" else environment end @@ -84,12 +86,16 @@ class Crystal::Codegen::Target @environment.starts_with?("openbsd") end + def netbsd? + @environment.starts_with?("netbsd") + end + def linux? @environment.starts_with?("linux") end def bsd? - freebsd? || openbsd? || dragonfly? + freebsd? || netbsd? || openbsd? || dragonfly? end def unix? diff --git a/src/compiler/crystal/semantic/flags.cr b/src/compiler/crystal/semantic/flags.cr index 76ecd26a5783..d412dfbc011f 100644 --- a/src/compiler/crystal/semantic/flags.cr +++ b/src/compiler/crystal/semantic/flags.cr @@ -44,6 +44,7 @@ class Crystal::Program flags.add "freebsd" flags.add "freebsd#{target.freebsd_version}" end + flags.add "netbsd" if target.netbsd? flags.add "openbsd" if target.openbsd? flags.add "dragonfly" if target.dragonfly? diff --git a/src/crystal/iconv.cr b/src/crystal/iconv.cr index eaac0a8e74a3..9985e361664d 100644 --- a/src/crystal/iconv.cr +++ b/src/crystal/iconv.cr @@ -10,7 +10,7 @@ struct Crystal::Iconv original_from, original_to = from, to @skip_invalid = invalid == :skip - {% unless flag?(:freebsd) || flag?(:musl) || flag?(:dragonfly) %} + {% unless flag?(:freebsd) || flag?(:musl) || flag?(:dragonfly) || flag?(:netbsd) %} if @skip_invalid from = "#{from}//IGNORE" to = "#{to}//IGNORE" diff --git a/src/crystal/system.cr b/src/crystal/system.cr index 805ac04b6654..88827a3f6552 100644 --- a/src/crystal/system.cr +++ b/src/crystal/system.cr @@ -11,7 +11,7 @@ end {% if flag?(:unix) %} require "./system/unix/hostname" - {% if flag?(:freebsd) || flag?(:openbsd) || flag?(:dragonfly) %} + {% if flag?(:bsd) %} require "./system/unix/sysctl_cpucount" {% else %} require "./system/unix/sysconf_cpucount" diff --git a/src/crystal/system/random.cr b/src/crystal/system/random.cr index e03cc6ca6118..4581a3011f33 100644 --- a/src/crystal/system/random.cr +++ b/src/crystal/system/random.cr @@ -12,7 +12,7 @@ end {% if flag?(:linux) %} require "./unix/getrandom" -{% elsif flag?(:openbsd) %} +{% elsif flag?(:openbsd) || flag?(:netbsd) %} require "./unix/arc4random" {% elsif flag?(:unix) %} require "./unix/urandom" diff --git a/src/crystal/system/unix/arc4random.cr b/src/crystal/system/unix/arc4random.cr index 67aa0867560a..4338e3341974 100644 --- a/src/crystal/system/unix/arc4random.cr +++ b/src/crystal/system/unix/arc4random.cr @@ -1,11 +1,9 @@ -{% skip_file unless flag?(:openbsd) %} +{% skip_file unless flag?(:openbsd) || flag?(:netbsd) %} require "c/stdlib" module Crystal::System::Random # Fills *buffer* with random bytes using arc4random. - # - # NOTE: only secure on OpenBSD and CloudABI def self.random_bytes(buffer : Bytes) : Nil LibC.arc4random_buf(buffer.to_unsafe.as(Void*), buffer.size) end diff --git a/src/crystal/system/unix/pthread.cr b/src/crystal/system/unix/pthread.cr index 005d71cfc17f..df9fc026447d 100644 --- a/src/crystal/system/unix/pthread.cr +++ b/src/crystal/system/unix/pthread.cr @@ -140,7 +140,7 @@ class Thread {% if flag?(:darwin) %} # FIXME: pthread_get_stacksize_np returns bogus value on macOS X 10.9.0: address = LibC.pthread_get_stackaddr_np(@th) - LibC.pthread_get_stacksize_np(@th) - {% elsif flag?(:freebsd) || flag?(:dragonfly) %} + {% elsif flag?(:bsd) && !flag?(:openbsd) %} ret = LibC.pthread_attr_init(out attr) unless ret == 0 LibC.pthread_attr_destroy(pointerof(attr)) diff --git a/src/crystal/system/unix/sysconf_cpucount.cr b/src/crystal/system/unix/sysconf_cpucount.cr index 728f5bf80d18..83dea23c0593 100644 --- a/src/crystal/system/unix/sysconf_cpucount.cr +++ b/src/crystal/system/unix/sysconf_cpucount.cr @@ -1,4 +1,4 @@ -{% skip_file if flag?(:openbsd) || flag?(:freebsd) || flag?(:dragonfly) %} +{% skip_file if flag?(:bsd) %} require "c/unistd" diff --git a/src/crystal/system/unix/sysctl_cpucount.cr b/src/crystal/system/unix/sysctl_cpucount.cr index 2c818c4c0343..496ace615a50 100644 --- a/src/crystal/system/unix/sysctl_cpucount.cr +++ b/src/crystal/system/unix/sysctl_cpucount.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:openbsd) || flag?(:freebsd) || flag?(:dragonfly) %} +{% skip_file unless flag?(:bsd) %} require "c/sysctl" diff --git a/src/crystal/system/unix/urandom.cr b/src/crystal/system/unix/urandom.cr index f2998b772fb0..7ac025f43e6b 100644 --- a/src/crystal/system/unix/urandom.cr +++ b/src/crystal/system/unix/urandom.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) && !flag?(:openbsd) && !flag?(:linux) %} +{% skip_file unless flag?(:unix) && !flag?(:netbsd) && !flag?(:openbsd) && !flag?(:linux) %} module Crystal::System::Random @@initialized = false diff --git a/src/errno.cr b/src/errno.cr index d5d4b291eddf..7ee985ab794e 100644 --- a/src/errno.cr +++ b/src/errno.cr @@ -6,7 +6,7 @@ lib LibC fun __errno_location : Int* {% elsif flag?(:darwin) || flag?(:freebsd) %} fun __error : Int* - {% elsif flag?(:openbsd) %} + {% elsif flag?(:netbsd) || flag?(:openbsd) %} fun __error = __errno : Int* {% elsif flag?(:win32) %} fun _get_errno(value : Int*) : ErrnoT @@ -43,7 +43,7 @@ enum Errno def self.value : self {% if flag?(:linux) || flag?(:dragonfly) %} Errno.new LibC.__errno_location.value - {% elsif flag?(:darwin) || flag?(:freebsd) || flag?(:openbsd) %} + {% elsif flag?(:darwin) || flag?(:bsd) %} Errno.new LibC.__error.value {% elsif flag?(:win32) %} ret = LibC._get_errno(out errno) @@ -56,7 +56,7 @@ enum Errno def self.value=(errno : Errno) {% if flag?(:linux) || flag?(:dragonfly) %} LibC.__errno_location.value = errno.value - {% elsif flag?(:darwin) || flag?(:freebsd) || flag?(:openbsd) %} + {% elsif flag?(:darwin) || flag?(:bsd) %} LibC.__error.value = errno.value {% elsif flag?(:win32) %} ret = LibC._set_errno(errno.value) diff --git a/src/lib_c/x86_64-netbsd/c/arpa/inet.cr b/src/lib_c/x86_64-netbsd/c/arpa/inet.cr new file mode 100644 index 000000000000..afac8795f66f --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/arpa/inet.cr @@ -0,0 +1,9 @@ +require "../netinet/in" +require "../stdint" + +lib LibC + fun htons(x0 : UInt16T) : UInt16T + fun ntohs(x0 : UInt16T) : UInt16T + fun inet_ntop(x0 : Int, x1 : Void*, x2 : Char*, x3 : SocklenT) : Char* + fun inet_pton(x0 : Int, x1 : Char*, x2 : Void*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/dirent.cr b/src/lib_c/x86_64-netbsd/c/dirent.cr new file mode 100644 index 000000000000..4b3186ab6a97 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/dirent.cr @@ -0,0 +1,30 @@ +require "./sys/types" + +lib LibC + struct DIR + dd_fd : Int + dd_loc : Long + dd_size : Long + dd_buf : Char* + dd_len : Int + dd_seek : OffT + dd_internal : Void* + dd_flags : Int + dd_lock : Void* + end + + DT_DIR = 4 + + struct Dirent + d_fileno : InoT + d_reclen : UInt16 + d_namlen : UInt16 + d_type : UInt8 + d_name : StaticArray(Char, 512) + end + + fun closedir(x0 : DIR*) : Int + fun opendir = __opendir30(x0 : Char*) : DIR* + fun readdir = __readdir30(x0 : DIR*) : Dirent* + fun rewinddir(x0 : DIR*) : Void +end diff --git a/src/lib_c/x86_64-netbsd/c/dlfcn.cr b/src/lib_c/x86_64-netbsd/c/dlfcn.cr new file mode 100644 index 000000000000..cbdf854f1912 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/dlfcn.cr @@ -0,0 +1,21 @@ +lib LibC + RTLD_LAZY = 1 + RTLD_NOW = 2 + RTLD_GLOBAL = 0x100 + RTLD_LOCAL = 0x200 + RTLD_NEXT = Pointer(Void).new(-1) + RTLD_DEFAULT = Pointer(Void).new(-2) + + struct DlInfo + dli_fname : Char* + dli_fbase : Void* + dli_sname : Char* + dli_saddr : Void* + end + + fun dlclose(x0 : Void*) : Int + fun dlerror : Char* + fun dlopen(x0 : Char*, x1 : Int) : Void* + fun dlsym(x0 : Void*, x1 : Char*) : Void* + fun dladdr(x0 : Void*, x1 : DlInfo*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/elf.cr b/src/lib_c/x86_64-netbsd/c/elf.cr new file mode 100644 index 000000000000..f9ccc3a115ab --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/elf.cr @@ -0,0 +1,24 @@ +require "./sys/types" + +lib LibC + alias Elf_Half = UInt16T + alias Elf_Word = UInt32T + alias Elf_Sword = Int32T + alias Elf_Xword = UInt64T + alias Elf_Sxword = Int64T + alias Elf_Addr = UInt64T + alias Elf_Off = UInt64T + alias Elf_Section = UInt16T + alias Elf_Versym = Elf_Half + + struct Elf_Phdr + type : Elf_Word # Segment type + flags : Elf_Word # Segment flags + offset : Elf_Off # Segment file offset + vaddr : Elf_Addr # Segment virtual address + paddr : Elf_Addr # Segment physical address + filesz : Elf_Xword # Segment size in file + memsz : Elf_Xword # Segment size in memory + align : Elf_Xword # Segment alignment + end +end diff --git a/src/lib_c/x86_64-netbsd/c/errno.cr b/src/lib_c/x86_64-netbsd/c/errno.cr new file mode 100644 index 000000000000..43eaeed4f02c --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/errno.cr @@ -0,0 +1,99 @@ +lib LibC + EPERM = 1 # Operation not permitted + ENOENT = 2 # No such file or directory + ESRCH = 3 # No such process + EINTR = 4 # Interrupted system call + EIO = 5 # Input/output error + ENXIO = 6 # Device not configured + E2BIG = 7 # Argument list too long + ENOEXEC = 8 # Exec format error + EBADF = 9 # Bad file descriptor + ECHILD = 10 # No child processes + EDEADLK = 11 # Resource deadlock avoided + ENOMEM = 12 # Cannot allocate memory + EACCES = 13 # Permission denied + EFAULT = 14 # Bad address + ENOTBLK = 15 # Block device required + EBUSY = 16 # Device busy + EEXIST = 17 # File exists + EXDEV = 18 # Cross-device link + ENODEV = 19 # Operation not supported by device + ENOTDIR = 20 # Not a directory + EISDIR = 21 # Is a directory + EINVAL = 22 # Invalid argument + ENFILE = 23 # Too many open files in system + EMFILE = 24 # Too many open files + ENOTTY = 25 # Inappropriate ioctl for device + ETXTBSY = 26 # Text file busy + EFBIG = 27 # File too large + ENOSPC = 28 # No space left on device + ESPIPE = 29 # Illegal seek + EROFS = 30 # Read-only file system + EMLINK = 31 # Too many links + EPIPE = 32 # Broken pipe + EDOM = 33 # Numerical argument out of domain + ERANGE = 34 # Result too large or too small + EAGAIN = 35 # Resource temporarily unavailable + EWOULDBLOCK = EAGAIN # Operation would block + EINPROGRESS = 36 # Operation now in progress + EALREADY = 37 # Operation already in progress + ENOTSOCK = 38 # Socket operation on non-socket + EDESTADDRREQ = 39 # Destination address required + EMSGSIZE = 40 # Message too long + EPROTOTYPE = 41 # Protocol wrong type for socket + ENOPROTOOPT = 42 # Protocol option not available + EPROTONOSUPPORT = 43 # Protocol not supported + ESOCKTNOSUPPORT = 44 # Socket type not supported + EOPNOTSUPP = 45 # Operation not supported + EPFNOSUPPORT = 46 # Protocol family not supported + EAFNOSUPPORT = 47 # Address family not supported by protocol family + EADDRINUSE = 48 # Address already in use + EADDRNOTAVAIL = 49 # Can't assign requested address + ENETDOWN = 50 # Network is down + ENETUNREACH = 51 # Network is unreachable + ENETRESET = 52 # Network dropped connection on reset + ECONNABORTED = 53 # Software caused connection abort + ECONNRESET = 54 # Connection reset by peer + ENOBUFS = 55 # No buffer space available + EISCONN = 56 # Socket is already connected + ENOTCONN = 57 # Socket is not connected + ESHUTDOWN = 58 # Can't send after socket shutdown + ETOOMANYREFS = 59 # Too many references: can't splice + ETIMEDOUT = 60 # Operation timed out + ECONNREFUSED = 61 # Connection refused + ELOOP = 62 # Too many levels of symbolic links + ENAMETOOLONG = 63 # File name too long + EHOSTDOWN = 64 # Host is down + EHOSTUNREACH = 65 # No route to host + ENOTEMPTY = 66 # Directory not empty + EPROCLIM = 67 # Too many processes + EUSERS = 68 # Too many users + EDQUOT = 69 # Disc quota exceeded + ESTALE = 70 # Stale NFS file handle + EREMOTE = 71 # Too many levels of remote in path + EBADRPC = 72 # RPC struct is bad + ERPCMISMATCH = 73 # RPC version wrong + EPROGUNAVAIL = 74 # RPC prog. not avail + EPROGMISMATCH = 75 # Program version wrong + EPROCUNAVAIL = 76 # Bad procedure for program + ENOLCK = 77 # No locks available + ENOSYS = 78 # Function not implemented + EFTYPE = 79 # Inappropriate file type or format + EAUTH = 80 # Authentication error + ENEEDAUTH = 81 # Need authenticator + EIDRM = 82 # Identifier removed + ENOMSG = 83 # No message of desired type + EOVERFLOW = 84 # Value too large to be stored in data type + EILSEQ = 85 # Illegal byte sequence + ENOTSUP = 86 # Not supported + ECANCELED = 87 # Operation canceled + EBADMSG = 88 # Bad or Corrupt message + ENODATA = 89 # No message available + ENOSR = 90 # No STREAM resources + ENOSTR = 91 # Not a STREAM + ETIME = 92 # STREAM ioctl timeout + ENOATTR = 93 # Attribute not found + EMULTIHOP = 94 # Multihop attempted + ENOLINK = 95 # Link has been severed + EPROTO = 96 # Protocol error +end diff --git a/src/lib_c/x86_64-netbsd/c/fcntl.cr b/src/lib_c/x86_64-netbsd/c/fcntl.cr new file mode 100644 index 000000000000..76ff615bef36 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/fcntl.cr @@ -0,0 +1,32 @@ +require "./sys/types" +require "./sys/stat" +require "./unistd" + +lib LibC + F_GETFD = 1 + F_SETFD = 2 + F_GETFL = 3 + F_SETFL = 4 + FD_CLOEXEC = 1 + O_CLOEXEC = 0x400000 + O_CREAT = 0x0200 + O_NOFOLLOW = 0x0100 + O_TRUNC = 0x0400 + O_APPEND = 0x0008 + O_NONBLOCK = 0x0004 + O_SYNC = 0x0080 + O_RDONLY = 0x0000 + O_RDWR = 0x0002 + O_WRONLY = 0x0001 + + struct Flock + l_start : OffT + l_len : OffT + l_pid : PidT + l_type : Short + l_whence : Short + end + + fun fcntl(x0 : Int, x1 : Int, ...) : Int + fun open(x0 : Char*, x1 : Int, ...) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/grp.cr b/src/lib_c/x86_64-netbsd/c/grp.cr new file mode 100644 index 000000000000..2fc351260b20 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/grp.cr @@ -0,0 +1,11 @@ +lib LibC + struct Group + gr_name : Char* + gr_passwd : Char* + gr_gid : GidT + gr_mem : Char** + end + + fun getgrnam_r(name : Char*, grp : Group*, buf : Char*, bufsize : SizeT, result : Group**) : Int + fun getgrgid_r(gid : GidT, grp : Group*, buf : Char*, bufsize : SizeT, result : Group**) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/iconv.cr b/src/lib_c/x86_64-netbsd/c/iconv.cr new file mode 100644 index 000000000000..6814db786946 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/iconv.cr @@ -0,0 +1,9 @@ +require "./stddef" + +lib LibC + type IconvT = Void* + + fun iconv(cd : IconvT, inbuf : Char**, inbytesleft : SizeT*, outbuf : Char**, outbytesleft : SizeT*) : SizeT + fun iconv_close(cd : IconvT) : Int + fun iconv_open(tocode : Char*, fromcode : Char*) : IconvT +end diff --git a/src/lib_c/x86_64-netbsd/c/link.cr b/src/lib_c/x86_64-netbsd/c/link.cr new file mode 100644 index 000000000000..eb469a885889 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/link.cr @@ -0,0 +1,17 @@ +require "./elf" + +lib LibC + struct DlPhdrInfo + addr : Elf_Addr + name : Char* + phdr : Elf_Phdr* + phnum : Elf_Half + adds : UInt64 + subs : UInt64 + tls_modid : SizeT + tls_data : Void* + end + + alias DlPhdrCallback = (DlPhdrInfo*, LibC::SizeT, Void*) -> LibC::Int + fun dl_iterate_phdr(callback : DlPhdrCallback, data : Void*) +end diff --git a/src/lib_c/x86_64-netbsd/c/netdb.cr b/src/lib_c/x86_64-netbsd/c/netdb.cr new file mode 100644 index 000000000000..4443325cd487 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/netdb.cr @@ -0,0 +1,37 @@ +require "./netinet/in" +require "./sys/socket" +require "./stdint" + +lib LibC + AI_PASSIVE = 0x1 + AI_CANONNAME = 0x2 + AI_NUMERICHOST = 0x4 + AI_NUMERICSERV = 0x8 + AI_ADDRCONFIG = 0x400 + EAI_AGAIN = 2 + EAI_BADFLAGS = 3 + EAI_FAIL = 4 + EAI_FAMILY = 5 + EAI_MEMORY = 6 + EAI_NONAME = 8 + EAI_SERVICE = 9 + EAI_SOCKTYPE = 10 + EAI_SYSTEM = 11 + EAI_OVERFLOW = 14 + + struct Addrinfo + ai_flags : Int + ai_family : Int + ai_socktype : Int + ai_protocol : Int + ai_addrlen : SocklenT + ai_canonname : Char* + ai_addr : Void* + ai_next : Addrinfo* + end + + fun freeaddrinfo(x0 : Addrinfo*) : Void + fun gai_strerror(x0 : Int) : Char* + fun getaddrinfo(x0 : Char*, x1 : Char*, x2 : Addrinfo*, x3 : Addrinfo**) : Int + fun getnameinfo(x0 : Void*, x1 : SocklenT, x2 : Char*, x3 : SizeT, x4 : Char*, x5 : SizeT, x6 : Int) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/netinet/in.cr b/src/lib_c/x86_64-netbsd/c/netinet/in.cr new file mode 100644 index 000000000000..c400a8cec4d2 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/netinet/in.cr @@ -0,0 +1,70 @@ +require "../sys/socket" +require "../stdint" + +lib LibC + IPPROTO_IP = 0 + IPPROTO_IPV6 = 41 + IPPROTO_ICMP = 1 + IPPROTO_RAW = 255 + IPPROTO_TCP = 6 + IPPROTO_UDP = 17 + + alias InPortT = UInt16 + alias InAddrT = UInt32 + + struct InAddr + s_addr : InAddrT + end + + union In6AddrU6Addr + __u6_addr8 : StaticArray(UInt8T, 16) + __u6_addr16 : StaticArray(UInt16T, 8) + __u6_addr32 : StaticArray(UInt32T, 4) + end + + struct In6Addr + __u6_addr : In6AddrU6Addr + end + + struct SockaddrIn + sin_len : UInt8T + sin_family : SaFamilyT + sin_port : InPortT + sin_addr : InAddr + sin_zero : StaticArray(Int8, 8) + end + + struct SockaddrIn6 + sin6_len : UInt8T + sin6_family : SaFamilyT + sin6_port : InPortT + sin6_flowinfo : UInt32T + sin6_addr : In6Addr + sin6_scope_id : UInt32T + end + + IP_MULTICAST_IF = 9 + IPV6_MULTICAST_IF = 9 + + IP_MULTICAST_TTL = 10 + IPV6_MULTICAST_HOPS = 10 + + IP_MULTICAST_LOOP = 11 + IPV6_MULTICAST_LOOP = 11 + + IP_ADD_MEMBERSHIP = 12 + IPV6_JOIN_GROUP = 12 + + IP_DROP_MEMBERSHIP = 13 + IPV6_LEAVE_GROUP = 13 + + struct IpMreq + imr_multiaddr : InAddr + imr_interface : InAddr + end + + struct Ipv6Mreq + ipv6mr_multiaddr : In6Addr + ipv6mr_interface : UInt + end +end diff --git a/src/lib_c/x86_64-netbsd/c/netinet/tcp.cr b/src/lib_c/x86_64-netbsd/c/netinet/tcp.cr new file mode 100644 index 000000000000..3f164c117c68 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/netinet/tcp.cr @@ -0,0 +1,5 @@ +lib LibC + TCP_NODELAY = 1 + TCP_KEEPINTVL = 5 + TCP_KEEPCNT = 6 +end diff --git a/src/lib_c/x86_64-netbsd/c/pthread.cr b/src/lib_c/x86_64-netbsd/c/pthread.cr new file mode 100644 index 000000000000..133d4922c57f --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/pthread.cr @@ -0,0 +1,39 @@ +require "./sys/types" + +@[Link("pthread")] +lib LibC + PTHREAD_MUTEX_ERRORCHECK = 1 + + fun pthread_attr_destroy(attr : PthreadAttrT*) : Int + fun pthread_attr_get_np(x0 : PthreadT, x1 : PthreadAttrT*) : Int + fun pthread_attr_getstack(addr : PthreadAttrT*, stackaddr : Void**, stacksize : SizeT*) : Int + fun pthread_attr_init(attr : PthreadAttrT*) : Int + fun pthread_condattr_destroy(x0 : PthreadCondattrT*) : Int + fun pthread_condattr_init(x0 : PthreadCondattrT*) : Int + fun pthread_condattr_setclock(x0 : PthreadCondattrT*, x1 : ClockidT) : Int + fun pthread_cond_broadcast(x0 : PthreadCondT*) : Int + fun pthread_cond_destroy(x0 : PthreadCondT*) : Int + fun pthread_cond_init(x0 : PthreadCondT*, x1 : PthreadCondattrT*) : Int + fun pthread_cond_signal(x0 : PthreadCondT*) : Int + fun pthread_cond_timedwait(x0 : PthreadCondT*, x1 : PthreadMutexT*, x2 : Timespec*) : Int + fun pthread_cond_wait(x0 : PthreadCondT*, x1 : PthreadMutexT*) : Int + fun pthread_create(x0 : PthreadT*, x1 : PthreadAttrT*, x2 : Void* -> Void*, x3 : Void*) : Int + fun pthread_detach(x0 : PthreadT) : Int + fun pthread_getattr_np(thread : PthreadT, attr : PthreadAttrT*) : Int + fun pthread_equal(x0 : PthreadT, x1 : PthreadT) : Int + fun pthread_getspecific(PthreadKeyT) : Void* + fun pthread_join(x0 : PthreadT, x1 : Void**) : Int + alias PthreadKeyDestructor = (Void*) -> + fun pthread_key_create(PthreadKeyT*, PthreadKeyDestructor) : Int + fun pthread_main_np : Int + fun pthread_mutexattr_destroy(x0 : PthreadMutexattrT*) : Int + fun pthread_mutexattr_init(x0 : PthreadMutexattrT*) : Int + fun pthread_mutexattr_settype(x0 : PthreadMutexattrT*, x1 : Int) : Int + fun pthread_mutex_destroy(x0 : PthreadMutexT*) : Int + fun pthread_mutex_init(x0 : PthreadMutexT*, x1 : PthreadMutexattrT*) : Int + fun pthread_mutex_lock(x0 : PthreadMutexT*) : Int + fun pthread_mutex_trylock(x0 : PthreadMutexT*) : Int + fun pthread_mutex_unlock(x0 : PthreadMutexT*) : Int + fun pthread_self : PthreadT + fun pthread_setspecific(PthreadKeyT, Void*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/pwd.cr b/src/lib_c/x86_64-netbsd/c/pwd.cr new file mode 100644 index 000000000000..250e8397f709 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/pwd.cr @@ -0,0 +1,17 @@ +lib LibC + struct Passwd + pw_name : Char* + pw_passwd : Char* + pw_uid : UidT + pw_gid : GidT + pw_change : TimeT + pw_class : Char* + pw_gecos : Char* + pw_dir : Char* + pw_shell : Char* + pw_expire : TimeT + end + + fun getpwnam_r = __getpwnam_r50(login : Char*, pwstore : Passwd*, buf : Char*, bufsize : SizeT, result : Passwd**) : Int + fun getpwuid_r = __getpwuid_r50(uid : UidT, pwstore : Passwd*, buf : Char*, bufsize : SizeT, result : Passwd**) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sched.cr b/src/lib_c/x86_64-netbsd/c/sched.cr new file mode 100644 index 000000000000..9d83ed18504e --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sched.cr @@ -0,0 +1,3 @@ +lib LibC + fun sched_yield : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/signal.cr b/src/lib_c/x86_64-netbsd/c/signal.cr new file mode 100644 index 000000000000..a9d866834f17 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/signal.cr @@ -0,0 +1,55 @@ +require "./sys/types" +require "./time" + +lib LibC + SIGHUP = 1 + SIGINT = 2 + SIGQUIT = 3 + SIGILL = 4 + SIGTRAP = 5 + SIGIOT = LibC::SIGABRT + SIGABRT = 6 + SIGEMT = 7 + SIGFPE = 8 + SIGKILL = 9 + SIGBUS = 10 + SIGSEGV = 11 + SIGSYS = 12 + SIGPIPE = 13 + SIGALRM = 14 + SIGTERM = 15 + SIGURG = 16 + SIGSTOP = 17 + SIGTSTP = 18 + SIGCONT = 19 + SIGCHLD = 20 + SIGTTIN = 21 + SIGTTOU = 22 + SIGIO = 23 + SIGXCPU = 24 + SIGXFSZ = 25 + SIGVTALRM = 26 + SIGUSR1 = 30 + SIGUSR2 = 31 + SIGINFO = 29 + SIGWINCH = 28 + + SIG_SETMASK = 3 + + alias SighandlerT = Int -> + SIG_DFL = SighandlerT.new(Pointer(Void).new(0_u64), Pointer(Void).null) + SIG_IGN = SighandlerT.new(Pointer(Void).new(1_u64), Pointer(Void).null) + + struct SigsetT + bits : UInt32[4] + end + + fun kill(x0 : PidT, x1 : Int) : Int + fun pthread_sigmask(Int, SigsetT*, SigsetT*) : Int + fun signal(x0 : Int, x1 : Int -> Void) : Int -> Void + fun sigemptyset = __sigemptyset14(SigsetT*) : Int + fun sigfillset = __sigfillset14(SigsetT*) : Int + fun sigaddset = __sigaddset14(SigsetT*, Int) : Int + fun sigdelset = __sigdelset14(SigsetT*, Int) : Int + fun sigismember = __sigismember14(SigsetT*, Int) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/stdarg.cr b/src/lib_c/x86_64-netbsd/c/stdarg.cr new file mode 100644 index 000000000000..fcad7714f16a --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/stdarg.cr @@ -0,0 +1,10 @@ +lib LibC + struct VaListTag + gp_offset : UInt + fp_offset : UInt + overflow_arg_area : Void* + reg_save_area : Void* + end + + type VaList = VaListTag[1] +end diff --git a/src/lib_c/x86_64-netbsd/c/stddef.cr b/src/lib_c/x86_64-netbsd/c/stddef.cr new file mode 100644 index 000000000000..4afcdf34d723 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/stddef.cr @@ -0,0 +1,3 @@ +lib LibC + alias SizeT = ULong +end diff --git a/src/lib_c/x86_64-netbsd/c/stdint.cr b/src/lib_c/x86_64-netbsd/c/stdint.cr new file mode 100644 index 000000000000..5cb258eccd2c --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/stdint.cr @@ -0,0 +1,10 @@ +lib LibC + alias Int8T = SChar + alias Int16T = Short + alias Int32T = Int + alias Int64T = LongLong + alias UInt8T = Char + alias UInt16T = UShort + alias UInt32T = UInt + alias UInt64T = ULongLong +end diff --git a/src/lib_c/x86_64-netbsd/c/stdio.cr b/src/lib_c/x86_64-netbsd/c/stdio.cr new file mode 100644 index 000000000000..44fb5b8a1e74 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/stdio.cr @@ -0,0 +1,9 @@ +require "./sys/types" +require "./stddef" + +lib LibC + fun printf(x0 : Char*, ...) : Int + fun dprintf(fd : Int, format : Char*, ...) : Int + fun rename(x0 : Char*, x1 : Char*) : Int + fun snprintf(x0 : Char*, x1 : SizeT, x2 : Char*, ...) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/stdlib.cr b/src/lib_c/x86_64-netbsd/c/stdlib.cr new file mode 100644 index 000000000000..49aada507901 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/stdlib.cr @@ -0,0 +1,27 @@ +require "./stddef" +require "./sys/wait" + +lib LibC + struct DivT + quot : Int + rem : Int + end + + fun arc4random : UInt32 + fun arc4random_buf(x0 : Void*, x1 : SizeT) : Void + fun atof(x0 : Char*) : Double + fun div(x0 : Int, x1 : Int) : DivT + fun exit(x0 : Int) : NoReturn + fun free(x0 : Void*) : Void + fun getenv(x0 : Char*) : Char* + fun malloc(x0 : SizeT) : Void* + fun mkstemp(x0 : Char*) : Int + fun mkstemps(x0 : Char*, x1 : Int) : Int + fun putenv(x0 : Char*) : Int + fun realloc(x0 : Void*, x1 : SizeT) : Void* + fun realpath(x0 : Char*, x1 : Char*) : Char* + fun setenv(x0 : Char*, x1 : Char*, x2 : Int) : Int + fun strtof(x0 : Char*, x1 : Char**) : Float + fun strtod(x0 : Char*, x1 : Char**) : Double + fun unsetenv = __unsetenv13(x0 : Char*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/string.cr b/src/lib_c/x86_64-netbsd/c/string.cr new file mode 100644 index 000000000000..471d1ed82b36 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/string.cr @@ -0,0 +1,9 @@ +require "./stddef" + +lib LibC + fun memchr(x0 : Void*, c : Int, n : SizeT) : Void* + fun memcmp(x0 : Void*, x1 : Void*, x2 : SizeT) : Int + fun strcmp(x0 : Char*, x1 : Char*) : Int + fun strerror(x0 : Int) : Char* + fun strlen(x0 : Char*) : ULong +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/file.cr b/src/lib_c/x86_64-netbsd/c/sys/file.cr new file mode 100644 index 000000000000..e2bd3fd4b816 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/file.cr @@ -0,0 +1,11 @@ +lib LibC + @[Flags] + enum FlockOp + SH = 0x1 + EX = 0x2 + NB = 0x4 + UN = 0x8 + end + + fun flock(fd : Int, op : FlockOp) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/mman.cr b/src/lib_c/x86_64-netbsd/c/sys/mman.cr new file mode 100644 index 000000000000..2c6675659c2f --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/mman.cr @@ -0,0 +1,30 @@ +require "./types" + +lib LibC + PROT_NONE = 0x00 + PROT_READ = 0x01 + PROT_WRITE = 0x02 + PROT_EXEC = 0x04 + MAP_SHARED = 0x0001 + MAP_PRIVATE = 0x0002 + MAP_FIXED = 0x0010 + MAP_ANON = 0x1000 + MAP_ANONYMOUS = LibC::MAP_ANON + MAP_FAILED = Pointer(Void).new(-1) + MAP_STACK = 0x2000 + POSIX_MADV_NORMAL = 0 + POSIX_MADV_RANDOM = 1 + POSIX_MADV_SEQUENTIAL = 2 + POSIX_MADV_WILLNEED = 3 + POSIX_MADV_DONTNEED = 4 + MADV_NORMAL = LibC::POSIX_MADV_NORMAL + MADV_RANDOM = LibC::POSIX_MADV_RANDOM + MADV_SEQUENTIAL = LibC::POSIX_MADV_SEQUENTIAL + MADV_WILLNEED = LibC::POSIX_MADV_WILLNEED + MADV_DONTNEED = LibC::POSIX_MADV_DONTNEED + + fun mmap(x0 : Void*, x1 : SizeT, x2 : Int, x3 : Int, x4 : Int, x5 : OffT) : Void* + fun mprotect(x0 : Void*, x1 : SizeT, x2 : Int) : Int + fun munmap(x0 : Void*, x1 : SizeT) : Int + fun madvise(x0 : Void*, x1 : SizeT, x2 : Int) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/resource.cr b/src/lib_c/x86_64-netbsd/c/sys/resource.cr new file mode 100644 index 000000000000..d52182f69bce --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/resource.cr @@ -0,0 +1,25 @@ +lib LibC + struct RUsage + ru_utime : Timeval + ru_stime : Timeval + ru_maxrss : Long + ru_ixrss : Long + ru_idrss : Long + ru_isrss : Long + ru_minflt : Long + ru_majflt : Long + ru_nswap : Long + ru_inblock : Long + ru_oublock : Long + ru_msgsnd : Long + ru_msgrcv : Long + ru_nsignals : Long + ru_nvcsw : Long + ru_nivcsw : Long + end + + RUSAGE_SELF = 0 + RUSAGE_CHILDREN = -1 + + fun getrusage(who : Int, usage : RUsage*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/select.cr b/src/lib_c/x86_64-netbsd/c/sys/select.cr new file mode 100644 index 000000000000..cbbd58604883 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/select.cr @@ -0,0 +1,12 @@ +require "./types" +require "./time" +require "../time" +require "../signal" + +lib LibC + struct FdSet + fds_bits : StaticArray(UInt32T, 8) + end + + fun select(x0 : Int, x1 : FdSet*, x2 : FdSet*, x3 : FdSet*, x4 : Timeval*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/socket.cr b/src/lib_c/x86_64-netbsd/c/sys/socket.cr new file mode 100644 index 000000000000..d96f245bc42a --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/socket.cr @@ -0,0 +1,68 @@ +require "./types" + +lib LibC + SOCK_STREAM = 1 + SOCK_DGRAM = 2 + SOCK_RAW = 3 + SOCK_SEQPACKET = 5 + SOL_SOCKET = 0xffff + SO_BROADCAST = 0x0020 + SO_KEEPALIVE = 0x0008 + SO_LINGER = 0x0080 + SO_RCVBUF = 0x1002 + SO_REUSEADDR = 0x0004 + SO_REUSEPORT = 0x0200 + SO_SNDBUF = 0x1001 + PF_INET = LibC::AF_INET + PF_INET6 = LibC::AF_INET6 + PF_UNIX = LibC::PF_LOCAL + PF_UNSPEC = LibC::AF_UNSPEC + PF_LOCAL = LibC::AF_LOCAL + AF_INET = 2 + AF_INET6 = 24 + AF_UNIX = LibC::AF_LOCAL + AF_UNSPEC = 0 + AF_LOCAL = 1 + SHUT_RD = 0 + SHUT_WR = 1 + SHUT_RDWR = 2 + SOCK_CLOEXEC = 0x10000000 + + alias SocklenT = UInt + alias SaFamilyT = UInt8 + + struct Sockaddr + sa_len : UInt8 + sa_family : SaFamilyT + sa_data : StaticArray(Char, 14) + end + + struct SockaddrStorage + ss_len : UInt8 + ss_family : SaFamilyT + __ss_pad1 : StaticArray(Char, 6) + __ss_pad2 : UInt64 + __ss_pad3 : StaticArray(Char, 240) + end + + struct Linger + l_onoff : Int + l_linger : Int + end + + fun accept(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun bind(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int + fun connect(x0 : Int, x1 : Sockaddr*, x2 : SocklenT) : Int + fun getpeername(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun getsockname(x0 : Int, x1 : Sockaddr*, x2 : SocklenT*) : Int + fun getsockopt(x0 : Int, x1 : Int, x2 : Int, x3 : Void*, x4 : SocklenT*) : Int + fun listen(x0 : Int, x1 : Int) : Int + fun recv(x0 : Int, x1 : Void*, x2 : SizeT, x3 : Int) : SSizeT + fun recvfrom(x0 : Int, x1 : Void*, x2 : SizeT, x3 : Int, x4 : Sockaddr*, x5 : SocklenT*) : SSizeT + fun send(x0 : Int, x1 : Void*, x2 : SizeT, x3 : Int) : SSizeT + fun sendto(x0 : Int, x1 : Void*, x2 : SizeT, x3 : Int, x4 : Sockaddr*, x5 : SocklenT) : SSizeT + fun setsockopt(x0 : Int, x1 : Int, x2 : Int, x3 : Void*, x4 : SocklenT) : Int + fun shutdown(x0 : Int, x1 : Int) : Int + fun socket = __socket30(x0 : Int, x1 : Int, x2 : Int) : Int + fun socketpair(x0 : Int, x1 : Int, x2 : Int, x3 : Int*) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/stat.cr b/src/lib_c/x86_64-netbsd/c/sys/stat.cr new file mode 100644 index 000000000000..68c44a28df9e --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/stat.cr @@ -0,0 +1,57 @@ +require "./types" +require "../time" + +lib LibC + S_IFMT = 0o170000 + S_IFBLK = 0o060000 + S_IFCHR = 0o020000 + S_IFIFO = 0o010000 + S_IFREG = 0o100000 + S_IFDIR = 0o040000 + S_IFLNK = 0o120000 + S_IFSOCK = 0o140000 + S_IRUSR = 0o000400 + S_IWUSR = 0o000200 + S_IXUSR = 0o000100 + S_IRWXU = 0o000700 + S_IRGRP = 0o000040 + S_IWGRP = 0o000020 + S_IXGRP = 0o000010 + S_IRWXG = 0o000070 + S_IROTH = 0o000004 + S_IWOTH = 0o000002 + S_IXOTH = 0o000001 + S_IRWXO = 0o000007 + S_ISUID = 0o004000 + S_ISGID = 0o002000 + S_ISVTX = 0o001000 + + struct Stat + st_dev : DevT + st_mode : ModeT + st_ino : InoT + st_nlink : NlinkT + st_uid : UidT + st_gid : GidT + st_rdev : DevT + st_atim : Timespec + st_mtim : Timespec + st_ctim : Timespec + st_birthtim : Timespec + st_size : OffT + st_blocks : BlkcntT + st_blksize : BlksizeT + st_flags : UInt32T + st_gen : UInt32T + st_spare : UInt32[2] + end + + fun chmod(x0 : Char*, x1 : ModeT) : Int + fun fstat = __fstat50(x0 : Int, x1 : Stat*) : Int + fun lstat = __lstat50(x0 : Char*, x1 : Stat*) : Int + fun mkdir(x0 : Char*, x1 : ModeT) : Int + fun mkfifo(x0 : Char*, x1 : ModeT) : Int + fun mknod = __mknod50(x0 : Char*, x1 : ModeT, x2 : DevT) : Int + fun stat = __stat50(x0 : Char*, x1 : Stat*) : Int + fun umask(x0 : ModeT) : ModeT +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/time.cr b/src/lib_c/x86_64-netbsd/c/sys/time.cr new file mode 100644 index 000000000000..1677150d4851 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/time.cr @@ -0,0 +1,16 @@ +require "./types" + +lib LibC + struct Timeval + tv_sec : TimeT + tv_usec : SusecondsT + end + + struct Timezone + tz_minuteswest : Int + tz_dsttime : Int + end + + fun gettimeofday = __gettimeofday50(x0 : Timeval*, x1 : Timezone*) : Int + fun utimes = __utimes50(path : Char*, times : Timeval[2]) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/types.cr b/src/lib_c/x86_64-netbsd/c/sys/types.cr new file mode 100644 index 000000000000..198bfb681218 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/types.cr @@ -0,0 +1,67 @@ +require "../stddef" +require "../stdint" + +lib LibC + alias BlkcntT = Int64 + alias BlksizeT = Int32 + alias ClockT = UInt + alias ClockidT = Int + alias DevT = UInt64 + alias GidT = UInt32 + alias IdT = UInt32 + alias InoT = UInt64 + alias ModeT = UInt32 + alias NlinkT = UInt32 + alias OffT = Int64 + alias PidT = Int32 + + type PthreadT = Void* + + struct PthreadAttrT + pta_magic : UInt + pta_flags : Int + pta_private : Void* + end + + struct PthreadQueueT + ptqh_first : Void* + ptqh_last : Void* + end + + struct PthreadCondT + ptc_magic : UInt + ptc_lock : UInt8 + ptc_waiters : PthreadQueueT + ptc_mutex : Void* + ptc_private : Void* + end + + struct PthreadCondattrT + ptca_magic : UInt + ptca_private : Void* + end + + type PthreadKeyT = Int + + struct PthreadMutexT + ptm_magic : UInt + ptm_errorcheck : UInt8 + ptm_pad1 : UInt8[3] + ptm_ceiling : UInt8 + ptm_pad2 : UInt8[2] + ptm_owner : PthreadT + ptm_waiters : PthreadT* + ptm_recursed : UInt + ptm_spare2 : Void* + end + + struct PthreadMutexattrT + ptma_magic : UInt + ptma_private : Void* + end + + alias SSizeT = Long + alias SusecondsT = UInt + alias TimeT = Int64 + alias UidT = UInt32 +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/un.cr b/src/lib_c/x86_64-netbsd/c/sys/un.cr new file mode 100644 index 000000000000..476b28c6ed57 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/un.cr @@ -0,0 +1,9 @@ +require "./socket" + +lib LibC + struct SockaddrUn + sun_len : UInt8 + sun_family : SaFamilyT + sun_path : StaticArray(Char, 104) + end +end diff --git a/src/lib_c/x86_64-netbsd/c/sys/wait.cr b/src/lib_c/x86_64-netbsd/c/sys/wait.cr new file mode 100644 index 000000000000..e9c185b5ac9c --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sys/wait.cr @@ -0,0 +1,8 @@ +require "./types" +require "../signal" + +lib LibC + WNOHANG = 1 + + fun waitpid(x0 : PidT, x1 : Int*, x2 : Int) : PidT +end diff --git a/src/lib_c/x86_64-netbsd/c/sysctl.cr b/src/lib_c/x86_64-netbsd/c/sysctl.cr new file mode 100644 index 000000000000..c9102ff0a7b3 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/sysctl.cr @@ -0,0 +1,13 @@ +lib LibC + CTL_HW = 6 + HW_NCPU = 3 + + CTL_KERN = 1 + KERN_PROC = 14 + KERN_PROC_ARGS = 48 + KERN_PROC_PATHNAME = 5 + + PATH_MAX = 1024 + + fun sysctl(name : Int*, namelen : UInt, oldp : Void*, oldlenp : SizeT*, newp : Void*, newlen : SizeT) : Int +end diff --git a/src/lib_c/x86_64-netbsd/c/termios.cr b/src/lib_c/x86_64-netbsd/c/termios.cr new file mode 100644 index 000000000000..f788b9bd41f1 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/termios.cr @@ -0,0 +1,95 @@ +require "./sys/types" + +lib LibC + VEOF = 0 + VEOL = 1 + VERASE = 3 + VINTR = 8 + VKILL = 5 + VMIN = 16 + VQUIT = 9 + VSTART = 12 + VSTOP = 13 + VSUSP = 10 + BRKINT = 0x00000002 + ICRNL = 0x00000100 + IGNBRK = 0x00000001 + IGNCR = 0x00000080 + IGNPAR = 0x00000004 + INLCR = 0x00000040 + INPCK = 0x00000010 + ISTRIP = 0x00000020 + IXANY = 0x00000800 + IXOFF = 0x00000400 + IXON = 0x00000200 + PARMRK = 0x00000008 + OPOST = 0x00000001 + ONLCR = 0x00000002 + OCRNL = 0x00000010 + ONOCR = 0x00000040 + ONLRET = 0x00000080 + B0 = 0 + B50 = 50 + B75 = 75 + B110 = 110 + B134 = 134 + B150 = 150 + B200 = 200 + B300 = 300 + B600 = 600 + B1200 = 1200 + B1800 = 1800 + B2400 = 2400 + B4800 = 4800 + B9600 = 9600 + B19200 = 19200 + B38400 = 38400 + CSIZE = 0x00000300 + CS5 = 0x00000000 + CS6 = 0x00000100 + CS7 = 0x00000200 + CS8 = 0x00000300 + CSTOPB = 0x00000400 + CREAD = 0x00000800 + PARENB = 0x00001000 + PARODD = 0x00002000 + HUPCL = 0x00004000 + CLOCAL = 0x00008000 + ECHO = 0x00000008 + ECHOE = 0x00000002 + ECHOK = 0x00000004 + ECHONL = 0x00000010 + ICANON = 0x00000100 + IEXTEN = 0x00000400 + ISIG = 0x00000080 + NOFLSH = 0x80000000 + TOSTOP = 0x00400000 + TCSANOW = 0 + TCSADRAIN = 1 + TCSAFLUSH = 2 + TCIFLUSH = 1 + TCIOFLUSH = 3 + TCOFLUSH = 2 + TCIOFF = 3 + TCION = 4 + TCOOFF = 1 + TCOON = 2 + + alias CcT = Char + alias SpeedT = UInt + alias TcflagT = UInt + + struct Termios + c_iflag : TcflagT + c_oflag : TcflagT + c_cflag : TcflagT + c_lflag : TcflagT + c_cc : StaticArray(CcT, 20) + c_ispeed : Int + c_ospeed : Int + end + + fun tcgetattr(x0 : Int, x1 : Termios*) : Int + fun tcsetattr(x0 : Int, x1 : Int, x2 : Termios*) : Int + fun cfmakeraw(x0 : Termios*) : Void +end diff --git a/src/lib_c/x86_64-netbsd/c/time.cr b/src/lib_c/x86_64-netbsd/c/time.cr new file mode 100644 index 000000000000..17fb6b2dcaa6 --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/time.cr @@ -0,0 +1,37 @@ +require "./sys/types" + +lib LibC + CLOCK_REALTIME = 0 + CLOCK_MONOTONIC = 3 + + struct Tm + tm_sec : Int + tm_min : Int + tm_hour : Int + tm_mday : Int + tm_mon : Int + tm_year : Int + tm_wday : Int + tm_yday : Int + tm_isdst : Int + tm_gmtoff : Long + tm_zone : Char* + end + + struct Timespec + tv_sec : TimeT + tv_nsec : Long + end + + fun clock_gettime = __clock_gettime50(x0 : ClockidT, x1 : Timespec*) : Int + fun clock_settime = __clock_settime50(x0 : ClockidT, x1 : Timespec*) : Int + fun gmtime_r = __gmtime_r50(x0 : TimeT*, x1 : Tm*) : Tm* + fun localtime_r = __localtime_r50(x0 : TimeT*, x1 : Tm*) : Tm* + fun mktime = __mktime50(x0 : Tm*) : TimeT + fun tzset : Void + fun timegm = __timegm50(x0 : Tm*) : TimeT + + $daylight : Int + $timezone : Long + $tzname : StaticArray(Char*, 2) +end diff --git a/src/lib_c/x86_64-netbsd/c/unistd.cr b/src/lib_c/x86_64-netbsd/c/unistd.cr new file mode 100644 index 000000000000..89c28c5d3f9f --- /dev/null +++ b/src/lib_c/x86_64-netbsd/c/unistd.cr @@ -0,0 +1,46 @@ +require "./sys/types" +require "./stdint" + +lib LibC + F_OK = 0 + X_OK = 0x01 + W_OK = 0x02 + R_OK = 0x04 + SC_CLK_TCK = 39 + SC_PAGESIZE = 28 + + fun chroot(dirname : Char*) : Int + fun access(x0 : Char*, x1 : Int) : Int + fun chdir(x0 : Char*) : Int + fun chown = __posix_chown(x0 : Char*, x1 : UidT, x2 : GidT) : Int + fun close(x0 : Int) : Int + fun dup2(x0 : Int, x1 : Int) : Int + fun _exit(x0 : Int) : NoReturn + fun execvp(x0 : Char*, x1 : Char**) : Int + fun fdatasync(x0 : Int) : Int + @[ReturnsTwice] + fun fork : PidT + fun fsync(x0 : Int) : Int + fun ftruncate(x0 : Int, x1 : OffT) : Int + fun getcwd(x0 : Char*, x1 : SizeT) : Char* + fun gethostname(x0 : Char*, x1 : SizeT) : Int + fun getpgid(x0 : PidT) : PidT + fun getpid : PidT + fun getppid : PidT + fun isatty(x0 : Int) : Int + fun ttyname_r(fd : Int, buf : Char*, buffersize : SizeT) : Int + fun lchown = __posix_lchown(x0 : Char*, x1 : UidT, x2 : GidT) : Int + fun link(x0 : Char*, x1 : Char*) : Int + fun lockf(x0 : Int, x1 : Int, x2 : OffT) : Int + fun lseek(x0 : Int, x1 : OffT, x2 : Int) : OffT + fun pipe(x0 : Int*) : Int + fun read(x0 : Int, x1 : Void*, x2 : SizeT) : SSizeT + fun pread(x0 : Int, x1 : Void*, x2 : SizeT, x3 : OffT) : SSizeT + fun rmdir(x0 : Char*) : Int + fun symlink(x0 : Char*, x1 : Char*) : Int + fun readlink(path : Char*, buf : Char*, size : SizeT) : SSizeT + fun syscall(x0 : Int, ...) : Int + fun sysconf(x0 : Int) : Long + fun unlink(x0 : Char*) : Int + fun write(x0 : Int, x1 : Void*, x2 : SizeT) : SSizeT +end diff --git a/src/math/libm.cr b/src/math/libm.cr index e04a83c505b9..0899fc3998ac 100644 --- a/src/math/libm.cr +++ b/src/math/libm.cr @@ -1,4 +1,4 @@ -{% if flag?(:linux) || flag?(:freebsd) || flag?(:openbsd) || flag?(:dragonfly) %} +{% if flag?(:linux) || flag?(:bsd) %} @[Link("m")] {% end %} diff --git a/src/process/executable_path.cr b/src/process/executable_path.cr index d713d5fe9620..b9c7498413da 100644 --- a/src/process/executable_path.cr +++ b/src/process/executable_path.cr @@ -87,6 +87,20 @@ end buf = GC.malloc_atomic(LibC::PATH_MAX).as(UInt8*) size = LibC::SizeT.new(LibC::PATH_MAX) + if LibC.sysctl(mib, 4, buf, pointerof(size), nil, 0) == 0 + String.new(buf, size - 1) + end + end + end +{% elsif flag?(:netbsd) %} + require "c/sysctl" + + class Process + private def self.executable_path_impl + mib = Int32[LibC::CTL_KERN, LibC::KERN_PROC_ARGS, -1, LibC::KERN_PROC_PATHNAME] + buf = GC.malloc_atomic(LibC::PATH_MAX).as(UInt8*) + size = LibC::SizeT.new(LibC::PATH_MAX) + if LibC.sysctl(mib, 4, buf, pointerof(size), nil, 0) == 0 String.new(buf, size - 1) end diff --git a/src/socket/address.cr b/src/socket/address.cr index b3dcaa5103ae..14b90be5023d 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -229,7 +229,7 @@ class Socket end private def ipv6_addr8(addr : LibC::In6Addr) - {% if flag?(:darwin) || flag?(:openbsd) || flag?(:freebsd) || flag?(:dragonfly) %} + {% if flag?(:darwin) || flag?(:bsd) %} addr.__u6_addr.__u6_addr8 {% elsif flag?(:linux) && flag?(:musl) %} addr.__in6_union.__s6_addr diff --git a/src/socket/tcp_socket.cr b/src/socket/tcp_socket.cr index 7a5bdbcd2d0f..e31736e947bc 100644 --- a/src/socket/tcp_socket.cr +++ b/src/socket/tcp_socket.cr @@ -75,6 +75,8 @@ class TCPSocket < IPSocket def tcp_keepalive_idle optname = {% if flag?(:darwin) %} LibC::TCP_KEEPALIVE + {% elsif flag?(:netbsd) %} + LibC::SO_KEEPALIVE {% else %} LibC::TCP_KEEPIDLE {% end %} @@ -84,6 +86,8 @@ class TCPSocket < IPSocket def tcp_keepalive_idle=(val : Int) optname = {% if flag?(:darwin) %} LibC::TCP_KEEPALIVE + {% elsif flag?(:netbsd) %} + LibC::SO_KEEPALIVE {% else %} LibC::TCP_KEEPIDLE {% end %} diff --git a/src/termios.cr b/src/termios.cr index 698086db9329..db34be00c8b1 100644 --- a/src/termios.cr +++ b/src/termios.cr @@ -42,7 +42,7 @@ module Termios TAB0 = LibC::TAB0 TAB3 = LibC::TAB3 end - {% elsif flag?(:openbsd) %} + {% elsif flag?(:netbsd) || flag?(:openbsd) %} @[Flags] enum OutputMode OPOST = LibC::OPOST From 586d7c36eb06c0edf196ce04b80c4927c446904f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Thu, 2 Jul 2020 22:23:47 +0200 Subject: [PATCH 169/263] Print original exception after failing to raise it (#9220) --- src/raise.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/raise.cr b/src/raise.cr index 638b81553c88..126923c534ad 100644 --- a/src/raise.cr +++ b/src/raise.cr @@ -195,6 +195,7 @@ end ret = LibUnwind.raise_exception(unwind_ex) Crystal::System.print_error "Failed to raise an exception: %s\n", ret.to_s Exception::CallStack.print_backtrace + Crystal::System.print_exception("\nTried to raise:", unwind_ex.value.exception_object.as(Exception)) LibC.exit(ret) end From 3857fbb80f8cfa70eee7af5a9c3209ecd62d38e0 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Fri, 3 Jul 2020 11:39:32 -0300 Subject: [PATCH 170/263] Fix: `Socket#accept` obeys `read_timeout` (#9538) --- spec/std/socket/socket_spec.cr | 11 +++++++++++ src/io/evented.cr | 12 ++++++------ src/socket.cr | 11 +++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/spec/std/socket/socket_spec.cr b/spec/std/socket/socket_spec.cr index c0ae54291d84..2ab9ea5f6f82 100644 --- a/spec/std/socket/socket_spec.cr +++ b/spec/std/socket/socket_spec.cr @@ -42,6 +42,17 @@ describe Socket do end end + it "accept raises timeout error if read_timeout is specified" do + server = Socket.new(Socket::Family::INET, Socket::Type::STREAM, Socket::Protocol::TCP) + port = unused_local_port + server.bind("0.0.0.0", port) + server.read_timeout = 0.1 + server.listen + + expect_raises(IO::TimeoutError) { server.accept } + expect_raises(IO::TimeoutError) { server.accept? } + end + it "sends messages" do port = unused_local_port server = Socket.tcp(Socket::Family::INET) diff --git a/src/io/evented.cr b/src/io/evented.cr index 15e802eb1c07..2b47725b3837 100644 --- a/src/io/evented.cr +++ b/src/io/evented.cr @@ -117,11 +117,11 @@ module IO::Evented # :nodoc: def wait_readable(timeout = @read_timeout) - wait_readable(timeout: timeout) { |err| raise err } + wait_readable(timeout: timeout) { raise TimeoutError.new("Read timed out") } end # :nodoc: - def wait_readable(timeout = @read_timeout) : Nil + def wait_readable(timeout = @read_timeout, *, raise_if_closed = true) : Nil readers = @readers.get { Deque(Fiber).new } readers << Fiber.current add_read_event(timeout) @@ -129,10 +129,10 @@ module IO::Evented if @read_timed_out @read_timed_out = false - yield TimeoutError.new("Read timed out") + yield end - check_open + check_open if raise_if_closed end private def add_read_event(timeout = @read_timeout) : Nil @@ -142,7 +142,7 @@ module IO::Evented # :nodoc: def wait_writable(timeout = @write_timeout) - wait_writable(timeout: timeout) { |err| raise err } + wait_writable(timeout: timeout) { raise TimeoutError.new("Write timed out") } end # :nodoc: @@ -154,7 +154,7 @@ module IO::Evented if @write_timed_out @write_timed_out = false - yield TimeoutError.new("Write timed out") + yield end check_open diff --git a/src/socket.cr b/src/socket.cr index 634939432ba6..5e13b00f0dee 100644 --- a/src/socket.cr +++ b/src/socket.cr @@ -155,7 +155,7 @@ class Socket < IO when Errno::EISCONN return when Errno::EINPROGRESS, Errno::EALREADY - wait_writable(timeout: timeout) do |error| + wait_writable(timeout: timeout) do return yield IO::TimeoutError.new("connect timed out") end else @@ -271,7 +271,8 @@ class Socket < IO if closed? return elsif Errno.value == Errno::EAGAIN - wait_readable rescue nil + wait_acceptable + return if closed? else raise Socket::Error.from_errno("accept") end @@ -281,6 +282,12 @@ class Socket < IO end end + private def wait_acceptable + wait_readable(raise_if_closed: false) do + raise TimeoutError.new("Accept timed out") + end + end + # Sends a message to a previously connected remote address. # # ``` From 794a98a879a19bfdc6f37ee640c60ceccbbd742e Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 3 Jul 2020 19:09:39 -0300 Subject: [PATCH 171/263] Compiler: implement autocasting in a better way (#9501) --- spec/compiler/codegen/automatic_cast.cr | 21 +++++ spec/compiler/semantic/automatic_cast_spec.cr | 29 ++++++ src/compiler/crystal/program.cr | 4 +- src/compiler/crystal/semantic/call.cr | 44 ++++----- .../crystal/semantic/method_lookup.cr | 94 ++++++++++++++----- src/compiler/crystal/semantic/restrictions.cr | 15 +++ 6 files changed, 155 insertions(+), 52 deletions(-) diff --git a/spec/compiler/codegen/automatic_cast.cr b/spec/compiler/codegen/automatic_cast.cr index 0de8d9eef7d4..1d3b0575e1b5 100644 --- a/spec/compiler/codegen/automatic_cast.cr +++ b/spec/compiler/codegen/automatic_cast.cr @@ -239,4 +239,25 @@ describe "Code gen: automatic cast" do foo(1, "a" || 1) )).to_i.should eq(20) end + + it "does multidispatch with automatic casting (3)" do + run(%( + abstract class Foo + end + + class Bar < Foo + def foo(x : UInt8) + 2 + end + end + + class Baz < Foo + def foo(x : UInt8) + 3 + end + end + + Bar.new.as(Foo).foo(1) + )).to_i.should eq(2) + end end diff --git a/spec/compiler/semantic/automatic_cast_spec.cr b/spec/compiler/semantic/automatic_cast_spec.cr index 375379992b21..feed10392f62 100644 --- a/spec/compiler/semantic/automatic_cast_spec.cr +++ b/spec/compiler/semantic/automatic_cast_spec.cr @@ -546,4 +546,33 @@ describe "Semantic: automatic cast" do fill() )) { types["AnotherColor"] } end + + it "doesn't do multidispatch if an overload matches exactly (#8217)" do + assert_type(%( + abstract class Foo + end + + class Bar < Foo + def foo(x : Int64) + x + end + + def foo(*xs : Int64) + xs + end + end + + class Baz < Foo + def foo(x : Int64) + x + end + + def foo(*xs : Int64) + xs + end + end + + Baz.new.as(Foo).foo(1) + )) { int64 } + end end diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr index cd576c8f57bd..1730958be238 100644 --- a/src/compiler/crystal/program.cr +++ b/src/compiler/crystal/program.cr @@ -581,8 +581,8 @@ module Crystal end end - def lookup_private_matches(filename, signature) - file_module?(filename).try &.lookup_matches(signature) + def lookup_private_matches(filename, signature, analyze_all = false) + file_module?(filename).try &.lookup_matches(signature, analyze_all: analyze_all) end def file_module?(filename) diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index 33808a8427ac..3c69bfb6ecac 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -247,7 +247,7 @@ class Crystal::Call matches = lookup_matches_checking_expansion(owner, signature, with_literals: with_literals) if matches.empty? && owner.class? && owner.abstract? - matches = owner.virtual_type.lookup_matches(signature) + matches = owner.virtual_type.lookup_matches(signature, analyze_all: with_literals) end if matches.empty? @@ -263,27 +263,27 @@ class Crystal::Call signature = CallSignature.new(def_name, arg_types, block, named_args_types) matches = check_tuple_indexer(owner, def_name, args, arg_types) - matches ||= lookup_matches_checking_expansion(owner, signature, search_in_parents) + matches ||= lookup_matches_checking_expansion(owner, signature, search_in_parents, with_literals: with_literals) # If we didn't find a match and this call doesn't have a receiver, # and we are not at the top level, let's try searching the top-level if matches.empty? && !obj && owner != program && search_in_toplevel - program_matches = lookup_matches_with_signature(program, signature, search_in_parents) + program_matches = lookup_matches_with_signature(program, signature, search_in_parents, with_literals) matches = program_matches unless program_matches.empty? end if matches.empty? && owner.class? && owner.abstract? && name != "super" - matches = owner.virtual_type.lookup_matches(signature) + matches = owner.virtual_type.lookup_matches(signature, analyze_all: with_literals) end if matches.empty? defined_method_missing = owner.check_method_missing(signature, self) if defined_method_missing - matches = owner.lookup_matches(signature) + matches = owner.lookup_matches(signature, analyze_all: with_literals) elsif with_scope = @with_scope defined_method_missing = with_scope.check_method_missing(signature, self) if defined_method_missing - matches = with_scope.lookup_matches(signature) + matches = with_scope.lookup_matches(signature, analyze_all: with_literals) @uses_with_scope = true end end @@ -321,9 +321,9 @@ class Crystal::Call matches = bubbling_exception do target = parent_visitor.typed_def.original_owner if search_in_parents - target.lookup_matches signature + target.lookup_matches(signature, analyze_all: with_literals) else - target.lookup_matches_without_parents signature + target.lookup_matches_without_parents(signature, analyze_all: with_literals) end end matches.each do |match| @@ -332,48 +332,36 @@ class Crystal::Call end matches else - bubbling_exception { lookup_matches_with_signature(owner, signature, search_in_parents) } + bubbling_exception { lookup_matches_with_signature(owner, signature, search_in_parents, with_literals) } end end - def lookup_matches_with_signature(owner : Program, signature, search_in_parents) + def lookup_matches_with_signature(owner : Program, signature, search_in_parents, with_literals) location = self.location if location && (filename = location.original_filename) - matches = owner.lookup_private_matches filename, signature + matches = owner.lookup_private_matches(filename, signature, analyze_all: with_literals) end if matches if matches.empty? - matches = owner.lookup_matches signature + matches = owner.lookup_matches(signature, analyze_all: with_literals) end else - matches = owner.lookup_matches signature + matches = owner.lookup_matches(signature, analyze_all: with_literals) end matches end - def lookup_matches_with_signature(owner, signature, search_in_parents) + def lookup_matches_with_signature(owner, signature, search_in_parents, with_literals) if search_in_parents - owner.lookup_matches signature + owner.lookup_matches(signature, analyze_all: with_literals) else - owner.lookup_matches_without_parents signature + owner.lookup_matches_without_parents(signature, analyze_all: with_literals) end end def instantiate(signature, matches, owner, self_type, with_literals) - if with_literals - # Now that we have all our matches, check if any of them matches exactly - # all types, assuming autocasted values will always match (because they - # matches and they were not ambiguous). If so, only keep matches up to - # that exact match. We need to do this here because with autocasting - # we consider all overloads to detect ambiguous usage. - stop_index = matches.index do |match| - signature.matches_exactly?(match, with_literals: true) - end - matches = matches[..stop_index] if stop_index - end - matches.each &.remove_literals if with_literals block = @block diff --git a/src/compiler/crystal/semantic/method_lookup.cr b/src/compiler/crystal/semantic/method_lookup.cr index 0b13572ec657..78569c9d1a9a 100644 --- a/src/compiler/crystal/semantic/method_lookup.cr +++ b/src/compiler/crystal/semantic/method_lookup.cr @@ -1,5 +1,55 @@ require "../types" +# Looking up matches involves two steps: +# +# 1. Lookup is done with autocasting disabled. +# +# In this scenario as soon as we find an exact match we don't look at other +# overloads because the exact match will prevent them from being considered. +# +# If no matches are found we try again but this time with autocasting enabled. +# In `semantic/call.cr` this is when `with_literals` is `true`, and this is when +# `analyze_all` will be `true` here. +# +# 2. Lookup is done with autocasting enabled. +# +# In this mode the types for NumberLiteral and SymbolLiteral are not the usual +# types but instead the special NumberLiteralType and SymbolLiteralType. +# +# In this mode we also need to stop as soon as we find an exact match +# (which just means when the first overload matches with autocasting, which +# is for example when passing 1 to an Int64 restriction) but we still need +# to analyze all possible methods in case there's an ambiguity. For example: +# +# ``` +# def foo(x : Int64) +# end +# +# def foo(x : Int8) +# end +# +# foo(1) +# ``` +# +# In the example above we can't just stop at the first overload because +# we need to analyze the second overload to find out that the call is ambiguous. +# +# However, consider this: +# +# ``` +# def foo(x : Int64) +# end +# +# def foo(x : *Int64) +# end +# +# foo(1) +# ``` +# +# In this case there's no ambiguity: 1 means `Int64`. However, the first overload +# is an exact match and there's no need to consider the second overload in the +# multidispatch. However, we do need to analyze it to check if there's an ambiguity. + module Crystal record NamedArgumentType, name : String, type : Type do def self.from_args(named_args : Array(NamedArgument)?, with_literals = false) @@ -16,8 +66,8 @@ module Crystal named_args : Array(NamedArgumentType)? class Type - def lookup_matches(signature, owner = self, path_lookup = self, matches_array = nil) - matches = lookup_matches_without_parents(signature, owner, path_lookup, matches_array) + def lookup_matches(signature, owner = self, path_lookup = self, matches_array = nil, analyze_all = false) + matches = lookup_matches_without_parents(signature, owner, path_lookup, matches_array, analyze_all: analyze_all) return matches if matches.cover_all? matches_array = matches.matches @@ -38,7 +88,7 @@ module Crystal # and can be known by invoking `lookup_new_in_ancestors?` if my_parents && !(is_new && !lookup_new_in_ancestors?) my_parents.each do |parent| - matches = parent.lookup_matches(signature, owner, parent, matches_array) + matches = parent.lookup_matches(signature, owner, parent, matches_array, analyze_all: analyze_all) if matches.cover_all? return matches else @@ -55,10 +105,12 @@ module Crystal Matches.new(matches_array, cover, owner, false) end - def lookup_matches_without_parents(signature, owner = self, path_lookup = self, matches_array = nil) + def lookup_matches_without_parents(signature, owner = self, path_lookup = self, matches_array = nil, analyze_all = false) if defs = self.defs.try &.[signature.name]? context = MatchContext.new(owner, path_lookup) + exact_match = nil + defs.each do |item| next if item.def.abstract? @@ -72,6 +124,8 @@ module Crystal match = signature.match(item, context) + next if exact_match + if match matches_array ||= [] of Match matches_array << match @@ -81,7 +135,8 @@ module Crystal # a function type with return T can be transpass a restriction of a function # with the same arguments but which returns Void. if signature.matches_exactly?(match) - return Matches.new(matches_array, true, owner) + exact_match = Matches.new(matches_array, true, owner) + break unless analyze_all end context = MatchContext.new(owner, path_lookup) @@ -90,13 +145,17 @@ module Crystal context.def_free_vars = nil end end + + if exact_match + return exact_match + end end Matches.new(matches_array, Cover.create(signature, matches_array), owner) end - def lookup_matches_with_modules(signature, owner = self, path_lookup = self, matches_array = nil) - matches = lookup_matches_without_parents(signature, owner, path_lookup, matches_array) + def lookup_matches_with_modules(signature, owner = self, path_lookup = self, matches_array = nil, analyze_all = false) + matches = lookup_matches_without_parents(signature, owner, path_lookup, matches_array, analyze_all: analyze_all) return matches unless matches.empty? is_new = owner.metaclass? && signature.name == "new" @@ -115,7 +174,7 @@ module Crystal my_parents.each do |parent| break unless parent.module? - matches = parent.lookup_matches_with_modules(signature, owner, parent, matches_array) + matches = parent.lookup_matches_with_modules(signature, owner, parent, matches_array, analyze_all: analyze_all) return matches unless matches.empty? end end @@ -304,10 +363,6 @@ module Crystal def matches_exactly?(match : Match, *, with_literals : Bool = false) arg_types_equal = self.arg_types.equals?(match.arg_types) do |x, y| - if with_literals && x.is_a?(LiteralType) - x = x.match || x.remove_literal - end - x.compatible_with?(y) end if (match_named_args = match.named_arg_types) && (signature_named_args = self.named_args) && @@ -315,12 +370,7 @@ module Crystal match_named_args = match_named_args.sort_by &.name signature_named_args = signature_named_args.sort_by &.name named_arg_types_equal = signature_named_args.equals?(match_named_args) do |x, y| - x_type = x.type - if with_literals && x_type.is_a?(LiteralType) - x_type = x_type.match || x_type.remove_literal - end - - x.name == y.name && x_type.compatible_with?(y.type) + x.name == y.name && x.type.compatible_with?(y.type) end else named_arg_types_equal = !match.named_arg_types && !self.named_args @@ -341,11 +391,11 @@ module Crystal type end - def lookup_matches(signature, owner = self, path_lookup = self) + def lookup_matches(signature, owner = self, path_lookup = self, analyze_all = false) is_new = virtual_metaclass? && signature.name == "new" base_type_lookup = virtual_lookup(base_type) - base_type_matches = base_type_lookup.lookup_matches(signature, self) + base_type_matches = base_type_lookup.lookup_matches(signature, self, analyze_all: analyze_all) # If there are no subclasses no need to look further if leaf? @@ -369,7 +419,7 @@ module Crystal subtype_virtual_lookup = virtual_lookup(subtype.virtual_type) # Check matches but without parents: only included modules - subtype_matches = subtype_lookup.lookup_matches_with_modules(signature, subtype_virtual_lookup, subtype_virtual_lookup) + subtype_matches = subtype_lookup.lookup_matches_with_modules(signature, subtype_virtual_lookup, subtype_virtual_lookup, analyze_all: analyze_all) # For Foo+.class#new we need to check that this subtype doesn't define # an incompatible initialize: if so, we return empty matches, because @@ -390,7 +440,7 @@ module Crystal base_type_matches.each do |base_type_match| if base_type_match.def.macro_def? # We need to copy each submatch if it's a macro def - full_subtype_matches = subtype_lookup.lookup_matches(signature, subtype_virtual_lookup, subtype_virtual_lookup) + full_subtype_matches = subtype_lookup.lookup_matches(signature, subtype_virtual_lookup, subtype_virtual_lookup, analyze_all: analyze_all) full_subtype_matches.each do |full_subtype_match| cloned_def = full_subtype_match.def.clone cloned_def.macro_owner = full_subtype_match.def.macro_owner diff --git a/src/compiler/crystal/semantic/restrictions.cr b/src/compiler/crystal/semantic/restrictions.cr index 1e862e60c1b7..d6049159304f 100644 --- a/src/compiler/crystal/semantic/restrictions.cr +++ b/src/compiler/crystal/semantic/restrictions.cr @@ -1240,6 +1240,10 @@ module Crystal type end end + + def compatible_with?(type) + literal.type == type || literal.can_be_autocast_to?(type) + end end class SymbolLiteralType @@ -1264,6 +1268,17 @@ module Crystal type end end + + def compatible_with?(type) + case type + when SymbolType + true + when EnumType + !!(type.find_member(literal.value)) + else + false + end + end end end From e7db73fb74fdac6193898554df15271761d1670f Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 6 Jul 2020 10:35:09 -0300 Subject: [PATCH 172/263] Reject annotations on ivars defined in base class (#9502) --- spec/compiler/semantic/annotation_spec.cr | 19 ++++++++++++++++++- .../semantic/type_declaration_processor.cr | 5 +++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/spec/compiler/semantic/annotation_spec.cr b/spec/compiler/semantic/annotation_spec.cr index f9596fa36db4..c6aa7131bccb 100644 --- a/spec/compiler/semantic/annotation_spec.cr +++ b/spec/compiler/semantic/annotation_spec.cr @@ -730,7 +730,7 @@ describe "Semantic: annotation" do end def foo - {% if @type.instance_vars.first.annotations(Foo) %} + {% if @type.instance_vars.first.annotation(Foo) %} 1 {% else %} 'a' @@ -918,6 +918,23 @@ describe "Semantic: annotation" do end end + it "errors when annotate instance variable in subclass" do + assert_error %( + annotation Foo + end + + class Base + @x : Nil + end + + class Child < Base + @[Foo] + @x : Nil + end + ), + "can't annotate @x in Child because it was first defined in Base" + end + it "errors if wanting to add type inside annotation (1) (#8614)" do assert_error %( annotation Ann diff --git a/src/compiler/crystal/semantic/type_declaration_processor.cr b/src/compiler/crystal/semantic/type_declaration_processor.cr index f1a5461bbf05..52175e8652cd 100644 --- a/src/compiler/crystal/semantic/type_declaration_processor.cr +++ b/src/compiler/crystal/semantic/type_declaration_processor.cr @@ -266,6 +266,11 @@ struct Crystal::TypeDeclarationProcessor unless supervar.type.same?(type_decl.type) raise TypeException.new("instance variable '#{name}' of #{supervar.owner}, with #{owner} < #{supervar.owner}, is already declared as #{supervar.type} (trying to re-declare as #{type_decl.type})", type_decl.location) end + + # Reject annotations to existing instance var + type_decl.annotations.try &.each do |_, ann| + ann.raise "can't annotate #{name} in #{owner} because it was first defined in #{supervar.owner}" + end else declare_meta_type_var(owner.instance_vars, owner, name, type_decl, instance_var: true, check_nilable: !owner.module?) remove_error owner, name From 09299124f13773bd56f0e5968f533620d59869be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 6 Jul 2020 15:36:55 +0200 Subject: [PATCH 173/263] Make specs pending instead of failing in no multicast environments (#9566) --- spec/std/socket/udp_socket_spec.cr | 22 ++++++++++- src/spec/dsl.cr | 4 ++ src/spec/example.cr | 2 + src/spec/methods.cr | 60 +++++++++++++++++++++--------- 4 files changed, 68 insertions(+), 20 deletions(-) diff --git a/spec/std/socket/udp_socket_spec.cr b/spec/std/socket/udp_socket_spec.cr index 087acca25aa5..b99e4d286b81 100644 --- a/spec/std/socket/udp_socket_spec.cr +++ b/spec/std/socket/udp_socket_spec.cr @@ -78,14 +78,32 @@ describe UDPSocket do expect_raises(Socket::Error, "Unsupported IP address family: INET. For use with IPv6 only") do udp.multicast_interface 0 end - udp.multicast_interface Socket::IPAddress.new(unspecified_address, 0) + + begin + udp.multicast_interface Socket::IPAddress.new(unspecified_address, 0) + rescue e : Socket::Error + if e.os_error == Errno::ENOPROTOOPT + pending!("Multicast device selection not available on this host") + else + raise e + end + end Socket::IPAddress.new("224.0.0.254", port) when Socket::Family::INET6 expect_raises(Socket::Error, "Unsupported IP address family: INET6. For use with IPv4 only") do udp.multicast_interface(Socket::IPAddress.new(unspecified_address, 0)) end - udp.multicast_interface(0) + + begin + udp.multicast_interface(0) + rescue e : Socket::Error + if e.os_error == Errno::ENOPROTOOPT + pending!("Multicast device selection not available on this host") + else + raise e + end + end Socket::IPAddress.new("ff02::102", port) else diff --git a/src/spec/dsl.cr b/src/spec/dsl.cr index c6524e4edf93..bc199149a01c 100644 --- a/src/spec/dsl.cr +++ b/src/spec/dsl.cr @@ -53,6 +53,10 @@ module Spec class AssertionFailed < SpecError end + # :nodoc: + class ExamplePending < SpecError + end + # :nodoc: class NestingSpecError < SpecError end diff --git a/src/spec/example.cr b/src/spec/example.cr index efd639604679..01f46a6fae62 100644 --- a/src/spec/example.cr +++ b/src/spec/example.cr @@ -47,6 +47,8 @@ module Spec rescue ex : Spec::AssertionFailed @parent.report(:fail, description, file, line, Time.monotonic - start, ex) Spec.abort! if Spec.fail_fast? + rescue ex : Spec::ExamplePending + @parent.report(:pending, description, file, line, Time.monotonic - start) rescue ex @parent.report(:error, description, file, line, Time.monotonic - start, ex) Spec.abort! if Spec.fail_fast? diff --git a/src/spec/methods.cr b/src/spec/methods.cr index 019c90f34c41..ebde51f7023a 100644 --- a/src/spec/methods.cr +++ b/src/spec/methods.cr @@ -6,6 +6,8 @@ module Spec::Methods # # Example: # ``` + # require "spec" + # # describe "Int32" do # describe "+" do # it "adds" { (1 + 1).should eq 2 } @@ -35,6 +37,8 @@ module Spec::Methods # # Example: # ``` + # require "spec" + # # it "adds" { (1 + 1).should eq 2 } # ``` # @@ -52,6 +56,8 @@ module Spec::Methods # # Example: # ``` + # require "spec" + # # pending "check cat" { cat.alive? } # ``` # @@ -76,6 +82,24 @@ module Spec::Methods raise Spec::AssertionFailed.new(msg, file, line) end + # Marks the current example pending + # + # In case an example needs to be pending on some condition that requires executing it, + # this allows to mark it as such rather than letting it fail or never run. + # + # ``` + # require "spec" + # + # it "test git" do + # cmd = Process.find_executable("git") + # pending!("git is not available") unless cmd + # cmd.ends_with?("git").should be_true + # end + # ``` + def pending!(msg = "Cannot run example", file = __FILE__, line = __LINE__) + raise Spec::ExamplePending.new(msg, file, line) + end + # Executes the given block before each spec in the current context runs. # # A context is defined by `describe` or `context` blocks, or outside of them @@ -87,16 +111,16 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # before_each do # puts "runs before sample_b" # end # - # it "sample_b" {} + # it "sample_b" { } # end # ``` def before_each(&block) @@ -117,16 +141,16 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # after_each do # puts "runs after sample_b" # end # - # it "sample_b" {} + # it "sample_b" { } # end # ``` def after_each(&block) @@ -147,16 +171,16 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # before_all do # puts "runs at start of nested_context" # end # - # it "sample_b" {} + # it "sample_b" { } # end # ``` def before_all(&block) @@ -177,16 +201,16 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # after_all do # puts "runs at end of nested_context" # end # - # it "sample_b" {} + # it "sample_b" { } # end # ``` def after_all(&block) @@ -213,9 +237,9 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # around_each do |example| @@ -224,7 +248,7 @@ module Spec::Methods # puts "runs after sample_b" # end # - # it "sample_b" {} + # it "sample_b" { } # end # ``` def around_each(&block : Example::Procsy ->) @@ -250,7 +274,7 @@ module Spec::Methods # order of definition. # # ``` - # require "spec + # require "spec" # # describe "main_context" do # around_each do |example| @@ -259,7 +283,7 @@ module Spec::Methods # puts "runs at end of main_context" # end # - # it "sample_a" {} + # it "sample_a" { } # # describe "nested_context" do # around_each do |example| @@ -268,7 +292,7 @@ module Spec::Methods # puts "runs at end of nested_context" # end # - # it "sample_b" {} + # it "sample_b" { } # end # end # ``` From 13c79348fb1fdf3ab185229b77ae51bf684a6737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonne=20Ha=C3=9F?= Date: Mon, 6 Jul 2020 15:49:07 +0200 Subject: [PATCH 174/263] AArch64 CI (#9508) --- .github/workflows/aarch64.yml | 101 +++++++++++++++++++++++++++++++ spec/std/http/web_socket_spec.cr | 2 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/aarch64.yml diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml new file mode 100644 index 000000000000..8d5d13d6220b --- /dev/null +++ b/.github/workflows/aarch64.yml @@ -0,0 +1,101 @@ +name: AArch64 CI + +on: [push, pull_request] + +jobs: + musl-build: + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Build Crystal + uses: docker://jhass/crystal:0.35.0-alpine-build + with: + args: make crystal + - name: Upload Crystal executable + uses: actions/upload-artifact@v1 + with: + name: crystal-aarch64-musl + path: .build/crystal + musl-test-stdlib: + needs: musl-build + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Download Crystal executable + uses: actions/download-artifact@v1 + with: + name: crystal-aarch64-musl + path: .build/ + - name: Mark downloaded compiler as executable + run: chmod +x .build/crystal + - name: Run stdlib specs + uses: docker://jhass/crystal:0.35.0-alpine-build + with: + args: make std_spec + musl-test-compiler: + needs: musl-build + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Download Crystal executable + uses: actions/download-artifact@v1 + with: + name: crystal-aarch64-musl + path: .build/ + - name: Mark downloaded compiler as executable + run: chmod +x .build/crystal + - name: Run compiler specs + uses: docker://jhass/crystal:0.35.0-alpine-build + with: + args: make compiler_spec + gnu-build: + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Build Crystal + uses: docker://jhass/crystal:0.35.0-build + with: + args: make crystal + - name: Upload Crystal executable + uses: actions/upload-artifact@v1 + with: + name: crystal-aarch64-gnu + path: .build/crystal + gnu-test-stdlib: + needs: gnu-build + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Download Crystal executable + uses: actions/download-artifact@v1 + with: + name: crystal-aarch64-gnu + path: .build/ + - name: Mark downloaded compiler as executable + run: chmod +x .build/crystal + - name: Run stdlib specs + uses: docker://jhass/crystal:0.35.0-build + with: + args: make std_spec + gnu-test-compiler: + needs: gnu-build + runs-on: [linux, ARM64] + steps: + - name: Download Crystal source + uses: actions/checkout@v2 + - name: Download Crystal executable + uses: actions/download-artifact@v1 + with: + name: crystal-aarch64-gnu + path: .build/ + - name: Mark downloaded compiler as executable + run: chmod +x .build/crystal + - name: Run compiler specs + uses: docker://jhass/crystal:0.35.0-build + with: + args: make compiler_spec diff --git a/spec/std/http/web_socket_spec.cr b/spec/std/http/web_socket_spec.cr index ad3b35c2e167..a0d9cf0d0e04 100644 --- a/spec/std/http/web_socket_spec.cr +++ b/spec/std/http/web_socket_spec.cr @@ -393,6 +393,7 @@ describe HTTP::WebSocket do ws.on_message do |str| ws.send("pong #{str}") + ws.close end ws.on_close do @@ -415,7 +416,6 @@ describe HTTP::WebSocket do random = Random::Secure.hex ws2.on_message do |str| str.should eq("pong #{random}") - ws2.close end ws2.send(random) From 5b12be99a5e84c44719d21d8c66b5df387729e56 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 6 Jul 2020 10:51:46 -0300 Subject: [PATCH 175/263] Logging dispatcher (#9432) --- .../http/server/handlers/log_handler_spec.cr | 5 +- spec/std/log/dispatch_spec.cr | 57 ++++++++++++ spec/std/log/io_backend_spec.cr | 9 +- spec/support/retry.cr | 17 ++++ src/log.cr | 1 + src/log/backend.cr | 15 +++ src/log/broadcast_backend.cr | 7 +- src/log/builder.cr | 5 + src/log/dispatch.cr | 91 +++++++++++++++++++ src/log/io_backend.cr | 21 +++-- src/log/log.cr | 2 +- src/log/main.cr | 2 + src/log/memory_backend.cr | 4 + 13 files changed, 224 insertions(+), 12 deletions(-) create mode 100644 spec/std/log/dispatch_spec.cr create mode 100644 spec/support/retry.cr create mode 100644 src/log/dispatch.cr diff --git a/spec/std/http/server/handlers/log_handler_spec.cr b/spec/std/http/server/handlers/log_handler_spec.cr index 4f78b8dcd100..08d26e0a9300 100644 --- a/spec/std/http/server/handlers/log_handler_spec.cr +++ b/spec/std/http/server/handlers/log_handler_spec.cr @@ -2,6 +2,7 @@ require "spec" require "log/spec" require "http/server/handler" require "../../../../support/io" +require "../../../../support/retry" describe HTTP::LogHandler do it "logs" do @@ -49,7 +50,9 @@ describe HTTP::LogHandler do handler.next = ->(ctx : HTTP::Server::Context) {} handler.call(context) - io.to_s.should match(%r(- - GET / HTTP/1.1 - 200 \(\d+(\.\d+)?[mµn]s\)$)) + retry do + io.to_s.should match(%r(- - GET / HTTP/1.1 - 200 \(\d+(\.\d+)?[mµn]s\)$)) + end end it "log failed request" do diff --git a/spec/std/log/dispatch_spec.cr b/spec/std/log/dispatch_spec.cr new file mode 100644 index 000000000000..1bf5c2260fd2 --- /dev/null +++ b/spec/std/log/dispatch_spec.cr @@ -0,0 +1,57 @@ +require "spec" +require "log" +require "../../support/retry" + +class Log + describe Dispatcher do + it "create dispatcher from enum" do + Dispatcher.for(:direct).should eq(DirectDispatcher) + Dispatcher.for(:async).should be_a(AsyncDispatcher) + Dispatcher.for(:sync).should be_a(SyncDispatcher) + end + end + + describe DirectDispatcher do + it "dispatches entry" do + backend = Log::MemoryBackend.new + backend.dispatcher = DirectDispatcher + backend.dispatch entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + backend.entries.size.should eq(1) + end + end + + describe SyncDispatcher do + it "dispatches entry" do + backend = Log::MemoryBackend.new + backend.dispatcher = SyncDispatcher.new + backend.dispatch entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + backend.entries.size.should eq(1) + end + end + + describe AsyncDispatcher do + it "dispatches entry" do + backend = Log::MemoryBackend.new + backend.dispatcher = AsyncDispatcher.new + backend.dispatch entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + retry { backend.entries.size.should eq(1) } + end + + it "wait for entries to flush before closing" do + backend = Log::MemoryBackend.new + backend.dispatcher = AsyncDispatcher.new + backend.dispatch entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + backend.close + backend.entries.size.should eq(1) + end + + it "can be closed twice" do + backend = Log::MemoryBackend.new + backend.dispatcher = AsyncDispatcher.new + backend.dispatch entry = Entry.new("source", :info, "message", Log::Metadata.empty, nil) + backend.close + backend.close + backend.entries.size.should eq(1) + end + end +end diff --git a/spec/std/log/io_backend_spec.cr b/spec/std/log/io_backend_spec.cr index 8c03f8e1bc17..80f66c08c202 100644 --- a/spec/std/log/io_backend_spec.cr +++ b/spec/std/log/io_backend_spec.cr @@ -1,4 +1,4 @@ -require "spec" +require "../spec_helper" require "log" private def s(value : Log::Severity) @@ -14,6 +14,13 @@ private def io_logger(*, stdout : IO, config = nil, source : String = "") end describe Log::IOBackend do + pending_win32 "creates with defaults" do + backend = Log::IOBackend.new + backend.io.should eq(STDOUT) + backend.formatter.should eq(Log::ShortFormat) + backend.dispatcher.should be_a(Log::AsyncDispatcher) + end + it "logs messages" do IO.pipe do |r, w| logger = io_logger(stdout: w) diff --git a/spec/support/retry.cr b/spec/support/retry.cr new file mode 100644 index 000000000000..1c368dee0134 --- /dev/null +++ b/spec/support/retry.cr @@ -0,0 +1,17 @@ +def retry(n = 5) + exception = nil + n.times do |i| + yield + rescue ex + exception = ex + if i == 0 + Fiber.yield + else + sleep 0.01 * (2**i) + end + else + return + end + + raise exception.not_nil! +end diff --git a/src/log.cr b/src/log.cr index 23b32ec51558..bebc82cc6dc1 100644 --- a/src/log.cr +++ b/src/log.cr @@ -155,5 +155,6 @@ require "./log/setup" require "./log/log" require "./log/memory_backend" require "./log/io_backend" +require "./log/dispatch" Log.setup diff --git a/src/log/backend.cr b/src/log/backend.cr index cdecb1749429..8be7c1974fac 100644 --- a/src/log/backend.cr +++ b/src/log/backend.cr @@ -2,10 +2,25 @@ require "crystal/datum" # Base class for all backends. abstract class Log::Backend + property dispatcher : Dispatcher + + def initialize(dispatch_mode : DispatchMode = :async) + @dispatcher = Dispatcher.for(dispatch_mode) + end + + def initialize(@dispatcher : Dispatcher) + end + # Writes the *entry* to this backend. abstract def write(entry : Entry) # Closes underlying resources used by this backend. def close + @dispatcher.close + end + + # :nodoc: + def dispatch(entry : Entry) + @dispatcher.dispatch entry, self end end diff --git a/src/log/broadcast_backend.cr b/src/log/broadcast_backend.cr index d5a81c07f58e..eae027187e05 100644 --- a/src/log/broadcast_backend.cr +++ b/src/log/broadcast_backend.cr @@ -11,17 +11,22 @@ class Log::BroadcastBackend < Log::Backend @backends = Hash(Log::Backend, Severity).new + def initialize + super(:direct) + end + def append(backend : Log::Backend, level : Severity) @backends[backend] = level end def write(entry : Entry) @backends.each do |backend, level| - backend.write(entry) if (@level || level) <= entry.severity + backend.dispatch(entry) if (@level || level) <= entry.severity end end def close + @backends.each_key &.close end # :nodoc: diff --git a/src/log/builder.cr b/src/log/builder.cr index 99c1e7c4de68..f4b1dedda33c 100644 --- a/src/log/builder.cr +++ b/src/log/builder.cr @@ -143,6 +143,11 @@ class Log::Builder end end + # :nodoc: + def close + @bindings.each &.backend.close + end + # :nodoc: def self.matches(source : String, pattern : String) : Bool return true if source == pattern diff --git a/src/log/dispatch.cr b/src/log/dispatch.cr new file mode 100644 index 000000000000..1a186f32662f --- /dev/null +++ b/src/log/dispatch.cr @@ -0,0 +1,91 @@ +class Log + # Base interface implemented by log entry dispatchers + # + # Dispatchers are in charge of sending log entries according + # to different strategies. + module Dispatcher + alias Spec = Dispatcher | DispatchMode + + # Dispatch a log entry to the specified backend + abstract def dispatch(entry : Entry, backend : Backend) + + # Close the dispatcher, releasing resources + def close + end + + # :nodoc: + def self.for(mode : DispatchMode) + case mode + in .sync? + SyncDispatcher.new + in .async? + AsyncDispatcher.new + in .direct? + DirectDispatcher + end + end + end + + enum DispatchMode + Sync + Async + Direct + end + + # Stateless dispatcher that deliver log entries immediately + module DirectDispatcher + extend Dispatcher + + def self.dispatch(entry : Entry, backend : Backend) + backend.write(entry) + end + end + + # Deliver log entries asynchronously through a channels + class AsyncDispatcher + include Dispatcher + + def initialize(buffer_size = 2048) + @channel = Channel({Entry, Backend}).new(buffer_size) + @done = Channel(Nil).new + spawn write_logs + end + + def dispatch(entry : Entry, backend : Backend) + @channel.send({entry, backend}) + end + + private def write_logs + while msg = @channel.receive? + entry, backend = msg + backend.write(entry) + end + + @done.send nil + end + + def close + # TODO: this might fail if being closed from different threads + unless @channel.closed? + @channel.close + @done.receive + end + end + end + + # Deliver log entries directly. It uses a mutex to guarantee + # one entry is delivered at a time. + class SyncDispatcher + include Dispatcher + + def initialize + @mutex = Mutex.new(:unchecked) + end + + def dispatch(entry : Entry, backend : Backend) + @mutex.synchronize do + backend.write(entry) + end + end + end +end diff --git a/src/log/io_backend.cr b/src/log/io_backend.cr index e9265c503f75..b449444ba439 100644 --- a/src/log/io_backend.cr +++ b/src/log/io_backend.cr @@ -3,16 +3,21 @@ class Log::IOBackend < Log::Backend property io : IO property formatter : Formatter - def initialize(@io = STDOUT, *, @formatter : Formatter = ShortFormat) - @mutex = Mutex.new(:unchecked) - end + {% if flag?(:win32) %} + # TODO: this constructor must go away once channels are fixed in Windows + def initialize(@io = STDOUT, *, @formatter : Formatter = ShortFormat, dispatcher : Dispatcher::Spec = DispatchMode::Sync) + super(dispatcher) + end + {% else %} + def initialize(@io = STDOUT, *, @formatter : Formatter = ShortFormat, dispatcher : Dispatcher::Spec = DispatchMode::Async) + super(dispatcher) + end + {% end %} def write(entry : Entry) - @mutex.synchronize do - format(entry) - io.puts - io.flush - end + format(entry) + io.puts + io.flush end # Emits the *entry* to the given *io*. diff --git a/src/log/log.cr b/src/log/log.cr index 831d31a1b4dc..698081e8a0fa 100644 --- a/src/log/log.cr +++ b/src/log/log.cr @@ -59,7 +59,7 @@ class Log dsl.emit(result.to_s) end - backend.write entry + backend.dispatch entry end {% end %} end diff --git a/src/log/main.cr b/src/log/main.cr index 28f956e9ab0f..5a695cca2582 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -46,6 +46,8 @@ class Log @@builder = Builder.new + at_exit { @@builder.close } + # Returns the default `Log::Builder` used for `Log.for` calls. def self.builder @@builder diff --git a/src/log/memory_backend.cr b/src/log/memory_backend.cr index 2e044ca1a8af..483f9aae0e51 100644 --- a/src/log/memory_backend.cr +++ b/src/log/memory_backend.cr @@ -3,6 +3,10 @@ class Log::MemoryBackend < Log::Backend getter entries = Array(Log::Entry).new + def initialize + super(:direct) + end + def write(entry : Log::Entry) @entries << entry end From afc214d2173114da6e2e8ac09ed5897c416bfd24 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 8 Jul 2020 14:23:18 -0300 Subject: [PATCH 176/263] Disable aarch64 jobs on forks (#9582) --- .github/workflows/aarch64.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/aarch64.yml b/.github/workflows/aarch64.yml index 8d5d13d6220b..491afbcbd10a 100644 --- a/.github/workflows/aarch64.yml +++ b/.github/workflows/aarch64.yml @@ -5,6 +5,7 @@ on: [push, pull_request] jobs: musl-build: runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -20,6 +21,7 @@ jobs: musl-test-stdlib: needs: musl-build runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -37,6 +39,7 @@ jobs: musl-test-compiler: needs: musl-build runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -53,6 +56,7 @@ jobs: args: make compiler_spec gnu-build: runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -68,6 +72,7 @@ jobs: gnu-test-stdlib: needs: gnu-build runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 @@ -85,6 +90,7 @@ jobs: gnu-test-compiler: needs: gnu-build runs-on: [linux, ARM64] + if: github.repository == 'crystal-lang/crystal' steps: - name: Download Crystal source uses: actions/checkout@v2 From 3df23bd28a49f38f632533baf6910f844760b2dd Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Thu, 9 Jul 2020 12:12:46 +0200 Subject: [PATCH 177/263] Hide Channel internal implementation details (#9564) * Hide Channel internal implementation methods * Add private/nodoc to more methods and types * Remove deprecated Channel#select --- spec/std/channel_spec.cr | 5 ----- src/channel.cr | 39 +++++++++++++++++---------------------- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/spec/std/channel_spec.cr b/spec/std/channel_spec.cr index 097c2945a579..dff88b48eddf 100644 --- a/spec/std/channel_spec.cr +++ b/spec/std/channel_spec.cr @@ -618,11 +618,6 @@ describe "unbuffered" do Channel.select(ch1.receive_select_action, ch2.receive_select_action).should eq({0, 123}) end - it "works with select else" do - ch1 = Channel(Int32).new - Channel.select({ch1.receive_select_action}, true).should eq({1, Channel::NotReady.new}) - end - it "can send and receive nil" do ch = Channel(Nil).new sender = Fiber.new { ch.send nil } diff --git a/src/channel.cr b/src/channel.cr index 5f4c77e77fa7..500960b8c9f9 100644 --- a/src/channel.cr +++ b/src/channel.cr @@ -26,9 +26,12 @@ class Channel(T) @lock = Crystal::SpinLock.new @queue : Deque(T)? + # :nodoc: record NotReady + # :nodoc: record UseDefault + # :nodoc: module SelectAction(S) abstract def execute : DeliveryState abstract def wait(context : SelectContext(S)) @@ -72,7 +75,7 @@ class Channel(T) end end - enum SelectState + private enum SelectState None = 0 Active = 1 Done = 2 @@ -117,7 +120,7 @@ class Channel(T) end end - enum DeliveryState + private enum DeliveryState None Delivered Closed @@ -273,7 +276,7 @@ class Channel(T) # end # channel.receive # => 1 # ``` - def receive + def receive : T receive_impl { raise ClosedError.new } end @@ -281,11 +284,11 @@ class Channel(T) # If there is a value waiting, it is returned immediately. Otherwise, this method blocks until a value is sent to the channel. # # Returns `nil` if the channel is closed or closes while waiting for receive. - def receive? + def receive? : T? receive_impl { return nil } end - def receive_impl + private def receive_impl receiver = Receiver(T).new @lock.lock @@ -318,7 +321,7 @@ class Channel(T) end end - def receive_internal + protected def receive_internal if (queue = @queue) && !queue.empty? deque_value = queue.shift if sender_ptr = dequeue_sender @@ -395,34 +398,29 @@ class Channel(T) nil end + # :nodoc: def self.select(*ops : SelectAction) self.select ops end + # :nodoc: def self.select(ops : Indexable(SelectAction)) i, m = select_impl(ops, false) raise "BUG: blocking select returned not ready status" if m.is_a?(NotReady) return i, m end - @[Deprecated("Use Channel.non_blocking_select")] - def self.select(ops : Indexable(SelectAction), has_else) - # The overload of Channel.select(Indexable(SelectAction), Bool) - # is used by LiteralExpander with the second argument as `true`. - # This overload is kept as a transition, but 0.32 will emit calls to - # Channel.select or Channel.non_blocking_select directly - non_blocking_select(ops) - end - + # :nodoc: def self.non_blocking_select(*ops : SelectAction) self.non_blocking_select ops end + # :nodoc: def self.non_blocking_select(ops : Indexable(SelectAction)) select_impl(ops, true) end - def self.select_impl(ops : Indexable(SelectAction), non_blocking) + private def self.select_impl(ops : Indexable(SelectAction), non_blocking) # Sort the operations by the channel they contain # This is to avoid deadlocks between concurrent `select` calls ops_locks = ops @@ -492,8 +490,7 @@ class Channel(T) LooseReceiveAction.new(self) end - # :nodoc: - class StrictReceiveAction(T) + private class StrictReceiveAction(T) include SelectAction(T) property receiver : Receiver(T) @@ -557,8 +554,7 @@ class Channel(T) end end - # :nodoc: - class LooseReceiveAction(T) + private class LooseReceiveAction(T) include SelectAction(T) property receiver : Receiver(T) @@ -622,8 +618,7 @@ class Channel(T) end end - # :nodoc: - class SendAction(T) + private class SendAction(T) include SelectAction(Nil) property sender : Sender(T) From ffed6d2ae2f9c973d228584a32f857c63b62065c Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 10 Jul 2020 07:11:15 -0500 Subject: [PATCH 178/263] Array rotate optimization (#8516) * Array rotate optimization Add special case for array rotate to optimize for small arrays. * fix typo in docs * rotate tmp array is now populated with filler * fix if statement in array#rotate! to use else * use uninitialized StaticArray for array#rotate! optimization * run crystal formatter * use case statment to optimize array#rotate! * fix formatting issue * update array#rotate docs * make array#rotate docs easierto read * add comment for ignore block * add large array for array#rotate * use if statment over case statment * add tests for array#rotate! * change array#rotate! to check for size - 1 * add comment for shifting in array#rotate! * update docs * remove comments --- spec/std/array_spec.cr | 24 ++++++++++++++++++++ src/array.cr | 50 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/spec/std/array_spec.cr b/spec/std/array_spec.cr index af5752f3c1fe..39a4d5cc65f8 100644 --- a/spec/std/array_spec.cr +++ b/spec/std/array_spec.cr @@ -1866,6 +1866,30 @@ describe "Array" do it { a = [1, 2, 3]; a.rotate(3001).should eq([2, 3, 1]); a.should eq([1, 2, 3]) } it { a = [1, 2, 3]; a.rotate(-1).should eq([3, 1, 2]); a.should eq([1, 2, 3]) } it { a = [1, 2, 3]; a.rotate(-3001).should eq([3, 1, 2]); a.should eq([1, 2, 3]) } + + it do + a = Array(Int32).new(50) { |i| i } + a.rotate!(5) + a.should eq([5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 0, 1, 2, 3, 4]) + end + + it do + a = Array(Int32).new(50) { |i| i } + a.rotate!(-5) + a.should eq([45, 46, 47, 48, 49, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44]) + end + + it do + a = Array(Int32).new(50) { |i| i } + a.rotate!(20) + a.should eq([20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]) + end + + it do + a = Array(Int32).new(50) { |i| i } + a.rotate!(-20) + a.should eq([30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]) + end end describe "permutations" do diff --git a/src/array.cr b/src/array.cr index 31042709f9cd..997004d422b1 100644 --- a/src/array.cr +++ b/src/array.cr @@ -47,8 +47,7 @@ class Array(T) include Indexable(T) include Comparable(Array) - # Size of an Array that we consider small to do linear scans - # or other optimizations instead of using a lookup Hash. + # Size of an Array that we consider small to do linear scans or other optimizations. private SMALL_ARRAY_SIZE = 16 # Returns the number of elements in the array. @@ -1601,11 +1600,45 @@ class Array(T) self end + # Returns `self` with all the elements shifted `n` times. + # + # ``` + # a1 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # a2 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # a3 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # + # a1.rotate! + # a2.rotate!(1) + # a3.rotate!(3) + # + # a1 # => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + # a2 # => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + # a3 # => [3, 4, 5, 6, 7, 8, 9, 0, 1, 2] + # ``` def rotate!(n = 1) return self if size == 0 n %= size - return self if n == 0 - if n <= size // 2 + + if n == 0 + elsif n == 1 + tmp = self[0] + @buffer.move_from(@buffer + n, size - n) + self[-1] = tmp + elsif n == (size - 1) + tmp = self[-1] + (@buffer + size - n).move_from(@buffer, n) + self[0] = tmp + elsif n <= SMALL_ARRAY_SIZE + tmp_buffer = uninitialized StaticArray(T, SMALL_ARRAY_SIZE) + tmp_buffer.to_unsafe.copy_from(@buffer, n) + @buffer.move_from(@buffer + n, size - n) + (@buffer + size - n).copy_from(tmp_buffer.to_unsafe, n) + elsif size - n <= SMALL_ARRAY_SIZE + tmp_buffer = uninitialized StaticArray(T, SMALL_ARRAY_SIZE) + tmp_buffer.to_unsafe.copy_from(@buffer + n, size - n) + (@buffer + size - n).move_from(@buffer, n) + @buffer.copy_from(tmp_buffer.to_unsafe, size - n) + elsif n <= size // 2 tmp = self[0..n] @buffer.move_from(@buffer + n, size - n) (@buffer + size - n).copy_from(tmp.to_unsafe, n) @@ -1617,6 +1650,15 @@ class Array(T) self end + # Returns an array with all the elements shifted `n` times. + # + # ``` + # a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # a.rotate # => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + # a.rotate(1) # => [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] + # a.rotate(3) # => [3, 4, 5, 6, 7, 8, 9, 0, 1, 2] + # a # => [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # ``` def rotate(n = 1) return self if size == 0 n %= size From 40263183ceeb38ce92898567dc816775c78364eb Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 10 Jul 2020 09:12:30 -0300 Subject: [PATCH 179/263] Use content instead of value when inspect XML::Attribute (#9592) --- spec/std/xml/xml_spec.cr | 6 ++++++ src/xml/node.cr | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/std/xml/xml_spec.cr b/spec/std/xml/xml_spec.cr index 7eed384c0062..f1c3e684f8e4 100644 --- a/spec/std/xml/xml_spec.cr +++ b/spec/std/xml/xml_spec.cr @@ -399,6 +399,12 @@ describe XML do res.should be_nil end + it "shows content when inspecting attribute" do + doc = XML.parse(%{}) + attr = doc.root.not_nil!.attributes.first + attr.inspect.should contain(%(content="baz")) + end + it ".build" do XML.build do |builder| builder.element "foo" { } diff --git a/src/xml/node.cr b/src/xml/node.cr index aff5b97f7999..bed80378d35a 100644 --- a/src/xml/node.cr +++ b/src/xml/node.cr @@ -204,7 +204,7 @@ struct XML::Node end if attribute? - io << " value=" + io << " content=" content.inspect(io) else attributes = self.attributes From 67b24ecc2f7832da1676d9eec0f05347b6847b57 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 15 Jul 2020 11:40:14 -0300 Subject: [PATCH 180/263] Use 'brew bundle' to install macOS dependencies in CI (#9609) --- .circleci/config.yml | 2 +- .github/workflows/macos.yml | 2 +- Brewfile | 6 ++++++ bin/ci | 7 +------ 4 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 Brewfile diff --git a/.circleci/config.yml b/.circleci/config.yml index d4220e769a67..59c9ab0cc2a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,7 @@ jobs: - brew-cache-v1 - checkout - run: bin/ci prepare_system - - run: echo 'export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/openssl/lib/pkgconfig"' >> $BASH_ENV + - run: echo 'export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/openssl@1.1/lib/pkgconfig"' >> $BASH_ENV - run: echo 'export CURRENT_TAG="$CIRCLE_TAG"' >> $BASH_ENV - run: bin/ci prepare_build - run: bin/ci build diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 9d29a7d243a3..9303f001bbfc 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -5,7 +5,7 @@ on: [push, pull_request] env: TRAVIS_OS_NAME: osx LLVM_CONFIG: /usr/local/opt/llvm/bin/llvm-config - PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig SPEC_SPLIT_DOTS: 160 jobs: diff --git a/Brewfile b/Brewfile new file mode 100644 index 000000000000..22ffcf245edb --- /dev/null +++ b/Brewfile @@ -0,0 +1,6 @@ +brew "gmp" +brew "libevent" +brew "pcre" +brew "pkg-config" +brew "openssl@1.1" +brew "llvm@10", link: true, conflicts_with: ["python@2"] diff --git a/bin/ci b/bin/ci index 5bff8a0a6797..098ce4efa34b 100755 --- a/bin/ci +++ b/bin/ci @@ -77,8 +77,6 @@ on_github() { prepare_system() { on_linux 'echo '"'"'{"ipv6":true, "fixed-cidr-v6":"2001:db8:1::/64"}'"'"' | sudo tee /etc/docker/daemon.json' on_linux sudo service docker restart - - on_osx brew update } build() { @@ -119,11 +117,8 @@ prepare_build() { on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-darwin-x86_64.tar.gz -o ~/crystal.tar.gz on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-0.35.1-1 crystal;popd' + on_osx brew bundle --no-lock - on_osx 'brew unlink python@2 || true' - on_osx brew install z3 llvm@10 gmp libevent pcre pkg-config - on_osx brew reinstall openssl - on_osx brew link --force llvm@10 # Note: brew link --force might show: # Warning: Refusing to link macOS-provided software: llvm # From 38e1cd264f2c91ea77689aaa74bba5108ba4fea0 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 17 Jul 2020 09:59:27 -0300 Subject: [PATCH 181/263] String: don't materialize Regex match[0] if not needed (#9615) This avoid creating an intermediate string --- src/string.cr | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/string.cr b/src/string.cr index 2e2314590d66..27c2891d5d19 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3611,7 +3611,7 @@ class String while match = separator.match_at_byte_index(self, match_offset) index = match.byte_begin(0) - match_bytesize = match[0].bytesize + match_bytesize = match.byte_end(0) - index next_offset = index + match_bytesize if next_offset == slice_offset @@ -3856,9 +3856,10 @@ class String String.new(bytesize) do |buffer| buffer += bytesize scan(/\X/) do |match| - grapheme = match[0] - buffer -= grapheme.bytesize - buffer.copy_from(grapheme.to_unsafe, grapheme.bytesize) + match_begin = match.byte_begin(0) + match_bytesize = match.byte_end(0) - match_begin + buffer -= match_bytesize + buffer.copy_from(to_unsafe + match_begin, match_bytesize) end {@bytesize, @length} end @@ -4175,7 +4176,7 @@ class String index = match.byte_begin(0) $~ = match yield match - match_bytesize = match[0].bytesize + match_bytesize = match.byte_end(0) - index match_bytesize += 1 if match_bytesize == 0 byte_offset = index + match_bytesize end From 7580060f3a3dce6c0061f275570f930a7a1c6da1 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 17 Jul 2020 10:45:09 -0300 Subject: [PATCH 182/263] HTTP::Params: `#[]=` replaces all values. (#9605) * HTTP::Params: `#[]=` replaces all values. * Fix HTTP::Request specs --- spec/std/http/params_spec.cr | 10 ++++++++-- spec/std/http/request_spec.cr | 6 +++--- src/http/params.cr | 24 ++++++++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/spec/std/http/params_spec.cr b/spec/std/http/params_spec.cr index 631dac4160c2..26107252a982 100644 --- a/spec/std/http/params_spec.cr +++ b/spec/std/http/params_spec.cr @@ -116,10 +116,10 @@ module HTTP end describe "#[]=(name, value)" do - it "sets first value for provided param name" do + it "sets value for provided param name" do params = Params.parse("foo=bar&foo=baz&baz=qux") params["foo"] = "notfoo" - params.fetch_all("foo").should eq(["notfoo", "baz"]) + params.fetch_all("foo").should eq(["notfoo"]) end it "adds new name => value pair if there is no such param" do @@ -127,6 +127,12 @@ module HTTP params["non_existent_param"] = "test" params.fetch_all("non_existent_param").should eq(["test"]) end + + it "sets value for provided param name (array)" do + params = Params.parse("foo=bar&foo=baz&baz=qux") + params["non_existent_param"] = ["test", "something"] + params.fetch_all("non_existent_param").should eq(["test", "something"]) + end end describe "#fetch(name, default)" do diff --git a/spec/std/http/request_spec.cr b/spec/std/http/request_spec.cr index e06a4c193dc4..536cb203ff92 100644 --- a/spec/std/http/request_spec.cr +++ b/spec/std/http/request_spec.cr @@ -408,7 +408,7 @@ module HTTP params = request.query_params params["foo"] = "not-bar" - request.query.should eq("foo=not-bar&foo=baz&baz=qux") + request.query.should eq("foo=not-bar&baz=qux") end it "updates @resource when modified" do @@ -416,7 +416,7 @@ module HTTP params = request.query_params params["foo"] = "not-bar" - request.resource.should eq("/api/v3/some/resource?foo=not-bar&foo=baz&baz=qux") + request.resource.should eq("/api/v3/some/resource?foo=not-bar&baz=qux") end it "updates serialized form when modified" do @@ -427,7 +427,7 @@ module HTTP io = IO::Memory.new request.to_io(io) - io.to_s.should eq("GET /api/v3/some/resource?foo=not-bar&foo=baz&baz=qux HTTP/1.1\r\n\r\n") + io.to_s.should eq("GET /api/v3/some/resource?foo=not-bar&baz=qux HTTP/1.1\r\n\r\n") end it "is affected when #query is modified" do diff --git a/src/http/params.cr b/src/http/params.cr index 3e0359872ebe..66527ce3e904 100644 --- a/src/http/params.cr +++ b/src/http/params.cr @@ -189,14 +189,26 @@ module HTTP # ``` delegate empty?, to: raw_params - # Sets first *value* for specified param *name*. + # Sets the *name* key to *value*. # # ``` - # params["item"] = "pencil" - # ``` - def []=(name, value) - raw_params[name] ||= [""] - raw_params[name][0] = value + # require "http/params" + # + # params = HTTP::Params{"a" => ["b", "c"]} + # params["a"] = "d" + # params["a"] # => "d" + # params.fetch_all("a") # => ["d"] + # + # params["a"] = ["e", "f"] + # params["a"] # => "e" + # params.fetch_all("a") # => ["e", "f"] + # ``` + def []=(name, value : String | Array(String)) + raw_params[name] = + case value + in String then [value] + in Array(String) then value + end end # Returns all values for specified param *name*. From f2b23307a81c94e87ef07fc7402917d0f9b1598d Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 17 Jul 2020 15:12:09 -0300 Subject: [PATCH 183/263] Compiler: try literal type in autocast first (#9610) * Compiler: try literal type in autocast first * Add a test for autocast against union type that doesn't match --- spec/compiler/codegen/automatic_cast.cr | 10 +++++++++ spec/compiler/semantic/automatic_cast_spec.cr | 21 +++++++++++++++++++ src/compiler/crystal/semantic/restrictions.cr | 8 +++---- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/spec/compiler/codegen/automatic_cast.cr b/spec/compiler/codegen/automatic_cast.cr index 1d3b0575e1b5..16dc08af8f08 100644 --- a/spec/compiler/codegen/automatic_cast.cr +++ b/spec/compiler/codegen/automatic_cast.cr @@ -260,4 +260,14 @@ describe "Code gen: automatic cast" do Bar.new.as(Foo).foo(1) )).to_i.should eq(2) end + + it "doesn't autocast number on union (#8655)" do + run(%( + def foo(x : UInt8 | Int32, y : Float64) + x + end + + foo(255, 60) + )).to_i.should eq(255) + end end diff --git a/spec/compiler/semantic/automatic_cast_spec.cr b/spec/compiler/semantic/automatic_cast_spec.cr index feed10392f62..bdb0a02f3d8e 100644 --- a/spec/compiler/semantic/automatic_cast_spec.cr +++ b/spec/compiler/semantic/automatic_cast_spec.cr @@ -575,4 +575,25 @@ describe "Semantic: automatic cast" do Baz.new.as(Foo).foo(1) )) { int64 } end + + it "doesn't autocast number on union (#8655)" do + assert_type(%( + def foo(x : UInt8 | Int32, y : Float64) + x + end + + foo(255, 60) + )) { int32 } + end + + it "says ambiguous call on union (#8655)" do + assert_error %( + def foo(x : UInt64 | Int64, y : Float64) + x + end + + foo(255, 60) + ), + "ambiguous call, implicit cast of 255 matches all of UInt64, Int64" + end end diff --git a/src/compiler/crystal/semantic/restrictions.cr b/src/compiler/crystal/semantic/restrictions.cr index d6049159304f..2a8b616702ea 100644 --- a/src/compiler/crystal/semantic/restrictions.cr +++ b/src/compiler/crystal/semantic/restrictions.cr @@ -1232,8 +1232,8 @@ module Crystal literal.type.restrict(other, context) end else - type = super(other, context) || - literal.type.restrict(other, context) + type = literal.type.restrict(other, context) || + super(other, context) if type == self type = @match || literal.type end @@ -1260,8 +1260,8 @@ module Crystal literal.type.restrict(other, context) end else - type = super(other, context) || - literal.type.restrict(other, context) + type = literal.type.restrict(other, context) || + super(other, context) if type == self type = @match || literal.type end From b9b1e4cf7b806609ed1c52f4a6fc5158a731ed92 Mon Sep 17 00:00:00 2001 From: Stephanie Wilde-Hobbs Date: Sun, 19 Jul 2020 14:46:47 +0100 Subject: [PATCH 184/263] Fix HTTP::FormData.parse documentation (#9612) --- src/http/formdata.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/formdata.cr b/src/http/formdata.cr index 380b5d87052c..ca26eb01c715 100644 --- a/src/http/formdata.cr +++ b/src/http/formdata.cr @@ -77,7 +77,7 @@ require "mime/multipart" # end # ``` module HTTP::FormData - # Parses a multipart/form-data message, yielding a `FormData::Parser`. + # Parses a multipart/form-data message, yielding a `FormData::Part`. # # ``` # require "http" @@ -97,7 +97,7 @@ module HTTP::FormData end end - # Parses a multipart/form-data message, yielding a `FormData::Parser`. + # Parses a multipart/form-data message, yielding a `FormData::Part`. # # ``` # require "http" From 2f0d1d0c6a3606d0ec1461278788f4ff28aa3b31 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Sun, 19 Jul 2020 18:46:15 -0300 Subject: [PATCH 185/263] Add various `String#delete_at` methods (#9398) * Add String#delete_at(index : Int) * Add String#delete_at(index, count) * Mark index in doc with italics * Fix typo * Add String#delete_at(Range) * Reuse `byte_delete_at` * Extract find_start_end_and_index --- spec/std/string_spec.cr | 65 ++++++++++++++ src/string.cr | 183 +++++++++++++++++++++++++++++++++++----- 2 files changed, 229 insertions(+), 19 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 121ab3c9d96c..408f6ec2f12f 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -2724,4 +2724,69 @@ describe "String" do String.interpolation("a", 123, "b", 456, "cde").should eq("a123b456cde") end end + + describe "delete_at" do + describe "char" do + it { "abcde".delete_at(0).should eq("bcde") } + it { "abcde".delete_at(1).should eq("acde") } + it { "abcde".delete_at(2).should eq("abde") } + it { "abcde".delete_at(4).should eq("abcd") } + it { "abcde".delete_at(-2).should eq("abce") } + it { expect_raises(IndexError) { "abcde".delete_at(5) } } + it { expect_raises(IndexError) { "abcde".delete_at(-6) } } + + it { "二ノ国".delete_at(0).should eq("ノ国") } + it { "二ノ国".delete_at(1).should eq("二国") } + it { "二ノ国".delete_at(2).should eq("二ノ") } + it { "二ノ国".delete_at(-2).should eq("二国") } + it { expect_raises(IndexError) { "二ノ国".delete_at(3) } } + it { expect_raises(IndexError) { "二ノ国".delete_at(-4) } } + end + + describe "start, count" do + it { "abcdefg".delete_at(0, 2).should eq("cdefg") } + it { "abcdefg".delete_at(1, 2).should eq("adefg") } + it { "abcdefg".delete_at(3, 10).should eq("abc") } + it { "abcdefg".delete_at(-3, 2).should eq("abcdg") } + it { "abcdefg".delete_at(7, 10).should eq("abcdefg") } + it { expect_raises(IndexError) { "abcdefg".delete_at(8, 1) } } + it { expect_raises(IndexError) { "abcdefg".delete_at(-8, 1) } } + + it "raises on negative count" do + expect_raises(ArgumentError, "Negative count: -1") { + "abcdefg".delete_at(1, -1) + } + end + + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(4, 6).should eq("セキロ：ダイ トゥワイス") } + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(0, 4).should eq("シャドウズ ダイ トゥワイス") } + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(3, 20).should eq("セキロ") } + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(-14, 6).should eq("セキロ：ダイ トゥワイス") } + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(18, 3).should eq("セキロ：シャドウズ ダイ トゥワイス") } + it { expect_raises(IndexError) { "セキロ：シャドウズ ダイ トゥワイス".delete_at(19, 1) } } + it { expect_raises(IndexError) { "セキロ：シャドウズ ダイ トゥワイス".delete_at(-19, 1) } } + + it "raises on negative count" do + expect_raises(ArgumentError, "Negative count: -1") { + "セキロ：シャドウズ ダイ トゥワイス".delete_at(1, -1) + } + end + end + + describe "range" do + it { "abcdefg".delete_at(0..1).should eq("cdefg") } + it { "abcdefg".delete_at(0...2).should eq("cdefg") } + it { "abcdefg".delete_at(1..3).should eq("aefg") } + it { "abcdefg".delete_at(3..10).should eq("abc") } + it { "abcdefg".delete_at(-3..-2).should eq("abcdg") } + it { "abcdefg".delete_at(3..).should eq("abc") } + it { "abcdefg".delete_at(..-3).should eq("fg") } + it { expect_raises(IndexError) { "abcdefg".delete_at(8..1) } } + it { expect_raises(IndexError) { "abcdefg".delete_at(-8..1) } } + + it { "セキロ：シャドウズ ダイ トゥワイス".delete_at(4...10).should eq("セキロ：ダイ トゥワイス") } + it { expect_raises(IndexError) { "セキロ：シャドウズ ダイ トゥワイス".delete_at(19..1) } } + it { expect_raises(IndexError) { "セキロ：シャドウズ ダイ トゥワイス".delete_at(-19..1) } } + end + end end diff --git a/src/string.cr b/src/string.cr index 27c2891d5d19..e9b70b1e3f4f 100644 --- a/src/string.cr +++ b/src/string.cr @@ -788,24 +788,7 @@ class String start += size if start < 0 - start_pos = nil - end_pos = nil - - reader = Char::Reader.new(self) - i = 0 - - reader.each do |char| - if i == start - start_pos = reader.pos - elsif count >= 0 && i == start + count - end_pos = reader.pos - i += 1 - break - end - i += 1 - end - - end_pos ||= reader.pos + start_pos, end_pos, end_index = find_start_end_and_index(start, count) if start_pos return "" if count == 0 @@ -817,7 +800,7 @@ class String buffer.copy_from(to_unsafe + start_pos, count) {count, 0} end - elsif start == i + elsif start == end_index "" end end @@ -900,6 +883,168 @@ class String end end + # Returns a new string that results from deleting characters + # at the given range. + # + # ``` + # "abcdef".delete_at(1..3) # => "aef" + # ``` + # + # Negative indices can be used to start counting from the end of the string: + # + # ``` + # "abcdef".delete_at(-3..-2) # => "abcf" + # ``` + # + # Raises `IndexError` if any index is outside the bounds of this string. + def delete_at(range : Range) + delete_at(*Indexable.range_to_index_and_count(range, size)) + end + + # Returns a new string that results from deleting the character + # at the given *index*. + # + # ``` + # "abcde".delete_at(0) # => "bcde" + # "abcde".delete_at(2) # => "abde" + # "abcde".delete_at(4) # => "abcd" + # ``` + # + # A negative *index* counts from the end of the string: + # + # ``` + # "abcde".delete_at(-2) # => "abce" + # ``` + # + # If *index* is outside the bounds of the string, `IndexError` is raised. + def delete_at(index : Int) : String + index += size if index < 0 + + byte_index = char_index_to_byte_index(index) + if byte_index && byte_index < @bytesize + char_bytesize = char_bytesize_at(byte_index) + + new_bytesize = self.bytesize - char_bytesize + String.new(new_bytesize) do |buffer| + # Copy left part + buffer.copy_from(to_unsafe, byte_index) + + # Copy right part + (buffer + byte_index).copy_from( + to_unsafe + byte_index + char_bytesize, + self.bytesize - byte_index - char_bytesize, + ) + + {new_bytesize, size - 1} + end + else + raise IndexError.new + end + end + + # Returns a new string that results from deleting *count* characters + # starting at *index*. + # + # ``` + # "abcdefg".delete_at(1, 3) # => "aefg" + # ``` + # + # Deleting more characters than those in the string is valid, and just + # results in deleting up to the last character: + # + # ``` + # "abcdefg".delete_at(3, 10) # => "abc" + # ``` + # + # A negative *index* counts from the end of the string: + # + # ``` + # "abcdefg".delete_at(-3, 2) # => "abcdg" + # ``` + # + # If *count* is negative, `ArgumentError` is raised. + # + # If *index* is outside the bounds of the string, `ArgumentError` + # is raised. + # + # However, *index* can be the position that is exactly the end of the string: + # + # ``` + # "abcd".delete_at(4, 3) # => "abcd" + # ``` + def delete_at(index : Int, count : Int) : String + raise ArgumentError.new "Negative count: #{count}" if count < 0 + + index += size if index < 0 + unless 0 <= index <= size + raise IndexError.new + end + + count = Math.min(count, size - index) + + case count + when 0 + return self + when size + return "" + else + if ascii_only? + byte_delete_at(index, count, count) + else + unicode_delete_at(index, count) + end + end + end + + private def byte_delete_at(start, count, byte_count) + new_bytesize = bytesize - byte_count + String.new(new_bytesize) do |buffer| + # Copy left part + buffer.copy_from(to_unsafe, start) + + # Copy right part + (buffer + start).copy_from( + to_unsafe + start + byte_count, + bytesize - start - byte_count, + ) + + {new_bytesize, size - count} + end + end + + private def unicode_delete_at(start, count) + start_pos, end_pos, _ = find_start_end_and_index(start, count) + + # That start is in bounds was already verified in `delete_at` + start_pos = start_pos.not_nil! + + byte_count = end_pos - start_pos.not_nil! + byte_delete_at(start_pos, count, byte_count) + end + + private def find_start_end_and_index(start, count) + start_pos = nil + end_pos = nil + + reader = Char::Reader.new(self) + i = 0 + + reader.each do |char| + if i == start + start_pos = reader.pos + elsif i == start + count + end_pos = reader.pos + i += 1 + break + end + i += 1 + end + + end_pos ||= reader.pos + + {start_pos, end_pos, i} + end + # Returns a new string built from *count* bytes starting at *start* byte. # # *start* can can be negative to start counting From 64ff87fea0d9da1a499870de9921ce9ad6a7f11b Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 21 Jul 2020 20:10:41 +0900 Subject: [PATCH 186/263] OptionParser: don't call handler if value is given to none value handler (#9603) Fixed #9553 It is regression fix. Before 0.34.0, it worked in this way. --- spec/std/option_parser_spec.cr | 16 ++++++++++++++++ src/option_parser.cr | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/spec/std/option_parser_spec.cr b/spec/std/option_parser_spec.cr index 531552a6ee23..527c81d840bb 100644 --- a/spec/std/option_parser_spec.cr +++ b/spec/std/option_parser_spec.cr @@ -230,6 +230,22 @@ describe "OptionParser" do end end + it "raises on invalid option if value is given to none value handler (short flag, #9553) " do + expect_raises OptionParser::InvalidOption, "Invalid option: -foo" do + OptionParser.parse(["-foo"]) do |opts| + opts.on("-f", "some flag") { } + end + end + end + + it "raises on invalid option if value is given to none value handler (long flag, #9553)" do + expect_raises OptionParser::InvalidOption, "Invalid option: --foo=bar" do + OptionParser.parse(["--foo=bar"]) do |opts| + opts.on("-foo", "some flag") { } + end + end + end + it "calls the handler for invalid options" do called = false OptionParser.parse(["-f", "-j"]) do |opts| diff --git a/src/option_parser.cr b/src/option_parser.cr index 95dd75178ed2..2df3f00bd616 100644 --- a/src/option_parser.cr +++ b/src/option_parser.cr @@ -366,7 +366,9 @@ class OptionParser value = nil end - if handler = @handlers[flag]? + # Fetch handler of the flag. + # If value is given even though handler does not take value, it is invalid, then it is skipped. + if (handler = @handlers[flag]?) && !(handler.value_type.none? && value) handled_args << arg_index # Pull in the next argument if we don't already have it and an argument From b55cbda19b4d10fb9d0fbb9287f57dffcaa78d97 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 21 Jul 2020 13:21:36 -0300 Subject: [PATCH 187/263] Compiler: consider `select` as an openning keyword in macros (#9624) --- spec/compiler/lexer/lexer_macro_spec.cr | 2 +- src/compiler/crystal/syntax/lexer.cr | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/spec/compiler/lexer/lexer_macro_spec.cr b/spec/compiler/lexer/lexer_macro_spec.cr index a8eb7bf4255b..c58e1a177eea 100644 --- a/spec/compiler/lexer/lexer_macro_spec.cr +++ b/spec/compiler/lexer/lexer_macro_spec.cr @@ -39,7 +39,7 @@ describe "Lexer macro" do token.type.should eq(:MACRO_END) end - ["begin", "do", "if", "unless", "class", "struct", "module", "def", "while", "until", "case", "macro", "fun", "lib", "union", "annotation"].each do |keyword| + ["begin", "do", "if", "unless", "class", "struct", "module", "def", "while", "until", "case", "macro", "fun", "lib", "union", "annotation", "select"].each do |keyword| it "lexes macro with nested #{keyword}" do lexer = Lexer.new(%(hello\n #{keyword} {{world}} end end)) diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index c7692ec49f51..a867370815b6 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -2564,7 +2564,14 @@ module Crystal (char == 'o' && next_char == 'd' && next_char == 'u' && next_char == 'l' && next_char == 'e' && peek_not_ident_part_or_end_next_char && :module) ) when 's' - next_char == 't' && next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't' && !ident_part_or_end?(peek_next_char) && next_char && :struct + case next_char + when 'e' + next_char == 'l' && next_char == 'e' && next_char == 'c' && next_char == 't' && !ident_part_or_end?(peek_next_char) && next_char && :select + when 't' + next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't' && !ident_part_or_end?(peek_next_char) && next_char && :struct + else + false + end when 'u' next_char == 'n' && (char = next_char) && ( (char == 'i' && next_char == 'o' && next_char == 'n' && peek_not_ident_part_or_end_next_char && :union) || From eb46097440bf20d22eff6c38fc82732927bb193e Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 21 Jul 2020 14:49:45 -0300 Subject: [PATCH 188/263] Replace "whitelisting" with "filtering" (#9627) --- src/http/server/handler.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/server/handler.cr b/src/http/server/handler.cr index 5bc57bd3cc14..3b1f97113b80 100644 --- a/src/http/server/handler.cr +++ b/src/http/server/handler.cr @@ -2,7 +2,7 @@ require "./context" # A handler is a class which includes `HTTP::Handler` and implements the `call` method. # You can use a handler to intercept any incoming request and can modify the response. -# These can be used for request throttling, ip-based whitelisting, adding custom headers e.g. +# These can be used for request throttling, ip-based filtering, adding custom headers e.g. # # ### A custom handler # From 884bac61d658f08c4a65c78479088c7d34fa6b26 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 22 Jul 2020 10:50:21 -0300 Subject: [PATCH 189/263] Check abstract def implementations with splats, default values and keyword arguments (#9585) --- spec/compiler/semantic/abstract_def_spec.cr | 203 ++++++++++++++++++ .../crystal/semantic/abstract_def_checker.cr | 126 ++++++++--- 2 files changed, 304 insertions(+), 25 deletions(-) diff --git a/spec/compiler/semantic/abstract_def_spec.cr b/spec/compiler/semantic/abstract_def_spec.cr index 59c3bd41ed84..068c0f6e4f38 100644 --- a/spec/compiler/semantic/abstract_def_spec.cr +++ b/spec/compiler/semantic/abstract_def_spec.cr @@ -674,4 +674,207 @@ describe "Semantic: abstract def" do end )) end + + it "doesn't error if implementation have default value" do + semantic %( + abstract class Foo + abstract def foo(x) + end + + class Bar < Foo + def foo(x = 1) + end + end + ) + end + + it "errors if implementation doesn't have default value" do + assert_error %( + abstract class Foo + abstract def foo(x = 1) + end + + class Bar < Foo + def foo(x) + end + end + ), + "abstract `def Foo#foo(x = 1)` must be implemented by Bar" + end + + it "errors if implementation doesn't have the same default value" do + assert_error %( + abstract class Foo + abstract def foo(x = 1) + end + + class Bar < Foo + def foo(x = 2) + end + end + ), + "abstract `def Foo#foo(x = 1)` must be implemented by Bar" + end + + it "errors if implementation doesn't have keyword arguments" do + assert_error %( + abstract class Foo + abstract def foo(*, x) + end + + class Bar < Foo + def foo(a = 0, b = 0) + end + end + ), + "abstract `def Foo#foo(*, x)` must be implemented by Bar" + end + + it "errors if implementation doesn't have a keyword argument" do + assert_error %( + abstract class Foo + abstract def foo(*, x) + end + + class Bar < Foo + def foo(*, y) + end + end + ), + "abstract `def Foo#foo(*, x)` must be implemented by Bar" + end + + it "doesn't error if implementation matches keyword argument" do + semantic %( + abstract class Foo + abstract def foo(*, x) + end + + class Bar < Foo + def foo(*, x) + end + end + ) + end + + it "errors if implementation doesn't match keyword argument type" do + assert_error %( + abstract class Foo + abstract def foo(*, x : Int32) + end + + class Bar < Foo + def foo(*, x : String) + end + end + ), + "abstract `def Foo#foo(*, x : Int32)` must be implemented by Bar" + end + + it "doesn't error if implementation have keyword arguments in different order" do + semantic %( + abstract class Foo + abstract def foo(*, x : Int32, y : String) + end + + class Bar < Foo + def foo(*, y : String, x : Int32) + end + end + ) + end + + it "errors if implementation has more keyword arguments" do + assert_error %( + abstract class Foo + abstract def foo(*, x) + end + + class Bar < Foo + def foo(*, x, y) + end + end + ), + "abstract `def Foo#foo(*, x)` must be implemented by Bar" + end + + it "doesn't error if implementation has more keyword arguments with default values" do + semantic %( + abstract class Foo + abstract def foo(*, x) + end + + class Bar < Foo + def foo(*, x, y = 1) + end + end + ) + end + + it "errors if implementation doesn't have a splat" do + assert_error %( + abstract class Foo + abstract def foo(*args) + end + + class Bar < Foo + def foo(x = 1) + end + end + ), + "abstract `def Foo#foo(*args)` must be implemented by Bar" + end + + it "errors if implementation doesn't match splat type" do + assert_error %( + abstract class Foo + abstract def foo(*args : Int32) + end + + class Bar < Foo + def foo(*args : String) + end + end + ), + "abstract `def Foo#foo(*args : Int32)` must be implemented by Bar" + end + + it "doesn't error with splat" do + semantic %( + abstract class Foo + abstract def foo(*args) + end + + class Bar < Foo + def foo(*args) + end + end + ) + end + + it "doesn't error with splat and args with default value" do + semantic %( + abstract class Foo + abstract def foo(*args) + end + + class Bar < Foo + def foo(a = 1, *args) + end + end + ) + end + + it "allows arguments to be collapsed into splat" do + semantic %( + abstract class Foo + abstract def foo(a : Int32, b : String) + end + + class Bar < Foo + def foo(*args : Int32 | String) + end + end + ) + end end diff --git a/src/compiler/crystal/semantic/abstract_def_checker.cr b/src/compiler/crystal/semantic/abstract_def_checker.cr index 418cf388c609..f7ead4d50411 100644 --- a/src/compiler/crystal/semantic/abstract_def_checker.cr +++ b/src/compiler/crystal/semantic/abstract_def_checker.cr @@ -22,8 +22,6 @@ # def foo(x); end # end # ``` -# -# TODO: the check currently ignores methods that involve splats. class Crystal::AbstractDefChecker def initialize(@program : Program) @all_checked = Set(Type).new @@ -48,9 +46,6 @@ class Crystal::AbstractDefChecker defs_with_metadata.each do |def_with_metadata| a_def = def_with_metadata.def if a_def.abstract? - # TODO: for now we skip methods with splats and default arguments - next if a_def.splat_index || a_def.args.any? &.default_value - check_implemented_in_subtypes(type, a_def) end end @@ -120,10 +115,8 @@ class Crystal::AbstractDefChecker return false unless m1.name == m2.name return false unless m1.yields == m2.yields - # TODO: for now we consider that if there's a splat, the method is implemented - return true if m1.splat_index - - return false if m1.args.size < m2.args.size + m1_args, m1_kargs = def_arg_ranges(m1) + m2_args, m2_kargs = def_arg_ranges(m2) # If the base type is a generic type, we find the generic instantiation of # t1 for it. This will have a mapping of type vars to types, for example @@ -137,25 +130,108 @@ class Crystal::AbstractDefChecker m2 = replace_method_arg_paths_with_type_vars(t2, m2, generic_base) end - m2.args.zip(m1.args) do |a2, a1| - r1 = a1.restriction - r2 = a2.restriction - if r2 && r1 && r1 != r2 - # Check if a1.restriction is contravariant with a2.restriction - begin - rt1 = t1.lookup_type(r1) - rt2 = t2.lookup_type(r2) - return false unless rt2.covariant?(rt1) - rescue Crystal::TypeException - # Ignore if we can't find a type (assume the method is implemented) - next - end + # First check positional arguments + # The following algorithm walk through the arguments in the abstract + # method and the implementation at the same time, until a splat argument is found + # or the end of the positional argument list is reached in both lists. + # The table below resumes the allowed cases (OK) and rejected (x) for each combination + # of the argument in the implementation (a1) and the abstract def (a2). + # `an = Dn` represents an argument with a default value. `-` represents that + # no more arguments are available to compare. + # Allowed cases are then verified that they have compatible default value + # and type restrictions. + # + # | a2 | a2 = D2 | *a2 | - | + # a1 | OK | x | x | x | + # a1 = D1 | OK | OK | OK | OK | + # *a1 | OK | x | OK | OK | + # - | x | x | x | OK | + i1 = i2 = 0 + loop do + a1 = i1 <= m1_args ? m1.args[i1] : nil + a2 = i2 <= m2_args ? m2.args[i2] : nil + + case + when !a1 + # No more arguments in the implementation + return false unless !a2 + when i1 == m1.splat_index + # The argument in the implementation is a splat + return false if a2 && a2.default_value + when !a1.default_value + # The argument in the implementation doesn't have a default value + return false if !a2 || a2.default_value || i2 == m2.splat_index + end + + if a1 && a2 + return false unless check_arg(t1, a1, t2, a2) + end + + # Move next, unless we're on the splat already or at the end of the arguments + done = true + unless i1 == m1.splat_index || a1 == nil + i1 += 1 + done = false + end + unless i2 == m2.splat_index || a2 == nil + i2 += 1 + done = false + end + break if done + end + + # Index keyword arguments + kargs = + m1_kargs.to_h do |i| + a1 = m1.args[i] + {a1.name, a1} + end + + # Check keyword arguments + m2_kargs.each do |i| + a2 = m2.args[i] + a1 = kargs.delete(a2.name) + return false unless a1 + return false unless check_arg(t1, a1, t2, a2) + end + + # Remaining keyword arguments must have a default value + kargs.each_value do |a1| + return false unless a1.default_value + end + + true + end + + private def def_arg_ranges(method : Def) + if splat = method.splat_index + if method.args[splat].name.size == 0 + {splat - 1, (splat + 1...method.args.size)} + else + {splat, (splat + 1...method.args.size)} end + else + {method.args.size - 1, (0...0)} + end + end + + def check_arg(t1 : Type, a1 : Arg, t2 : Type, a2 : Arg) + if a2.default_value + return false unless a1.default_value == a2.default_value end - # If the method has more arguments, but default values for them, it implements it - if m1.args.size > m2.args.size - return false unless m1.args[m2.args.size].default_value + r1 = a1.restriction + r2 = a2.restriction + if r2 && r1 && r1 != r2 + # Check if a1.restriction is contravariant with a2.restriction + begin + rt1 = t1.lookup_type(r1) + rt2 = t2.lookup_type(r2) + return false unless rt2.covariant?(rt1) + rescue Crystal::TypeException + # Ignore if we can't find a type (assume the method is implemented) + return true + end end true From 52bca246d63ad76ecd7938554419eadff8b1f466 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Fri, 24 Jul 2020 22:57:51 +0900 Subject: [PATCH 190/263] Fix ASTNode#to_s for parenthesized expression in block (#9629) Fixed #9607 --- spec/compiler/parser/to_s_spec.cr | 2 ++ src/compiler/crystal/syntax/to_s.cr | 1 + 2 files changed, 3 insertions(+) diff --git a/spec/compiler/parser/to_s_spec.cr b/spec/compiler/parser/to_s_spec.cr index 8a46d5d69438..5621376e3314 100644 --- a/spec/compiler/parser/to_s_spec.cr +++ b/spec/compiler/parser/to_s_spec.cr @@ -170,4 +170,6 @@ describe "ASTNode#to_s" do expect_to_s "offsetof(Foo, @bar)" expect_to_s "def foo(**options, &block)\nend" expect_to_s "macro foo\n 123\nend" + expect_to_s "if true\n( 1)\nend" + expect_to_s "begin\n( 1)\nrescue\nend" end diff --git a/src/compiler/crystal/syntax/to_s.cr b/src/compiler/crystal/syntax/to_s.cr index 5ba97ee3b6af..4624393303e7 100644 --- a/src/compiler/crystal/syntax/to_s.cr +++ b/src/compiler/crystal/syntax/to_s.cr @@ -1562,6 +1562,7 @@ module Crystal with_indent do node.accept self end + newline if node.keyword == :"(" end def accept_with_indent(node : Nop) From 7d13f6875d4a71783149953e7bbe5aadf3dc7f94 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 24 Jul 2020 11:26:02 -0300 Subject: [PATCH 191/263] Allow overriding CRYSTAL_PATH in the wrapper script (#9632) * Allow overriding CRYSTAL_PATH in the wrapper script * Add warning if CRYSTAL_ROOT/src is not included in CRYSTAL_PATH * Perform brew update in circleci --- .circleci/config.yml | 1 + bin/crystal | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59c9ab0cc2a3..8be9ceeb152d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,6 +92,7 @@ jobs: keys: - brew-cache-v1 - checkout + - run: brew update - run: bin/ci prepare_system - run: echo 'export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/openssl@1.1/lib/pkgconfig"' >> $BASH_ENV - run: echo 'export CURRENT_TAG="$CIRCLE_TAG"' >> $BASH_ENV diff --git a/bin/crystal b/bin/crystal index 7d2e3ebf1048..ec70846e497d 100755 --- a/bin/crystal +++ b/bin/crystal @@ -138,14 +138,18 @@ SCRIPT_ROOT="$(dirname "$SCRIPT_PATH")" CRYSTAL_ROOT="$(dirname "$SCRIPT_ROOT")" CRYSTAL_DIR="$CRYSTAL_ROOT/.build" -export CRYSTAL_PATH=lib:$CRYSTAL_ROOT/src +export CRYSTAL_PATH="${CRYSTAL_PATH:-lib:$CRYSTAL_ROOT/src}" +if [ -n "${CRYSTAL_PATH##*$CRYSTAL_ROOT/src*}" ]; then + __warning_msg "CRYSTAL_PATH env variable does not contains $CRYSTAL_ROOT/src" +fi + export CRYSTAL_HAS_WRAPPER=true export CRYSTAL="${CRYSTAL:-"crystal"}" if [ -z "$CRYSTAL_CONFIG_LIBRARY_PATH" ]; then export CRYSTAL_CONFIG_LIBRARY_PATH="$( - export PATH="$(remove_path_item "$(remove_path_item "$PATH" "$SCRIPT_ROOT")" "bin")" + export PATH="$(remove_path_item "$(remove_path_item "$PATH" "$SCRIPT_ROOT")" "bin")" crystal env CRYSTAL_LIBRARY_PATH || echo "" )" fi From 904177902b84854daa363e55f5d312ff01d89327 Mon Sep 17 00:00:00 2001 From: Kubo Takehiro Date: Tue, 28 Jul 2020 16:33:31 +0900 Subject: [PATCH 192/263] Fix misspelling of Japanese words in spec (#9636) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit こんいちは (ko-n-i-chi-wa) -> こんにちは (ko-n-ni-chi-wa) --- spec/std/string_spec.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 408f6ec2f12f..f059ff41ad9f 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -97,7 +97,7 @@ describe "String" do end it "gets with start and count with negative start" do - "こんいちは"[-3, 2].should eq("いち") + "こんにちは"[-3, 2].should eq("にち") end it "raises if index out of bounds" do @@ -108,7 +108,7 @@ describe "String" do it "raises if index out of bounds with utf-8" do expect_raises(IndexError) do - "こんいちは"[6, 1] + "こんにちは"[6, 1] end end @@ -120,7 +120,7 @@ describe "String" do it "raises if count is negative with utf-8" do expect_raises(ArgumentError) do - "こんいちは"[3, -1] + "こんにちは"[3, -1] end end @@ -1171,7 +1171,7 @@ describe "String" do end it "reverses utf-8 string" do - "こんいちは".reverse.should eq("はちいんこ") + "こんにちは".reverse.should eq("はちにんこ") end it "reverses taking grapheme clusters into account" do From 72b50e8943b85825848dcafe68d8235141a87655 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Tue, 28 Jul 2020 16:34:13 +0900 Subject: [PATCH 193/263] Remove unnecessary fallback 'def ==(other)' (#9571) There are many fallback 'def ==(other)' (it returns `false` always) in stdlib. However some fallback defs are unnecessary because such fallback is already defined in ancestor type. --- spec/std/bit_array_spec.cr | 4 ++++ spec/std/deque_spec.cr | 9 +++++++++ spec/std/http/params_spec.cr | 16 ++++++++++++++++ spec/std/socket/address_spec.cr | 11 +++++++++++ src/bit_array.cr | 4 ---- src/deque.cr | 5 ----- src/http/params.cr | 4 ---- src/socket/address.cr | 4 ---- 8 files changed, 40 insertions(+), 17 deletions(-) diff --git a/spec/std/bit_array_spec.cr b/spec/std/bit_array_spec.cr index f4ed5f6c880c..d0086fca63df 100644 --- a/spec/std/bit_array_spec.cr +++ b/spec/std/bit_array_spec.cr @@ -74,6 +74,10 @@ describe "BitArray" do (b == c).should be_false (a == d).should be_false end + + it "compares other type" do + from_int(3, 0b101).should_not eq("other type") + end end describe "[]" do diff --git a/spec/std/deque_spec.cr b/spec/std/deque_spec.cr index 0bef3b5b0960..fb6a43eb60f4 100644 --- a/spec/std/deque_spec.cr +++ b/spec/std/deque_spec.cr @@ -145,6 +145,15 @@ describe "Deque" do (b == c).should be_false (a == d).should be_false end + + it "compares other types" do + a = Deque{1, 2, 3} + b = Deque{:foo, :bar} + c = "other type" + (a == b).should be_false + (b == c).should be_false + (a == c).should be_false + end end describe "+" do diff --git a/spec/std/http/params_spec.cr b/spec/std/http/params_spec.cr index 26107252a982..0eb1fe027ea4 100644 --- a/spec/std/http/params_spec.cr +++ b/spec/std/http/params_spec.cr @@ -253,5 +253,21 @@ module HTTP Params.new.empty?.should be_true end end + + describe "#==" do + it "compares other" do + a = Params.parse("a=foo&b=bar") + b = Params.parse("a=bar&b=foo") + (a == a).should be_true + (b == b).should be_true + (a == b).should be_false + end + + it "compares other types" do + a = Params.parse("a=foo&b=bar") + b = "other type" + (a == b).should be_false + end + end end end diff --git a/spec/std/socket/address_spec.cr b/spec/std/socket/address_spec.cr index 59167fc9c041..c72e7c0bdf13 100644 --- a/spec/std/socket/address_spec.cr +++ b/spec/std/socket/address_spec.cr @@ -244,4 +244,15 @@ describe Socket do Socket.ip?("::0::ffff:c0a8:5e4").should be_false Socket.ip?("c0a8").should be_false end + + it "==" do + a = Socket::IPAddress.new("127.0.0.1", 8080) + b = Socket::UNIXAddress.new("some_path") + c = "sonme_path" + (a == a).should be_true + (b == b).should be_true + (a == b).should be_false + (a == c).should be_false + (b == c).should be_false + end end diff --git a/src/bit_array.cr b/src/bit_array.cr index efb9ba9b6766..48578c259955 100644 --- a/src/bit_array.cr +++ b/src/bit_array.cr @@ -38,10 +38,6 @@ struct BitArray return LibC.memcmp(@bits, other.@bits, malloc_size) == 0 end - def ==(other) - false - end - def unsafe_fetch(index : Int) bit_index, sub_index = index.divmod(32) (@bits[bit_index] & (1 << sub_index)) > 0 diff --git a/src/deque.cr b/src/deque.cr index 1300149e7aea..4175269496dc 100644 --- a/src/deque.cr +++ b/src/deque.cr @@ -112,11 +112,6 @@ class Deque(T) equals?(other) { |x, y| x == y } end - # :nodoc: - def ==(other) - false - end - # Concatenation. Returns a new `Deque` built by concatenating # two deques together to create a third. The type of the new deque # is the union of the types of both the other deques. diff --git a/src/http/params.cr b/src/http/params.cr index 66527ce3e904..e1897bd99234 100644 --- a/src/http/params.cr +++ b/src/http/params.cr @@ -146,10 +146,6 @@ module HTTP self.raw_params == other.raw_params end - def ==(other) - false - end - # Returns first value for specified param name. # # ``` diff --git a/src/socket/address.cr b/src/socket/address.cr index 14b90be5023d..2369de2cc809 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -50,10 +50,6 @@ class Socket end abstract def to_unsafe : LibC::Sockaddr* - - def ==(other) - false - end end # IP address representation. From fd5e53f922b63fccb2bb34cfe40c27b160cfb9c4 Mon Sep 17 00:00:00 2001 From: moe Date: Tue, 28 Jul 2020 22:06:23 +0200 Subject: [PATCH 194/263] Fix typos (#9638) --- CHANGELOG.md | 6 +++--- spec/compiler/semantic/class_var_spec.cr | 2 +- spec/compiler/semantic/reflection_spec.cr | 2 +- spec/generate_windows_spec.sh | 2 +- src/channel.cr | 2 +- src/compiler/crystal/codegen/call.cr | 2 +- src/compiler/crystal/codegen/primitives.cr | 2 +- src/compiler/crystal/macros.cr | 2 +- src/compiler/crystal/semantic.cr | 2 +- src/compiler/crystal/semantic/call.cr | 2 +- .../crystal/semantic/exhaustiveness_checker.cr | 6 +++--- src/compiler/crystal/semantic/main_visitor.cr | 10 +++++----- src/compiler/crystal/semantic/method_lookup.cr | 2 +- src/compiler/crystal/semantic/top_level_visitor.cr | 2 +- .../crystal/semantic/type_declaration_processor.cr | 2 +- src/compiler/crystal/tools/doc/method.cr | 2 +- .../tools/playground/agent_instrumentor_transformer.cr | 2 +- src/crystal/dwarf/line_numbers.cr | 2 +- src/docs_pseudo_methods.cr | 2 +- src/exception/call_stack/mach_o.cr | 2 +- src/float/printer/grisu3.cr | 2 +- src/hash.cr | 2 +- src/http/server/response.cr | 2 +- src/humanize.cr | 2 +- src/json/from_json.cr | 2 +- src/json/to_json.cr | 2 +- src/log/main.cr | 2 +- src/openssl/ssl/context.cr | 2 +- src/option_parser.cr | 2 +- src/slice.cr | 8 ++++---- src/socket/address.cr | 2 +- src/spec/expectations.cr | 2 +- src/time.cr | 6 +++--- src/yaml/builder.cr | 2 +- src/yaml/serialization.cr | 2 +- 35 files changed, 48 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 621699a07a92..da02a193c68c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -520,8 +520,8 @@ - Fixed indent after comment inside indexer. ([#8627](https://github.com/crystal-lang/crystal/pull/8627), thanks @asterite) - Fixed indent of comments at the end of a proc literal. ([#8778](https://github.com/crystal-lang/crystal/pull/8778), thanks @asterite) -- Fixed crash when formating comment after macro. ([#8697](https://github.com/crystal-lang/crystal/pull/8697), thanks @asterite) -- Fixed crash when formating `exp.!`. ([#8768](https://github.com/crystal-lang/crystal/pull/8768), thanks @asterite) +- Fixed crash when formatting comment after macro. ([#8697](https://github.com/crystal-lang/crystal/pull/8697), thanks @asterite) +- Fixed crash when formatting `exp.!`. ([#8768](https://github.com/crystal-lang/crystal/pull/8768), thanks @asterite) - Removes unnecessary escape sequences. ([#8619](https://github.com/crystal-lang/crystal/pull/8619), thanks @RX14) ### Doc generator @@ -3493,7 +3493,7 @@ * Added `IO#gets_to_end`. * Added backticks (`...`) and `%x(...)` for command execution. * Added `%r(...)` for regular expression literals. -* Allow interpolations in regular expresion literals. +* Allow interpolations in regular expression literals. * Compiling with `--release` sets a `release` flag that you can test with `ifdef`. * Allow passing splats to C functions * A C type can now be declared like `type Name = Type` (`type Name : Type` will be deprecated). diff --git a/spec/compiler/semantic/class_var_spec.cr b/spec/compiler/semantic/class_var_spec.cr index 1a266f14bf6c..f263b2abd6d3 100644 --- a/spec/compiler/semantic/class_var_spec.cr +++ b/spec/compiler/semantic/class_var_spec.cr @@ -437,7 +437,7 @@ describe "Semantic: class var" do )) { int32 } end - it "doesn't error on recursive depdendency if var is nilable (#2943)" do + it "doesn't error on recursive dependency if var is nilable (#2943)" do assert_type(%( class Foo @@foo : Int32? diff --git a/spec/compiler/semantic/reflection_spec.cr b/spec/compiler/semantic/reflection_spec.cr index f9bb33d8119d..2d79a19f6b61 100644 --- a/spec/compiler/semantic/reflection_spec.cr +++ b/spec/compiler/semantic/reflection_spec.cr @@ -9,7 +9,7 @@ describe "Semantic: reflection" do assert_type("Class") { types["Class"] } end - it "types Object and Class metaclases" do + it "types Object and Class metaclasses" do assert_type("Object.class") { types["Class"] } assert_type("Class.class") { types["Class"] } end diff --git a/spec/generate_windows_spec.sh b/spec/generate_windows_spec.sh index 046b6b64ca23..3cb1f4ebf0f0 100644 --- a/spec/generate_windows_spec.sh +++ b/spec/generate_windows_spec.sh @@ -6,7 +6,7 @@ set +x # * `failed codegen` annotates specs that error in the compiler. # This is mostly caused by some API not being ported to win32 (either the spec # target itself or some tools used by the spec). -# * `failed linking` annotats specs that compile but don't link (at least not on +# * `failed linking` annotates specs that compile but don't link (at least not on # basis of the libraries from *Porting to Windows* guide). # Most failers are caused by missing libraries (libxml2, libyaml, libgmp, # libllvm, libz, libssl), but there also seem to be some incompatibilities diff --git a/src/channel.cr b/src/channel.cr index 500960b8c9f9..3d6a47122a8c 100644 --- a/src/channel.cr +++ b/src/channel.cr @@ -187,7 +187,7 @@ class Channel(T) # All items successfully sent to the channel can be received, before `#receive` considers the channel closed. # Calling `#close` on a closed channel does not have any effect. # - # It returns `true` when the channel was successfuly closed, or `false` if it was already closed. + # It returns `true` when the channel was successfully closed, or `false` if it was already closed. def close : Bool sender_list = Crystal::PointerLinkedList(Sender(T)).new receiver_list = Crystal::PointerLinkedList(Receiver(T)).new diff --git a/src/compiler/crystal/codegen/call.cr b/src/compiler/crystal/codegen/call.cr index 567107050686..4016827f50d5 100644 --- a/src/compiler/crystal/codegen/call.cr +++ b/src/compiler/crystal/codegen/call.cr @@ -25,7 +25,7 @@ class Crystal::CodeGenVisitor return false if @builder.end if block = node.block - # A block might turn into a proc literal but not be used if it particpates in a dispatch + # A block might turn into a proc literal but not be used if it participates in a dispatch if (fun_literal = block.fun_literal) && node.target_def.uses_block_arg? codegen_call_with_block_as_fun_literal(node, fun_literal, owner, call_args) else diff --git a/src/compiler/crystal/codegen/primitives.cr b/src/compiler/crystal/codegen/primitives.cr index 13ae41ea4f04..5c7a00bd8bcd 100644 --- a/src/compiler/crystal/codegen/primitives.cr +++ b/src/compiler/crystal/codegen/primitives.cr @@ -736,7 +736,7 @@ class Crystal::CodeGenVisitor def codegen_primitive_pointer_set(node, target_def, call_args) type = context.type.remove_typedef.as(PointerInstanceType) - # Assinging to a Pointer(Void) has no effect + # Assigning to a Pointer(Void) has no effect return llvm_nil if type.element_type.void? value = call_args[1] diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr index ad26ceb38429..eaa57510e2a4 100644 --- a/src/compiler/crystal/macros.cr +++ b/src/compiler/crystal/macros.cr @@ -1615,7 +1615,7 @@ module Crystal::Macros # class MagicConstant < ASTNode # end - # A fictitious node representing an idenfitifer like, `foo`, `Bar` or `something_else`. + # A fictitious node representing an identifier like, `foo`, `Bar` or `something_else`. # # The parser doesn't create this nodes. Instead, you create them by invoking `id` # on some nodes. For example, invoking `id` on a `StringLiteral` returns a `MacroId` diff --git a/src/compiler/crystal/semantic.cr b/src/compiler/crystal/semantic.cr index b7ac36231e59..931781631bd1 100644 --- a/src/compiler/crystal/semantic.cr +++ b/src/compiler/crystal/semantic.cr @@ -4,7 +4,7 @@ require "./syntax/visitor" require "./semantic/*" # The overall algorithm for semantic analysis of a program is: -# - top level: declare clases, modules, macros, defs and other top-level stuff +# - top level: declare classes, modules, macros, defs and other top-level stuff # - new methods: create `new` methods for every `initialize` method # - type declarations: process type declarations like `@x : Int32` # - check abstract defs: check that abstract defs are implemented diff --git a/src/compiler/crystal/semantic/call.cr b/src/compiler/crystal/semantic/call.cr index 3c69bfb6ecac..6e11c94dd8f2 100644 --- a/src/compiler/crystal/semantic/call.cr +++ b/src/compiler/crystal/semantic/call.cr @@ -131,7 +131,7 @@ class Crystal::Call named_args_types = NamedArgumentType.from_args(named_args, with_literals) matches = lookup_matches_without_splat arg_types, named_args_types, with_literals - # If we checked for automatic casts, see if an ambigous call was produced + # If we checked for automatic casts, see if an ambiguous call was produced if with_literals arg_types.each &.check_restriction_exception named_args_types.try &.each &.type.check_restriction_exception diff --git a/src/compiler/crystal/semantic/exhaustiveness_checker.cr b/src/compiler/crystal/semantic/exhaustiveness_checker.cr index f481e1f202df..bc06f3b8dce2 100644 --- a/src/compiler/crystal/semantic/exhaustiveness_checker.cr +++ b/src/compiler/crystal/semantic/exhaustiveness_checker.cr @@ -16,7 +16,7 @@ struct Crystal::ExhaustivenessChecker cond_type = cond.type? # No type on condition means we couldn't type it so we can't - # check exhasutiveness. + # check exhaustiveness. return unless cond_type # Compute all types that we must cover. @@ -99,7 +99,7 @@ struct Crystal::ExhaustivenessChecker elements = cond.elements # No type on condition means we couldn't type it so we can't - # check exhasutiveness. + # check exhaustiveness. return unless elements.all? &.type? element_types = elements.map &.type @@ -179,7 +179,7 @@ struct Crystal::ExhaustivenessChecker end end - # Retuens an array of all the types inside `type`: + # Returns an array of all the types inside `type`: # for unions it's all the union types, otherwise it's just that type. private def expand_types(type) if type.is_a?(UnionType) diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index c6b4ec4009c4..bf5ddb6bfb65 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -58,11 +58,11 @@ module Crystal property yield_vars : Array(Var)? # In vars we store the types of variables as we traverse the nodes. - # These type are not cummulative: if you do `x = 1`, 'x' will have + # These type are not cumulative: if you do `x = 1`, 'x' will have # type Int32. Then if you do `x = false`, 'x' will have type Bool. getter vars - # Here we store the cummulative types of variables as we traverse the nodes. + # Here we store the cumulative types of variables as we traverse the nodes. getter meta_vars : MetaVars property is_initialize : Bool property exception_handler_vars : MetaVars? = nil @@ -127,7 +127,7 @@ module Crystal # We initialize meta_vars from vars given in the constructor. # We store those meta vars either in the typed def or in the program - # so the codegen phase knows the cummulative types to do allocas. + # so the codegen phase knows the cumulative types to do allocas. unless meta_vars if typed_def = @typed_def meta_vars = typed_def.vars = MetaVars.new @@ -843,7 +843,7 @@ module Crystal def type_assign(target : InstanceVar, value, node) # Check if this is an instance variable initializer unless @scope - # `InstanceVar` assignment appered in block is not checked + # `InstanceVar` assignment appeared in block is not checked # by `Crystal::InstanceVarsInitializerVisitor` because this block # may be passed to a macro. So, it checks here. if current_type.is_a?(Program) || current_type.is_a?(FileModule) @@ -1288,7 +1288,7 @@ module Crystal if expand_macro(node) # It can happen that this call is inside an ArrayLiteral or HashLiteral, # was expanded but isn't bound to the expansion because the call (together - # with its expantion) was cloned. + # with its expansion) was cloned. if (expanded = node.expanded) && (!node.dependencies? || !node.type?) node.bind_to(expanded) end diff --git a/src/compiler/crystal/semantic/method_lookup.cr b/src/compiler/crystal/semantic/method_lookup.cr index 78569c9d1a9a..8b7ac5672f3a 100644 --- a/src/compiler/crystal/semantic/method_lookup.cr +++ b/src/compiler/crystal/semantic/method_lookup.cr @@ -305,7 +305,7 @@ module Crystal if a_def.double_splat match_arg_type = named_arg.type - # If there's a restrction on the double splat, check that it matches + # If there's a restriction on the double splat, check that it matches if double_splat_restriction if double_splat_entries double_splat_entries << named_arg diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index deb9c4c861d3..35a41277519a 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -1147,7 +1147,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor next_type = base_type.lookup_path_item(name, lookup_in_namespace: false, include_private: true, location: path.location) if next_type if next_type.is_a?(ASTNode) - path.raise "execpted #{name} to be a type" + path.raise "expected #{name} to be a type" end else base_type = check_type_is_type_container(base_type, path) diff --git a/src/compiler/crystal/semantic/type_declaration_processor.cr b/src/compiler/crystal/semantic/type_declaration_processor.cr index 52175e8652cd..97f636207065 100644 --- a/src/compiler/crystal/semantic/type_declaration_processor.cr +++ b/src/compiler/crystal/semantic/type_declaration_processor.cr @@ -105,7 +105,7 @@ struct Crystal::TypeDeclarationProcessor # they are initialized in at least one of the initialize methods. @non_nilable_instance_vars = {} of Type => Array(String) - # Nilable variables there were detected to not be initilized in an initialize, + # Nilable variables there were detected to not be initialized in an initialize, # but a superclass does initialize it. It's only an error if the explicit/guessed # type is not nilable itself. @nilable_instance_vars = {} of Type => Hash(String, InitializeInfo) diff --git a/src/compiler/crystal/tools/doc/method.cr b/src/compiler/crystal/tools/doc/method.cr index 629845c3e8a5..320f270799bd 100644 --- a/src/compiler/crystal/tools/doc/method.cr +++ b/src/compiler/crystal/tools/doc/method.cr @@ -38,7 +38,7 @@ class Crystal::Doc::Method end # Returns this method's docs ready to be shown (before formatting) - # in the UI. This includes copiying docs from previous def or + # in the UI. This includes copying docs from previous def or # ancestors and replacing `:inherit:` with the ancestor docs. # This docs not include the "Description copied from ..." banner # in case it's needed. diff --git a/src/compiler/crystal/tools/playground/agent_instrumentor_transformer.cr b/src/compiler/crystal/tools/playground/agent_instrumentor_transformer.cr index e009bf1a8804..54ffec6cb6d4 100644 --- a/src/compiler/crystal/tools/playground/agent_instrumentor_transformer.cr +++ b/src/compiler/crystal/tools/playground/agent_instrumentor_transformer.cr @@ -61,7 +61,7 @@ module Crystal # so the instrumentor can ignore call's of methods with this name # this will avoid instrumenting calls to methods with the same name than # declared macros in the playground source. For a more accurate solution - # a compilation should be done to distigush whether each call refers to a macro or + # a compilation should be done to distinguish whether each call refers to a macro or # a method. Between the macro names collection and only instrumenting def's inside # modules/classes the generated instrumentation is pretty good enough. See #2355 collector = MacroDefNameCollector.new diff --git a/src/crystal/dwarf/line_numbers.cr b/src/crystal/dwarf/line_numbers.cr index ff6c836072f3..6d303f3f88cd 100644 --- a/src/crystal/dwarf/line_numbers.cr +++ b/src/crystal/dwarf/line_numbers.cr @@ -159,7 +159,7 @@ module Crystal end end - # Matrix of decompressed `Row` to search line number informations from the + # Matrix of decompressed `Row` to search line number information from the # address of an instruction. # # The matrix contains indexed references to `directories` and `files` to diff --git a/src/docs_pseudo_methods.cr b/src/docs_pseudo_methods.cr index 6a9b44253d7b..5dcbec4ebde0 100644 --- a/src/docs_pseudo_methods.cr +++ b/src/docs_pseudo_methods.cr @@ -50,7 +50,7 @@ end # Returns the instance size of the given class as number of bytes. # -# *type* must be a constant or `typeof()` expresion. It cannot be evaluated at runtime. +# *type* must be a constant or `typeof()` expression. It cannot be evaluated at runtime. # # ``` # instance_sizeof(String) # => 16 diff --git a/src/exception/call_stack/mach_o.cr b/src/exception/call_stack/mach_o.cr index 883237dadf95..b858fe9da691 100644 --- a/src/exception/call_stack/mach_o.cr +++ b/src/exception/call_stack/mach_o.cr @@ -44,7 +44,7 @@ struct Exception::CallStack end end - # DWARF uses fixed addresses but Darwin loads exectutables at a random + # DWARF uses fixed addresses but Darwin loads executables at a random # address, so we must remove the load offset from the IP to match the # addresses in DWARF sections. # diff --git a/src/float/printer/grisu3.cr b/src/float/printer/grisu3.cr index 4b6bb151314f..79e0a1a67386 100644 --- a/src/float/printer/grisu3.cr +++ b/src/float/printer/grisu3.cr @@ -341,7 +341,7 @@ module Float::Printer::Grisu3 # the difference between w and boundary_minus/plus (a power of 2) and to # compute scaled_boundary_minus/plus by subtracting/adding from # scaled_w. However the code becomes much less readable and the speed - # enhancements are not terriffic. + # enhancements are not terrific. scaled_boundary_minus = boundaries[:minus] * ten_mk scaled_boundary_plus = boundaries[:plus] * ten_mk diff --git a/src/hash.cr b/src/hash.cr index c5577b8d4a33..af982de29d66 100644 --- a/src/hash.cr +++ b/src/hash.cr @@ -140,7 +140,7 @@ class Hash(K, V) # shift of the buffer (expensive). # # There might be other optimizations to try out, like not using Linear Probing, - # but for now this implementaton is much faster than the old one which used + # but for now this implementation is much faster than the old one which used # linked lists (closed addressing). # # All methods that deal with this implementation come after the constructors. diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 5662656746de..86268cabd95b 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -96,7 +96,7 @@ class HTTP::Server raise "Can't read from HTTP::Server::Response" end - # Upgrades this response, writing headers and yieling the connection `IO` (a socket) to the given block. + # Upgrades this response, writing headers and yielding the connection `IO` (a socket) to the given block. # This is useful to implement protocol upgrades, such as websockets. def upgrade(&block : IO ->) write_headers diff --git a/src/humanize.cr b/src/humanize.cr index 794032bb5561..9f2d87640400 100644 --- a/src/humanize.cr +++ b/src/humanize.cr @@ -64,7 +64,7 @@ struct Number SI_PREFIXES = { {'y', 'z', 'a', 'f', 'p', 'n', 'µ', 'm'}, {nil, 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'} } # SI prefixes used by `#humanize`. Equal to `SI_PREFIXES` but prepends the - # prefix with a space charater. + # prefix with a space character. SI_PREFIXES_PADDED = ->(magnitude : Int32, _number : Float64) do magnitude = Number.prefix_index(magnitude) {magnitude, (magnitude == 0 ? " " : si_prefix(magnitude))} diff --git a/src/json/from_json.cr b/src/json/from_json.cr index e0b1c66a62c3..a9903648c785 100644 --- a/src/json/from_json.cr +++ b/src/json/from_json.cr @@ -310,7 +310,7 @@ def Union.new(pull : JSON::PullParser) {% end %} end -# Reads a string from JSON parser as a time formated according to [RFC 3339](https://tools.ietf.org/html/rfc3339) +# Reads a string from JSON parser as a time formatted according to [RFC 3339](https://tools.ietf.org/html/rfc3339) # or other variations of [ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf). # # The JSON format itself does not specify a time data type, this method just diff --git a/src/json/to_json.cr b/src/json/to_json.cr index ebb80566f146..6f0228eb0382 100644 --- a/src/json/to_json.cr +++ b/src/json/to_json.cr @@ -166,7 +166,7 @@ struct Enum end struct Time - # Emits a string formated according to [RFC 3339](https://tools.ietf.org/html/rfc3339) + # Emits a string formatted according to [RFC 3339](https://tools.ietf.org/html/rfc3339) # ([ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf) profile). # # The JSON format itself does not specify a time data type, this method just diff --git a/src/log/main.cr b/src/log/main.cr index 5a695cca2582..18b9aee1a12a 100644 --- a/src/log/main.cr +++ b/src/log/main.cr @@ -66,7 +66,7 @@ class Log # :ditto: def self.context=(value : Log::Context) # NOTE: There is a need for `Metadata` and `Context` setters in - # becuase `Log.context` returns a `Log::Context` for allowing DSL like `Log.context.set(a: 1)` + # because `Log.context` returns a `Log::Context` for allowing DSL like `Log.context.set(a: 1)` # but if the metadata is built manually the construct `Log.context = metadata` will be used. Log.context = value.metadata end diff --git a/src/openssl/ssl/context.cr b/src/openssl/ssl/context.cr index da4eb8d78bf1..5a867bc6267c 100644 --- a/src/openssl/ssl/context.cr +++ b/src/openssl/ssl/context.cr @@ -215,7 +215,7 @@ abstract class OpenSSL::SSL::Context LibSSL.ssl_ctx_free(@handle) end - # Sets the default paths for `ca_certiifcates=` and `ca_certificates_path=`. + # Sets the default paths for `ca_certificates=` and `ca_certificates_path=`. def set_default_verify_paths LibSSL.ssl_ctx_set_default_verify_paths(@handle) end diff --git a/src/option_parser.cr b/src/option_parser.cr index 2df3f00bd616..62fa89c19f65 100644 --- a/src/option_parser.cr +++ b/src/option_parser.cr @@ -221,7 +221,7 @@ class OptionParser end # Adds a separator, with an optional header message, that will be used to - # print the help. The seperator is placed between the flags registered (`#on`) + # print the help. The separator is placed between the flags registered (`#on`) # before, and the flags registered after the call. # # This way, you can group the different options in an easier to read way. diff --git a/src/slice.cr b/src/slice.cr index 0fae0e55155b..6ca401b85d77 100644 --- a/src/slice.cr +++ b/src/slice.cr @@ -357,7 +357,7 @@ struct Slice(T) # Copies the contents of this slice into *target*. # - # Raises `IndexError` if the desination slice cannot fit the data being transferred + # Raises `IndexError` if the destination slice cannot fit the data being transferred # e.g. dest.size < self.size. # # ``` @@ -376,7 +376,7 @@ struct Slice(T) # Copies the contents of *source* into this slice. # - # Raises `IndexError` if the desination slice cannot fit the data being transferred. + # Raises `IndexError` if the destination slice cannot fit the data being transferred. @[AlwaysInline] def copy_from(source : self) source.copy_to(self) @@ -396,7 +396,7 @@ struct Slice(T) # Moves the contents of this slice into *target*. *target* and `self` may # overlap; the copy is always done in a non-destructive manner. # - # Raises `IndexError` if the desination slice cannot fit the data being transferred + # Raises `IndexError` if the destination slice cannot fit the data being transferred # e.g. `dest.size < self.size`. # # ``` @@ -418,7 +418,7 @@ struct Slice(T) # Moves the contents of *source* into this slice. *source* and `self` may # overlap; the copy is always done in a non-destructive manner. # - # Raises `IndexError` if the desination slice cannot fit the data being transferred. + # Raises `IndexError` if the destination slice cannot fit the data being transferred. @[AlwaysInline] def move_from(source : self) source.move_to(self) diff --git a/src/socket/address.cr b/src/socket/address.cr index 2369de2cc809..6dba99637fee 100644 --- a/src/socket/address.cr +++ b/src/socket/address.cr @@ -67,7 +67,7 @@ class Socket # ``` # # `IPAddress` won't resolve domains, including `localhost`. If you must - # resolve an IP, or don't know whether a `String` constains an IP or a domain + # resolve an IP, or don't know whether a `String` contains an IP or a domain # name, you should use `Addrinfo.resolve` instead. struct IPAddress < Address UNSPECIFIED = "0.0.0.0" diff --git a/src/spec/expectations.cr b/src/spec/expectations.cr index 168facbfb42f..78ac5a47d53c 100644 --- a/src/spec/expectations.cr +++ b/src/spec/expectations.cr @@ -380,7 +380,7 @@ module Spec def expect_raises(klass : T.class, message : String | Regex | Nil = nil, file = __FILE__, line = __LINE__) forall T yield rescue ex : T - # We usually bubble Spec::AssertaionFailed, unless this is the expected exception + # We usually bubble Spec::AssertionFailed, unless this is the expected exception if ex.is_a?(Spec::AssertionFailed) && klass != Spec::AssertionFailed raise ex end diff --git a/src/time.cr b/src/time.cr index 0a7cebedde8b..02b2a8883747 100644 --- a/src/time.cr +++ b/src/time.cr @@ -189,7 +189,7 @@ require "crystal/system/time" # computer's wall clock has changed between both calls. # # As an alternative, the operating system also provides a monotonic clock. -# Its time-line has no specfied starting point but is strictly linearly +# Its time-line has no specified starting point but is strictly linearly # increasing. # # This monotonic clock should always be used for measuring elapsed time. @@ -584,7 +584,7 @@ struct Time # date `2007-04-31` which will be adjusted to `2007-04-30`. # # This operates on the local time-line, such that the local date-time - # represenations of month and year are increased by the specified amount. + # representations of month and year are increased by the specified amount. # # If the resulting date-time is ambiguous due to time zone transitions, # a correct time will be returned, but it does not guarantee which. @@ -602,7 +602,7 @@ struct Time # date `2007-04-31` which will be adjusted to `2007-04-30`. # # This operates on the local time-line, such that the local date-time - # represenations of month and year are decreased by the specified amount. + # representations of month and year are decreased by the specified amount. # # If the resulting date-time is ambiguous due to time zone transitions, # a correct time will be returned, but it does not guarantee which. diff --git a/src/yaml/builder.cr b/src/yaml/builder.cr index cc925fc5a107..03e5a86a46c7 100644 --- a/src/yaml/builder.cr +++ b/src/yaml/builder.cr @@ -26,7 +26,7 @@ class YAML::Builder @box : Void* - # By default the maximum nesting of sequences/amppings is 99. Nesting more + # By default the maximum nesting of sequences/mappings is 99. Nesting more # than this will result in a YAML::Error. Changing the value of this property # allows more/less nesting. property max_nesting = 99 diff --git a/src/yaml/serialization.cr b/src/yaml/serialization.cr index bd1b086d16d4..10b0ae2b8f1e 100644 --- a/src/yaml/serialization.cr +++ b/src/yaml/serialization.cr @@ -59,7 +59,7 @@ module YAML # ``` # # `YAML::Field` properties: - # * **ignore**: if `true` skip this field in seriazation and deserialization (by default false) + # * **ignore**: if `true` skip this field in serialization and deserialization (by default false) # * **key**: the value of the key in the yaml object (by default the name of the instance variable) # * **converter**: specify an alternate type for parsing and generation. The converter must define `from_yaml(YAML::ParseContext, YAML::Nodes::Node)` and `to_yaml(value, YAML::Nodes::Builder)` as class methods. Examples of converters are `Time::Format` and `Time::EpochConverter` for `Time`. # * **presence**: if `true`, a `@{{key}}_present` instance variable will be generated when the key was present (even if it has a `null` value), `false` by default From 0bf1bcdca40386cb1cf77c536f96452c81fcf303 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 29 Jul 2020 13:57:17 -0300 Subject: [PATCH 195/263] Delay setup of HTTP::CompressHandler until content is written (#9625) --- .../server/handlers/compress_handler_spec.cr | 84 +++++++++++++++++++ spec/std/http/web_socket_spec.cr | 24 ++++++ src/http/server/handlers/compress_handler.cr | 55 +++++++++--- 3 files changed, 153 insertions(+), 10 deletions(-) diff --git a/spec/std/http/server/handlers/compress_handler_spec.cr b/spec/std/http/server/handlers/compress_handler_spec.cr index 7d08e330538b..f6c0b5ab653d 100644 --- a/spec/std/http/server/handlers/compress_handler_spec.cr +++ b/spec/std/http/server/handlers/compress_handler_spec.cr @@ -66,4 +66,88 @@ describe HTTP::CompressHandler do gzip = Compress::Gzip::Reader.new(io2) gzip.gets_to_end.should eq("Hello") end + + it "doesn't compress twice" do + io = IO::Memory.new + request = HTTP::Request.new("GET", "/") + request.headers["Accept-Encoding"] = "gzip" + + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + handler1 = HTTP::CompressHandler.new + handler2 = HTTP::CompressHandler.new + handler1.next = handler2 + handler2.next = HTTP::Handler::HandlerProc.new do |ctx| + ctx.response.print "Hello" + end + handler1.call(context) + response.close + + io.rewind + response2 = HTTP::Client::Response.from_io(io) + response2.body.should eq("Hello") + end + + it "fix content-length header" do + io = IO::Memory.new + request = HTTP::Request.new("GET", "/") + request.headers["Accept-Encoding"] = "gzip" + + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + handler = HTTP::CompressHandler.new + handler.next = HTTP::Handler::HandlerProc.new do |ctx| + ctx.response.content_length = 5 + ctx.response.print "Hello" + ctx.response.flush + end + handler.call(context) + response.close + + io.rewind + response = HTTP::Client::Response.from_io(io) + response.body.should eq("Hello") + end + + it "don't try to compress for empty body responses" do + io = IO::Memory.new + request = HTTP::Request.new("GET", "/") + request.headers["Accept-Encoding"] = "gzip" + + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + handler = HTTP::CompressHandler.new + handler.next = HTTP::Handler::HandlerProc.new do |ctx| + context.response.status = :not_modified + end + handler.call(context) + response.close + + io.rewind + io.to_s.should eq("HTTP/1.1 304 Not Modified\r\nContent-Length: 0\r\n\r\n") + end + + it "don't try to compress upgraded response" do + io = IO::Memory.new + request = HTTP::Request.new("GET", "/") + request.headers["Accept-Encoding"] = "gzip" + + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + handler = HTTP::CompressHandler.new + handler.next = HTTP::Handler::HandlerProc.new do |ctx| + response.status = :switching_protocols + response.upgrade do |io| + end + end + handler.call(context) + response.close + + io.rewind + io.to_s.should eq("HTTP/1.1 101 Switching Protocols\r\n\r\n") + end end diff --git a/spec/std/http/web_socket_spec.cr b/spec/std/http/web_socket_spec.cr index a0d9cf0d0e04..c6ec2c4271e0 100644 --- a/spec/std/http/web_socket_spec.cr +++ b/spec/std/http/web_socket_spec.cr @@ -461,6 +461,30 @@ describe HTTP::WebSocket do end end + it "doesn't compress upgrade response body" do + compress_handler = HTTP::CompressHandler.new + ws_handler = HTTP::WebSocketHandler.new do |ws, ctx| + ws.on_message do |str| + ws.send(str) + end + end + http_server = HTTP::Server.new([compress_handler, ws_handler]) + + address = http_server.bind_unused_port + + run_server(http_server) do + client = HTTP::WebSocket.new("ws://#{address}", headers: HTTP::Headers{"Accept-Encoding" => "gzip"}) + message = nil + client.on_message do |msg| + message = msg + client.close + end + client.send "hello" + client.run + message.should eq("hello") + end + end + describe "handshake fails if server does not verify Sec-WebSocket-Key" do it "Sec-WebSocket-Accept missing" do http_server = HTTP::Server.new do |context| diff --git a/src/http/server/handlers/compress_handler.cr b/src/http/server/handlers/compress_handler.cr index d124fd477765..8dd5e66c4404 100644 --- a/src/http/server/handlers/compress_handler.cr +++ b/src/http/server/handlers/compress_handler.cr @@ -12,17 +12,52 @@ class HTTP::CompressHandler {% if flag?(:without_zlib) %} call_next(context) {% else %} - request_headers = context.request.headers - - if request_headers.includes_word?("Accept-Encoding", "gzip") - context.response.headers["Content-Encoding"] = "gzip" - context.response.output = Compress::Gzip::Writer.new(context.response.output, sync_close: true) - elsif request_headers.includes_word?("Accept-Encoding", "deflate") - context.response.headers["Content-Encoding"] = "deflate" - context.response.output = Compress::Deflate::Writer.new(context.response.output, sync_close: true) - end - + context.response.output = CompressIO.new(context.response.output, context) call_next(context) {% end %} end + + {% unless flag?(:without_zlib) %} + private class CompressIO < IO + def initialize(@io : IO, @context : HTTP::Server::Context) + @checked = false + end + + def read(slice : Bytes) + raise NotImplementedError.new("read") + end + + def write(slice : Bytes) : Nil + check_output unless @checked + @io.write(slice) + end + + def flush + @io.flush + end + + def close + @io.close + end + + private def check_output + @checked = true + + return if @context.response.wrote_headers? + return if @context.response.headers.has_key?("Content-Encoding") + + request_headers = @context.request.headers + + if request_headers.includes_word?("Accept-Encoding", "gzip") + @context.response.headers["Content-Encoding"] = "gzip" + @context.response.headers.delete("Content-Length") + @io = Compress::Gzip::Writer.new(@io, sync_close: true) + elsif request_headers.includes_word?("Accept-Encoding", "deflate") + @context.response.headers["Content-Encoding"] = "deflate" + @context.response.headers.delete("Content-Length") + @io = Compress::Deflate::Writer.new(@io, sync_close: true) + end + end + end + {% end %} end From a1190617b61b60dd8594f237b336c107a7d130fc Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Thu, 30 Jul 2020 10:17:33 -0300 Subject: [PATCH 196/263] Run `brew update --preinstall` before installing (#9642) --- .circleci/config.yml | 1 - bin/ci | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8be9ceeb152d..59c9ab0cc2a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,7 +92,6 @@ jobs: keys: - brew-cache-v1 - checkout - - run: brew update - run: bin/ci prepare_system - run: echo 'export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/openssl@1.1/lib/pkgconfig"' >> $BASH_ENV - run: echo 'export CURRENT_TAG="$CIRCLE_TAG"' >> $BASH_ENV diff --git a/bin/ci b/bin/ci index 098ce4efa34b..221760bf3f11 100755 --- a/bin/ci +++ b/bin/ci @@ -117,6 +117,7 @@ prepare_build() { on_osx curl -L https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-darwin-x86_64.tar.gz -o ~/crystal.tar.gz on_osx 'pushd ~;gunzip -c ~/crystal.tar.gz | tar xopf -;mv crystal-0.35.1-1 crystal;popd' + on_osx brew update --preinstall on_osx brew bundle --no-lock # Note: brew link --force might show: From bbcb615bf791f1abed55aa482acec10730e26bf5 Mon Sep 17 00:00:00 2001 From: Mauricio Gomes Date: Thu, 30 Jul 2020 09:18:17 -0400 Subject: [PATCH 197/263] =?UTF-8?q?Fix=20typo=20in=20Nagle=E2=80=99s=20alg?= =?UTF-8?q?orithm=20name=20(#9561)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/socket/tcp_socket.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/socket/tcp_socket.cr b/src/socket/tcp_socket.cr index e31736e947bc..cbf105c80b7b 100644 --- a/src/socket/tcp_socket.cr +++ b/src/socket/tcp_socket.cr @@ -60,7 +60,7 @@ class TCPSocket < IPSocket end end - # Returns `true` if the Nable algorithm is disabled. + # Returns `true` if the Nagle algorithm is disabled. def tcp_nodelay? getsockopt_bool LibC::TCP_NODELAY, level: Protocol::TCP end From 8de7f2d062d7bc7d3c89d960c933e0df95511db4 Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Thu, 30 Jul 2020 15:19:05 +0200 Subject: [PATCH 198/263] Cleanup YAML/JSON Any#dig methods (#9415) * Cleanup Any#dig methods Any#[] always return an Any, so it is always diggable. This also means the exception is superfluous. * Add return types to Any#dig methods * Rename `key` to `index_or_key` to match YAML::Any * Cleanup Crystal.datum `dig` methods --- src/crystal/datum.cr | 19 ++++++++----------- src/json/any.cr | 21 ++++++++------------- src/yaml/any.cr | 17 ++++++----------- 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/src/crystal/datum.cr b/src/crystal/datum.cr index 6e95c6bb6de3..d034a9cf3748 100644 --- a/src/crystal/datum.cr +++ b/src/crystal/datum.cr @@ -106,30 +106,27 @@ module Crystal # Traverses the depth of a structure and returns the value. # Returns `nil` if not found. - def dig?(index_or_key, *subkeys) - if value = self[index_or_key]? - value.dig?(*subkeys) - end + def dig?(index_or_key, *subkeys) : self? + self[index_or_key]?.try &.dig?(*subkeys) end # :nodoc: - def dig?(index_or_key) + def dig?(index_or_key) : self? case @raw when Hash, Array self[index_or_key]? + else + nil end end # Traverses the depth of a structure and returns the value, otherwise raises. - def dig(index_or_key, *subkeys) - if (value = self[index_or_key]) && value.responds_to?(:dig) - return value.dig(*subkeys) - end - raise "#{self.class} value not diggable for key: #{index_or_key.inspect}" + def dig(index_or_key, *subkeys) : self + self[index_or_key].dig(*subkeys) end # :nodoc: - def dig(index_or_key) + def dig(index_or_key) : self self[index_or_key] end diff --git a/src/json/any.cr b/src/json/any.cr index b68e18543f41..55f4533e6fbb 100644 --- a/src/json/any.cr +++ b/src/json/any.cr @@ -119,33 +119,28 @@ struct JSON::Any # Traverses the depth of a structure and returns the value. # Returns `nil` if not found. - def dig?(key : String | Int, *subkeys) - if value = self[key]? - value.dig?(*subkeys) - end + def dig?(index_or_key : String | Int, *subkeys) : JSON::Any? + self[index_or_key]?.try &.dig?(*subkeys) end # :nodoc: - def dig?(key : String | Int) + def dig?(index_or_key : String | Int) : JSON::Any? case @raw when Hash, Array - self[key]? + self[index_or_key]? else nil end end # Traverses the depth of a structure and returns the value, otherwise raises. - def dig(key : String | Int, *subkeys) - if (value = self[key]) && value.responds_to?(:dig) - return value.dig(*subkeys) - end - raise "JSON::Any value not diggable for key: #{key.inspect}" + def dig(index_or_key : String | Int, *subkeys) : JSON::Any + self[index_or_key].dig(*subkeys) end # :nodoc: - def dig(key : String | Int) - self[key] + def dig(index_or_key : String | Int) : JSON::Any + self[index_or_key] end # Checks that the underlying value is `Nil`, and returns `nil`. diff --git a/src/yaml/any.cr b/src/yaml/any.cr index 300e1e59cb69..e037afa2ceb7 100644 --- a/src/yaml/any.cr +++ b/src/yaml/any.cr @@ -128,14 +128,12 @@ struct YAML::Any # Traverses the depth of a structure and returns the value. # Returns `nil` if not found. - def dig?(index_or_key, *subkeys) - if value = self[index_or_key]? - value.dig?(*subkeys) - end + def dig?(index_or_key, *subkeys) : YAML::Any? + self[index_or_key]?.try &.dig?(*subkeys) end # :nodoc: - def dig?(index_or_key) + def dig?(index_or_key) : YAML::Any? case @raw when Hash, Array self[index_or_key]? @@ -145,15 +143,12 @@ struct YAML::Any end # Traverses the depth of a structure and returns the value, otherwise raises. - def dig(index_or_key, *subkeys) - if (value = self[index_or_key]) && value.responds_to?(:dig) - return value.dig(*subkeys) - end - raise "YAML::Any value not diggable for key: #{index_or_key.inspect}" + def dig(index_or_key, *subkeys) : YAML::Any + self[index_or_key].dig(*subkeys) end # :nodoc: - def dig(index_or_key) + def dig(index_or_key) : YAML::Any self[index_or_key] end From 4847a1411dd8ceaf1defabe88fa140ef117898c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Su=C3=A1rez?= Date: Thu, 30 Jul 2020 15:19:34 +0200 Subject: [PATCH 199/263] Print a message pointing to the online playground if compiled without it (#9622) * Print a message pointing to the online playground if compiled without it * Use more precise message when playground is not compiled in This way, is more clear, that the playground and the online page are not the same tools. --- src/compiler/crystal/command.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/compiler/crystal/command.cr b/src/compiler/crystal/command.cr index 11b687f8dc2f..704fa80e33bc 100644 --- a/src/compiler/crystal/command.cr +++ b/src/compiler/crystal/command.cr @@ -77,6 +77,7 @@ class Crystal::Command options.shift {% if flag?(:without_playground) %} puts "Crystal was compiled without playground support" + puts "Try the online code evaluation and sharing tool at https://play.crystal-lang.org" exit 1 {% else %} playground From 7e80d23de3f6f86513037595ec55e91cbe932af4 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Thu, 30 Jul 2020 22:20:41 +0900 Subject: [PATCH 200/263] Attach doc comment to annotation in macro expansion (#9630) Fixed #9628 --- spec/compiler/semantic/doc_spec.cr | 15 +++++++++++++++ src/compiler/crystal/semantic/semantic_visitor.cr | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/spec/compiler/semantic/doc_spec.cr b/spec/compiler/semantic/doc_spec.cr index 9c985ffef953..f4ce5a0d11c6 100644 --- a/spec/compiler/semantic/doc_spec.cr +++ b/spec/compiler/semantic/doc_spec.cr @@ -415,6 +415,21 @@ describe "Semantic: doc" do type.doc.should eq("Some description") end + it "attaches doc to annotation in macro expansion (#9628)" do + result = semantic %( + macro ann + annotation MyAnnotation + end + end + + # Some description + ann + ), wants_doc: true + program = result.program + type = program.types["MyAnnotation"] + type.doc.should eq("Some description") + end + context "doc before annotation" do it "attached to struct/class" do result = semantic %( diff --git a/src/compiler/crystal/semantic/semantic_visitor.cr b/src/compiler/crystal/semantic/semantic_visitor.cr index 966475700650..852c1b29fcee 100644 --- a/src/compiler/crystal/semantic/semantic_visitor.cr +++ b/src/compiler/crystal/semantic/semantic_visitor.cr @@ -367,7 +367,7 @@ abstract class Crystal::SemanticVisitor < Crystal::Visitor def initialize(@doc) end - def visit(node : ClassDef | ModuleDef | EnumDef | Def | FunDef | Alias | Assign | Call) + def visit(node : ClassDef | ModuleDef | EnumDef | Def | FunDef | AnnotationDef | Alias | Assign | Call) node.doc ||= @doc false end From 6251604ceb9391fa3f87aed08d10f55239054ab0 Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Mon, 3 Aug 2020 00:59:03 -0400 Subject: [PATCH 201/263] Raise if passing null Pointer to String --- src/string.cr | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/string.cr b/src/string.cr index e9b70b1e3f4f..b4fefb2e8cc6 100644 --- a/src/string.cr +++ b/src/string.cr @@ -191,6 +191,10 @@ class String # String.new(ptr) # => "abcd" # ``` def self.new(chars : UInt8*) + if chars.null? + raise "Cannot generate a string from a null pointer" + end + new(chars, LibC.strlen(chars)) end @@ -208,6 +212,10 @@ class String def self.new(chars : UInt8*, bytesize, size = 0) # Avoid allocating memory for the empty string return "" if bytesize == 0 + + if chars.null? + raise "Cannot generate a string from a null pointer" + end new(bytesize) do |buffer| buffer.copy_from(chars, bytesize) From 0c53ea21d16c4833b4d674daadede91e3c365156 Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Mon, 3 Aug 2020 04:01:14 -0400 Subject: [PATCH 202/263] Raise ArgumentError and improve error message --- src/string.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/string.cr b/src/string.cr index b4fefb2e8cc6..b988743e80d4 100644 --- a/src/string.cr +++ b/src/string.cr @@ -192,7 +192,7 @@ class String # ``` def self.new(chars : UInt8*) if chars.null? - raise "Cannot generate a string from a null pointer" + raise ArgumentError.new("Cannot create a string with a null pointer") end new(chars, LibC.strlen(chars)) @@ -214,7 +214,7 @@ class String return "" if bytesize == 0 if chars.null? - raise "Cannot generate a string from a null pointer" + raise ArgumentError.new("Cannot create a string with a null pointer") end new(bytesize) do |buffer| From 1637331c71f743d5591d722907696ee3ed3fe226 Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Mon, 3 Aug 2020 04:02:05 -0400 Subject: [PATCH 203/263] Pass through formatter --- src/string.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/string.cr b/src/string.cr index b988743e80d4..7baf1b854a0f 100644 --- a/src/string.cr +++ b/src/string.cr @@ -194,7 +194,7 @@ class String if chars.null? raise ArgumentError.new("Cannot create a string with a null pointer") end - + new(chars, LibC.strlen(chars)) end @@ -212,7 +212,7 @@ class String def self.new(chars : UInt8*, bytesize, size = 0) # Avoid allocating memory for the empty string return "" if bytesize == 0 - + if chars.null? raise ArgumentError.new("Cannot create a string with a null pointer") end From 4853a7fe186d49fd85522e51ebe9743c964dd983 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 3 Aug 2020 10:09:52 -0300 Subject: [PATCH 204/263] Drop deprecated JSON.mapping (#9527) * Drop deprecated JSON.mapping Use github:crystal-lang/json_mapping.cr * Use JSON::SerializableError instead of JSON::MappingError --- spec/std/json/mapping_spec.cr | 640 ----------------------------- spec/std/json/serializable_spec.cr | 14 +- spec/win32_std_spec.cr | 1 - src/json/mapping.cr | 255 ------------ src/json/serialization.cr | 34 +- 5 files changed, 34 insertions(+), 910 deletions(-) delete mode 100644 spec/std/json/mapping_spec.cr delete mode 100644 src/json/mapping.cr diff --git a/spec/std/json/mapping_spec.cr b/spec/std/json/mapping_spec.cr deleted file mode 100644 index d03c9fa3b127..000000000000 --- a/spec/std/json/mapping_spec.cr +++ /dev/null @@ -1,640 +0,0 @@ -require "../spec_helper" -require "json" -require "uuid" -require "uuid/json" -{% unless flag?(:win32) %} - require "big/json" -{% end %} - -private class JSONPerson - JSON.mapping({ - name: {type: String}, - age: {type: Int32, nilable: true}, - }) - - def_equals name, age - - def initialize(@name : String) - end -end - -private class StrictJSONPerson - JSON.mapping({ - name: {type: String}, - age: {type: Int32, nilable: true}, - }, true) -end - -private class JSONPersonEmittingNull - JSON.mapping({ - name: {type: String}, - age: {type: Int32, nilable: true, emit_null: true}, - }) -end - -private class JSONWithBool - JSON.mapping value: Bool -end - -private class JSONWithUUID - JSON.mapping value: UUID -end - -{% unless flag?(:win32) %} - private class JSONWithBigDecimal - JSON.mapping value: BigDecimal - end -{% end %} - -private class JSONWithTime - JSON.mapping({ - value: {type: Time, converter: Time::Format.new("%F %T")}, - }) -end - -private class JSONWithNilableTime - JSON.mapping({ - value: {type: Time, nilable: true, converter: Time::Format.new("%F")}, - }) - - def initialize - end -end - -private class JSONWithNilableTimeEmittingNull - JSON.mapping({ - value: {type: Time, nilable: true, converter: Time::Format.new("%F"), emit_null: true}, - }) - - def initialize - end -end - -private class JSONWithPropertiesKey - JSON.mapping( - properties: Hash(String, String), - ) -end - -private class JSONWithSimpleMapping - JSON.mapping({name: String, age: Int32}) -end - -private class JSONWithKeywordsMapping - JSON.mapping({end: Int32, abstract: Int32}) -end - -private class JSONWithAny - JSON.mapping({name: String, any: JSON::Any}) -end - -private class JsonWithProblematicKeys - JSON.mapping({ - key: Int32, - pull: Int32, - }) -end - -private class JsonWithSet - JSON.mapping({set: Set(String)}) -end - -private class JsonWithDefaults - JSON.mapping({ - a: {type: Int32, default: 11}, - b: {type: String, default: "Haha"}, - c: {type: Bool, default: true}, - d: {type: Bool, default: false}, - e: {type: Bool, nilable: true, default: false}, - f: {type: Int32, nilable: true, default: 1}, - g: {type: Int32, nilable: true, default: nil}, - h: {type: Array(Int32), default: [1, 2, 3]}, - }) -end - -private class JSONWithSmallIntegers - JSON.mapping({ - foo: Int16, - bar: Int8, - }) -end - -private class JSONWithTimeEpoch - JSON.mapping({ - value: {type: Time, converter: Time::EpochConverter}, - }) -end - -private class JSONWithTimeEpochMillis - JSON.mapping({ - value: {type: Time, converter: Time::EpochMillisConverter}, - }) -end - -private class JSONWithRaw - JSON.mapping({ - value: {type: String, converter: String::RawConverter}, - }) -end - -private class JSONWithArrayConverter - JSON.mapping({ - values: {type: Array(Time), converter: JSON::ArrayConverter(Time::EpochConverter)}, - }) -end - -private class JSONWithJSONHashValueConverter - JSON.mapping({ - birthdays: {type: Hash(String, Time), converter: JSON::HashValueConverter(Time::EpochConverter)}, - }) -end - -private class JSONWithRoot - JSON.mapping({ - result: {type: Array(JSONPerson), root: "heroes"}, - }) -end - -private class JSONWithNilableRoot - JSON.mapping({ - result: {type: Array(JSONPerson), root: "heroes", nilable: true}, - }) -end - -private class JSONWithNilableRootEmitNull - JSON.mapping({ - result: {type: Array(JSONPerson), root: "heroes", nilable: true, emit_null: true}, - }) -end - -private class JSONWithNilableUnion - JSON.mapping({ - value: Int32 | Nil, - }) -end - -private class JSONWithNilableUnion2 - JSON.mapping({ - value: Int32?, - }) -end - -private class JSONWithPresence - JSON.mapping({ - first_name: {type: String?, presence: true, nilable: true}, - last_name: {type: String?, presence: true, nilable: true}, - }) -end - -private class JSONWithQueryAttributes - JSON.mapping({ - foo?: Bool, - bar?: {type: Bool, default: false, presence: true, key: "is_bar"}, - }) -end - -private class JSONWithOverwritingQueryAttributes - property foo : Symbol? - property bar : Symbol? - JSON.mapping({ - foo?: Bool, - bar?: {type: Bool, default: false, presence: true, key: "is_bar"}, - }) -end - -describe "JSON mapping" do - it "parses person" do - person = JSONPerson.from_json(%({"name": "John", "age": 30})) - person.should be_a(JSONPerson) - person.name.should eq("John") - person.age.should eq(30) - end - - it "parses person without age" do - person = JSONPerson.from_json(%({"name": "John"})) - person.should be_a(JSONPerson) - person.name.should eq("John") - person.name.size.should eq(4) # This verifies that name is not nilable - person.age.should be_nil - end - - it "parses array of people" do - people = Array(JSONPerson).from_json(%([{"name": "John"}, {"name": "Doe"}])) - people.size.should eq(2) - end - - it "does to_json" do - person = JSONPerson.from_json(%({"name": "John", "age": 30})) - person2 = JSONPerson.from_json(person.to_json) - person2.should eq(person) - end - - it "parses person with unknown attributes" do - person = JSONPerson.from_json(%({"name": "John", "age": 30, "foo": "bar"})) - person.should be_a(JSONPerson) - person.name.should eq("John") - person.age.should eq(30) - end - - it "parses strict person with unknown attributes" do - error_message = <<-'MSG' - Unknown JSON attribute: foo - parsing StrictJSONPerson - MSG - ex = expect_raises JSON::MappingError, error_message do - StrictJSONPerson.from_json <<-JSON - { - "name": "John", - "age": 30, - "foo": "bar" - } - JSON - end - ex.location.should eq({4, 3}) - end - - it "raises if non-nilable attribute is nil" do - error_message = <<-'MSG' - Missing JSON attribute: name - parsing JSONPerson at 1:1 - MSG - ex = expect_raises JSON::MappingError, error_message do - JSONPerson.from_json(%({"age": 30})) - end - ex.location.should eq({1, 1}) - end - - it "raises if not an object" do - error_message = <<-'MSG' - Expected BeginObject but was String at 1:1 - parsing StrictJSONPerson at 0:0 - MSG - ex = expect_raises JSON::MappingError, error_message do - StrictJSONPerson.from_json <<-JSON - "foo" - JSON - end - ex.location.should eq({1, 1}) - end - - it "raises if data type does not match" do - error_message = <<-'MSG' - Expected Int but was String at 3:15 - parsing StrictJSONPerson#age at 3:3 - MSG - ex = expect_raises JSON::MappingError, error_message do - StrictJSONPerson.from_json <<-JSON - { - "name": "John", - "age": "foo", - "foo": "bar" - } - JSON - end - ex.location.should eq({3, 15}) - end - - it "doesn't emit null by default when doing to_json" do - person = JSONPerson.from_json(%({"name": "John"})) - (person.to_json =~ /age/).should be_falsey - end - - it "emits null on request when doing to_json" do - person = JSONPersonEmittingNull.from_json(%({"name": "John"})) - (person.to_json =~ /age/).should be_truthy - end - - it "doesn't raises on false value when not-nil" do - json = JSONWithBool.from_json(%({"value": false})) - json.value.should be_false - end - - it "parses UUID" do - uuid = JSONWithUUID.from_json(%({"value": "ba714f86-cac6-42c7-8956-bcf5105e1b81"})) - uuid.should be_a(JSONWithUUID) - uuid.value.should eq(UUID.new("ba714f86-cac6-42c7-8956-bcf5105e1b81")) - end - - it "parses json with Time::Format converter" do - json = JSONWithTime.from_json(%({"value": "2014-10-31 23:37:16"})) - json.value.should be_a(Time) - json.value.to_s.should eq("2014-10-31 23:37:16 UTC") - json.to_json.should eq(%({"value":"2014-10-31 23:37:16"})) - end - - it "allows setting a nilable property to nil" do - person = JSONPerson.new("John") - person.age = 1 - person.age = nil - end - - it "parses simple mapping" do - person = JSONWithSimpleMapping.from_json(%({"name": "John", "age": 30})) - person.should be_a(JSONWithSimpleMapping) - person.name.should eq("John") - person.age.should eq(30) - end - - it "outputs with converter when nilable" do - json = JSONWithNilableTime.new - json.to_json.should eq("{}") - end - - it "outputs with converter when nilable when emit_null is true" do - json = JSONWithNilableTimeEmittingNull.new - json.to_json.should eq(%({"value":null})) - end - - it "outputs JSON with properties key" do - string = %({"properties":{"foo":"bar"}}) - json = JSONWithPropertiesKey.from_json(string) - json.to_json.should eq(string) - end - - it "parses json with keywords" do - json = JSONWithKeywordsMapping.from_json(%({"end": 1, "abstract": 2})) - json.end.should eq(1) - json.abstract.should eq(2) - end - - it "parses json with any" do - json = JSONWithAny.from_json(%({"name": "Hi", "any": [{"x": 1}, 2, "hey", true, false, 1.5, null]})) - json.name.should eq("Hi") - json.any.raw.should eq([{"x" => 1}, 2, "hey", true, false, 1.5, nil]) - json.to_json.should eq(%({"name":"Hi","any":[{"x":1},2,"hey",true,false,1.5,null]})) - end - - it "parses json with problematic keys" do - json = JsonWithProblematicKeys.from_json(%({"key": 1, "pull": 2})) - json.key.should eq(1) - json.pull.should eq(2) - end - - it "parses json array as set" do - json = JsonWithSet.from_json(%({"set": ["a", "a", "b"]})) - json.set.should eq(Set(String){"a", "b"}) - end - - it "allows small types of integer" do - json = JSONWithSmallIntegers.from_json(%({"foo": 23, "bar": 7})) - - json.foo.should eq(23) - typeof(json.foo).should eq(Int16) - - json.bar.should eq(7) - typeof(json.bar).should eq(Int8) - end - - describe "parses json with defaults" do - it "mixed" do - json = JsonWithDefaults.from_json(%({"a":1,"b":"bla"})) - json.a.should eq 1 - json.b.should eq "bla" - - json = JsonWithDefaults.from_json(%({"a":1})) - json.a.should eq 1 - json.b.should eq "Haha" - - json = JsonWithDefaults.from_json(%({"b":"bla"})) - json.a.should eq 11 - json.b.should eq "bla" - - json = JsonWithDefaults.from_json(%({})) - json.a.should eq 11 - json.b.should eq "Haha" - - json = JsonWithDefaults.from_json(%({"a":null,"b":null})) - json.a.should eq 11 - json.b.should eq "Haha" - end - - it "bool" do - json = JsonWithDefaults.from_json(%({})) - json.c.should eq true - typeof(json.c).should eq Bool - json.d.should eq false - typeof(json.d).should eq Bool - - json = JsonWithDefaults.from_json(%({"c":false})) - json.c.should eq false - json = JsonWithDefaults.from_json(%({"c":true})) - json.c.should eq true - - json = JsonWithDefaults.from_json(%({"d":false})) - json.d.should eq false - json = JsonWithDefaults.from_json(%({"d":true})) - json.d.should eq true - end - - it "with nilable" do - json = JsonWithDefaults.from_json(%({})) - - json.e.should eq false - typeof(json.e).should eq(Bool | Nil) - - json.f.should eq 1 - typeof(json.f).should eq(Int32 | Nil) - - json.g.should eq nil - typeof(json.g).should eq(Int32 | Nil) - - json = JsonWithDefaults.from_json(%({"e":false})) - json.e.should eq false - json = JsonWithDefaults.from_json(%({"e":true})) - json.e.should eq true - end - - it "create new array every time" do - json = JsonWithDefaults.from_json(%({})) - json.h.should eq [1, 2, 3] - json.h << 4 - json.h.should eq [1, 2, 3, 4] - - json = JsonWithDefaults.from_json(%({})) - json.h.should eq [1, 2, 3] - end - end - - it "uses Time::EpochConverter" do - string = %({"value":1459859781}) - json = JSONWithTimeEpoch.from_json(string) - json.value.should be_a(Time) - json.value.should eq(Time.unix(1459859781)) - json.to_json.should eq(string) - end - - it "uses Time::EpochMillisConverter" do - string = %({"value":1459860483856}) - json = JSONWithTimeEpochMillis.from_json(string) - json.value.should be_a(Time) - json.value.should eq(Time.unix_ms(1459860483856)) - json.to_json.should eq(string) - end - - it "uses JSON::ArrayConverter" do - string = %({"values":[1459859781,1567628762]}) - json = JSONWithArrayConverter.from_json(string) - json.values.should be_a(Array(Time)) - json.values.should eq([Time.unix(1459859781), Time.unix(1567628762)]) - json.to_json.should eq(string) - end - - it "uses JSON::HashValueConverter" do - string = %({"birthdays":{"foo":1459859781,"bar":1567628762}}) - json = JSONWithJSONHashValueConverter.from_json(string) - json.birthdays.should be_a(Hash(String, Time)) - json.birthdays.should eq({"foo" => Time.unix(1459859781), "bar" => Time.unix(1567628762)}) - json.to_json.should eq(string) - end - - it "parses raw value from int" do - string = %({"value":123456789123456789123456789123456789}) - json = JSONWithRaw.from_json(string) - json.value.should eq("123456789123456789123456789123456789") - json.to_json.should eq(string) - end - - it "parses raw value from float" do - string = %({"value":123456789123456789.123456789123456789}) - json = JSONWithRaw.from_json(string) - json.value.should eq("123456789123456789.123456789123456789") - json.to_json.should eq(string) - end - - it "parses raw value from object" do - string = %({"value":[null,true,false,{"x":[1,1.5]}]}) - json = JSONWithRaw.from_json(string) - json.value.should eq(%([null,true,false,{"x":[1,1.5]}])) - json.to_json.should eq(string) - end - - it "parses with root" do - json = %({"result":{"heroes":[{"name":"Batman"}]}}) - result = JSONWithRoot.from_json(json) - result.result.should be_a(Array(JSONPerson)) - result.result.first.name.should eq "Batman" - result.to_json.should eq(json) - end - - it "parses with nilable root" do - json = %({"result":null}) - result = JSONWithNilableRoot.from_json(json) - result.result.should be_nil - result.to_json.should eq("{}") - end - - it "parses with nilable root and emit null" do - json = %({"result":null}) - result = JSONWithNilableRootEmitNull.from_json(json) - result.result.should be_nil - result.to_json.should eq(json) - end - - it "parses nilable union" do - obj = JSONWithNilableUnion.from_json(%({"value": 1})) - obj.value.should eq(1) - obj.to_json.should eq(%({"value":1})) - - obj = JSONWithNilableUnion.from_json(%({"value": null})) - obj.value.should be_nil - obj.to_json.should eq(%({})) - - obj = JSONWithNilableUnion.from_json(%({})) - obj.value.should be_nil - obj.to_json.should eq(%({})) - end - - it "parses nilable union2" do - obj = JSONWithNilableUnion2.from_json(%({"value": 1})) - obj.value.should eq(1) - obj.to_json.should eq(%({"value":1})) - - obj = JSONWithNilableUnion2.from_json(%({"value": null})) - obj.value.should be_nil - obj.to_json.should eq(%({})) - - obj = JSONWithNilableUnion2.from_json(%({})) - obj.value.should be_nil - obj.to_json.should eq(%({})) - end - - describe "parses JSON with presence markers" do - it "parses person with absent attributes" do - json = JSONWithPresence.from_json(%({"first_name": null})) - json.first_name.should be_nil - json.first_name_present?.should be_true - json.last_name.should be_nil - json.last_name_present?.should be_false - end - end - - describe "with query attributes" do - it "defines query getter" do - json = JSONWithQueryAttributes.from_json(%({"foo": true})) - json.foo?.should be_true - json.bar?.should be_false - end - - it "defines query getter with class restriction" do - {% begin %} - {% methods = JSONWithQueryAttributes.methods %} - {{ methods.find(&.name.==("foo?")).return_type }}.should eq(Bool) - {{ methods.find(&.name.==("bar?")).return_type }}.should eq(Bool) - {% end %} - end - - it "defines non-query setter and presence methods" do - json = JSONWithQueryAttributes.from_json(%({"foo": false})) - json.bar_present?.should be_false - json.bar = true - json.bar?.should be_true - end - - it "maps non-query attributes" do - json = JSONWithQueryAttributes.from_json(%({"foo": false, "is_bar": false})) - json.bar_present?.should be_true - json.bar?.should be_false - json.bar = true - json.to_json.should eq(%({"foo":false,"is_bar":true})) - end - - it "raises if non-nilable attribute is nil" do - error_message = <<-'MSG' - Missing JSON attribute: foo - parsing JSONWithQueryAttributes at 1:1 - MSG - ex = expect_raises JSON::MappingError, error_message do - JSONWithQueryAttributes.from_json(%({"is_bar": true})) - end - ex.location.should eq({1, 1}) - end - - it "overwrites non-query attributes" do - json = JSONWithOverwritingQueryAttributes.from_json(%({"foo": true})) - typeof(json.@foo).should eq(Bool) - typeof(json.@bar).should eq(Bool) - end - end - - pending_win32 describe: "BigDecimal" do - it "parses json string with BigDecimal" do - json = JSONWithBigDecimal.from_json(%({"value": "10.05"})) - json.value.should eq(BigDecimal.new("10.05")) - end - - it "parses large json ints with BigDecimal" do - json = JSONWithBigDecimal.from_json(%({"value": 9223372036854775808})) - json.value.should eq(BigDecimal.new("9223372036854775808")) - end - - it "parses json float with BigDecimal" do - json = JSONWithBigDecimal.from_json(%({"value": 10.05})) - json.value.should eq(BigDecimal.new("10.05")) - end - - it "parses large precision json floats with BigDecimal" do - json = JSONWithBigDecimal.from_json(%({"value": 0.00045808999999999997})) - json.value.should eq(BigDecimal.new("0.00045808999999999997")) - end - end -end diff --git a/spec/std/json/serializable_spec.cr b/spec/std/json/serializable_spec.cr index 9349c16672f5..3ba818df4c85 100644 --- a/spec/std/json/serializable_spec.cr +++ b/spec/std/json/serializable_spec.cr @@ -411,7 +411,7 @@ describe "JSON mapping" do Unknown JSON attribute: foo parsing StrictJSONAttrPerson MSG - ex = expect_raises JSON::MappingError, error_message do + ex = expect_raises ::JSON::SerializableError, error_message do StrictJSONAttrPerson.from_json <<-JSON { "name": "John", @@ -443,7 +443,7 @@ describe "JSON mapping" do Missing JSON attribute: name parsing JSONAttrPerson at 1:1 MSG - ex = expect_raises JSON::MappingError, error_message do + ex = expect_raises ::JSON::SerializableError, error_message do JSONAttrPerson.from_json(%({"age": 30})) end ex.location.should eq({1, 1}) @@ -454,7 +454,7 @@ describe "JSON mapping" do Expected BeginObject but was String at 1:1 parsing StrictJSONAttrPerson at 0:0 MSG - ex = expect_raises JSON::MappingError, error_message do + ex = expect_raises ::JSON::SerializableError, error_message do StrictJSONAttrPerson.from_json <<-JSON "foo" JSON @@ -466,7 +466,7 @@ describe "JSON mapping" do error_message = <<-MSG Couldn't parse (Int32 | Nil) from "foo" at 3:10 MSG - ex = expect_raises JSON::MappingError, error_message do + ex = expect_raises ::JSON::SerializableError, error_message do StrictJSONAttrPerson.from_json <<-JSON { "name": "John", @@ -778,7 +778,7 @@ describe "JSON mapping" do Missing JSON attribute: foo parsing JSONAttrWithQueryAttributes at 1:1 MSG - ex = expect_raises JSON::MappingError, error_message do + ex = expect_raises ::JSON::SerializableError, error_message do JSONAttrWithQueryAttributes.from_json(%({"is_bar": true})) end ex.location.should eq({1, 1}) @@ -847,13 +847,13 @@ describe "JSON mapping" do end it "raises if missing discriminator" do - expect_raises(JSON::MappingError, "Missing JSON discriminator field 'type'") do + expect_raises(::JSON::SerializableError, "Missing JSON discriminator field 'type'") do JSONShape.from_json("{}") end end it "raises if unknown discriminator value" do - expect_raises(JSON::MappingError, %(Unknown 'type' discriminator value: "unknown")) do + expect_raises(::JSON::SerializableError, %(Unknown 'type' discriminator value: "unknown")) do JSONShape.from_json(%({"type": "unknown"})) end end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index 16e912becd47..2f5fd9df0a99 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -107,7 +107,6 @@ require "./std/iterator_spec.cr" require "./std/json/any_spec.cr" require "./std/json/builder_spec.cr" require "./std/json/lexer_spec.cr" -require "./std/json/mapping_spec.cr" require "./std/json/parser_spec.cr" require "./std/json/pull_parser_spec.cr" require "./std/json/serializable_spec.cr" diff --git a/src/json/mapping.cr b/src/json/mapping.cr deleted file mode 100644 index a33a0529c088..000000000000 --- a/src/json/mapping.cr +++ /dev/null @@ -1,255 +0,0 @@ -module JSON - # The `JSON.mapping` macro defines how an object is mapped to JSON. - # - # ### Example - # - # ``` - # require "json" - # - # class Location - # JSON.mapping( - # lat: Float64, - # lng: Float64, - # ) - # end - # - # class House - # JSON.mapping( - # address: String, - # location: {type: Location, nilable: true}, - # ) - # end - # - # house = House.from_json(%({"address": "Crystal Road 1234", "location": {"lat": 12.3, "lng": 34.5}})) - # house.address # => "Crystal Road 1234" - # house.location # => # - # house.to_json # => %({"address":"Crystal Road 1234","location":{"lat":12.3,"lng":34.5}}) - # - # houses = Array(House).from_json(%([{"address": "Crystal Road 1234", "location": {"lat": 12.3, "lng": 34.5}}])) - # houses.size # => 1 - # houses.to_json # => %([{"address":"Crystal Road 1234","location":{"lat":12.3,"lng":34.5}}]) - # ``` - # - # ### Usage - # - # `JSON.mapping` must receive a series of named arguments, or a named tuple literal, or a hash literal, - # whose keys will define Crystal properties. - # - # The value of each key can be a type. Primitive types (numbers, string, boolean and nil) - # are supported, as well as custom objects which use `JSON.mapping` or define a `new` method - # that accepts a `JSON::PullParser` and returns an object from it. Union types are supported, - # if multiple types in the union can be mapped from the JSON, it is undefined which one will be chosen. - # - # The value can also be another hash literal with the following options: - # * **type**: (required) the type described above (you can use `JSON::Any` too) - # * **key**: the property name in the JSON document (as opposed to the property name in the Crystal code) - # * **nilable**: if `true`, the property can be `Nil`. Passing `T?` as a type has the same effect. - # * **default**: value to use if the property is missing in the JSON document, or if it's `null` and `nilable` was not set to `true`. If the default value creates a new instance of an object (for example `[1, 2, 3]` or `SomeObject.new`), a different instance will be used each time a JSON document is parsed. - # * **emit_null**: if `true`, emits a `null` value for nilable properties (by default nulls are not emitted) - # * **converter**: specify an alternate type for parsing and generation. The converter must define `from_json(JSON::PullParser)` and `to_json(value, JSON::Builder)` as class methods. Examples of converters are `Time::Format` and `Time::EpochConverter` for `Time`. - # * **root**: assume the value is inside a JSON object with a given key (see `Object.from_json(string_or_io, root)`) - # * **setter**: if `true`, will generate a setter for the variable, `true` by default - # * **getter**: if `true`, will generate a getter for the variable, `true` by default - # * **presence**: if `true`, a `{{key}}_present?` method will be generated when the key was present (even if it has a `null` value), `false` by default - # - # This macro by default defines getters and setters for each variable (this can be overrided with *setter* and *getter*). - # The mapping doesn't define a constructor accepting these variables as arguments, but you can provide an overload. - # - # The macro basically defines a constructor accepting a `JSON::PullParser` that reads from - # it and initializes this type's instance variables. It also defines a `to_json(JSON::Builder)` method - # by invoking `to_json(JSON::Builder)` on each of the properties (unless a converter is specified, in - # which case `to_json(value, JSON::Builder)` is invoked). - # - # This macro also declares instance variables of the types given in the mapping. - # - # If *strict* is `true`, unknown properties in the JSON - # document will raise a parse exception. The default is `false`, so unknown properties - # are silently ignored. - @[Deprecated("use JSON::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/json_mapping.cr)")] - macro mapping(_properties_, strict = false) - {% for key, value in _properties_ %} - {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} - {% end %} - - {% for key, value in _properties_ %} - {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %} - {% end %} - - {% for key, value in _properties_ %} - @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - - {% if value[:setter] == nil ? true : value[:setter] %} - def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}) - @{{value[:key_id]}} = _{{value[:key_id]}} - end - {% end %} - - {% if value[:getter] == nil ? true : value[:getter] %} - def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - @{{value[:key_id]}} - end - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present : Bool = false - - def {{value[:key_id]}}_present? - @{{value[:key_id]}}_present - end - {% end %} - {% end %} - - def initialize(%pull : ::JSON::PullParser) - {% for key, value in _properties_ %} - %var{key.id} = nil - %found{key.id} = false - {% end %} - - %location = %pull.location - begin - %pull.read_begin_object - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) - end - until %pull.kind.end_object? - %key_location = %pull.location - key = %pull.read_object_key - case key - {% for key, value in _properties_ %} - when {{value[:key] || value[:key_id].stringify}} - %found{key.id} = true - begin - %var{key.id} = - {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %} - - {% if value[:root] %} - %pull.on_key!({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - {{value[:converter]}}.from_json(%pull) - {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %} - {{value[:type]}}.new(%pull) - {% else %} - ::Union({{value[:type]}}).new(%pull) - {% end %} - - {% if value[:root] %} - end - {% end %} - - {% if value[:nilable] || value[:default] != nil %} } {% end %} - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc) - end - {% end %} - else - {% if strict %} - raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil) - {% else %} - %pull.skip - {% end %} - end - end - %pull.read_next - - {% for key, value in _properties_ %} - {% unless value[:nilable] || value[:default] != nil %} - if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable? - raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil) - end - {% end %} - - {% if value[:nilable] %} - {% if value[:default] != nil %} - @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : {{value[:default]}} - {% else %} - @{{value[:key_id]}} = %var{key.id} - {% end %} - {% elsif value[:default] != nil %} - @{{value[:key_id]}} = %var{key.id}.nil? ? {{value[:default]}} : %var{key.id} - {% else %} - @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}}) - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present = %found{key.id} - {% end %} - {% end %} - end - - def to_json(json : ::JSON::Builder) - json.object do - {% for key, value in _properties_ %} - _{{value[:key_id]}} = @{{value[:key_id]}} - - {% unless value[:emit_null] %} - unless _{{value[:key_id]}}.nil? - {% end %} - - json.field({{value[:key] || value[:key_id].stringify}}) do - {% if value[:root] %} - {% if value[:emit_null] %} - if _{{value[:key_id]}}.nil? - nil.to_json(json) - else - {% end %} - - json.object do - json.field({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - if _{{value[:key_id]}} - {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json) - else - nil.to_json(json) - end - {% else %} - _{{value[:key_id]}}.to_json(json) - {% end %} - - {% if value[:root] %} - {% if value[:emit_null] %} - end - {% end %} - end - end - {% end %} - end - - {% unless value[:emit_null] %} - end - {% end %} - {% end %} - end - end - end - - # This is a convenience method to allow invoking `JSON.mapping` - # with named arguments instead of with a hash/named-tuple literal. - @[Deprecated("use JSON::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/json_mapping.cr)")] - macro mapping(**_properties_) - ::JSON.mapping({{_properties_}}) - end - - class MappingError < ParseException - getter klass : String - getter attribute : String? - - def initialize(message : String?, @klass : String, @attribute : String?, line_number : Int32, column_number : Int32, cause) - message = String.build do |io| - io << message - io << "\n parsing " - io << klass - if attribute = @attribute - io << '#' << attribute - end - end - super(message, line_number, column_number, cause) - if cause - @line_number, @column_number = cause.location - end - end - end -end diff --git a/src/json/serialization.cr b/src/json/serialization.cr index 3793f60f099d..b2dd18c7ba17 100644 --- a/src/json/serialization.cr +++ b/src/json/serialization.cr @@ -183,7 +183,7 @@ module JSON begin pull.read_begin_object rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) + raise ::JSON::SerializableError.new(exc.message, self.class.to_s, nil, *%location, exc) end until pull.kind.end_object? %key_location = pull.location @@ -212,7 +212,7 @@ module JSON {% if value[:nilable] || value[:has_default] %} } {% end %} rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key]}}, *%key_location, exc) + raise ::JSON::SerializableError.new(exc.message, self.class.to_s, {{value[:key]}}, *%key_location, exc) end {% end %} else @@ -224,7 +224,7 @@ module JSON {% for name, value in properties %} {% unless value[:nilable] || value[:has_default] %} if %var{name}.nil? && !%found{name} && !::Union({{value[:type]}}).nilable? - raise ::JSON::MappingError.new("Missing JSON attribute: {{value[:key].id}}", self.class.to_s, nil, *%location, nil) + raise ::JSON::SerializableError.new("Missing JSON attribute: {{value[:key].id}}", self.class.to_s, nil, *%location, nil) end {% end %} @@ -329,7 +329,7 @@ module JSON module Strict protected def on_unknown_json_attribute(pull, key, key_location) - raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *key_location, nil) + raise ::JSON::SerializableError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *key_location, nil) end end @@ -341,7 +341,7 @@ module JSON json_unmapped[key] = begin JSON::Any.new(pull) rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, key, *key_location, exc) + raise ::JSON::SerializableError.new(exc.message, self.class.to_s, key, *key_location, exc) end end @@ -414,7 +414,7 @@ module JSON end unless discriminator_value - raise ::JSON::MappingError.new("Missing JSON discriminator field '{{field.id}}'", to_s, nil, *location, nil) + raise ::JSON::SerializableError.new("Missing JSON discriminator field '{{field.id}}'", to_s, nil, *location, nil) end case discriminator_value @@ -423,9 +423,29 @@ module JSON {{value.id}}.from_json(json) {% end %} else - raise ::JSON::MappingError.new("Unknown '{{field.id}}' discriminator value: #{discriminator_value.inspect}", to_s, nil, *location, nil) + raise ::JSON::SerializableError.new("Unknown '{{field.id}}' discriminator value: #{discriminator_value.inspect}", to_s, nil, *location, nil) end end end end + + class SerializableError < ParseException + getter klass : String + getter attribute : String? + + def initialize(message : String?, @klass : String, @attribute : String?, line_number : Int32, column_number : Int32, cause) + message = String.build do |io| + io << message + io << "\n parsing " + io << klass + if attribute = @attribute + io << '#' << attribute + end + end + super(message, line_number, column_number, cause) + if cause + @line_number, @column_number = cause.location + end + end + end end From 7662698616841e43a5a1caa537f9ff5f693d1a03 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 3 Aug 2020 10:10:10 -0300 Subject: [PATCH 205/263] Drop deprecated YAML.mapping (#9526) * Drop deprecated YAML.mapping Use github:crystal-lang/yaml_mapping.cr * Remove entry in win32_std_spec.cr --- spec/std/yaml/mapping_spec.cr | 561 ---------------------------------- spec/win32_std_spec.cr | 1 - src/yaml/mapping.cr | 225 -------------- 3 files changed, 787 deletions(-) delete mode 100644 spec/std/yaml/mapping_spec.cr delete mode 100644 src/yaml/mapping.cr diff --git a/spec/std/yaml/mapping_spec.cr b/spec/std/yaml/mapping_spec.cr deleted file mode 100644 index 95920ce88e60..000000000000 --- a/spec/std/yaml/mapping_spec.cr +++ /dev/null @@ -1,561 +0,0 @@ -require "spec" -require "yaml" -require "../../support/finalize" - -private class YAMLPerson - YAML.mapping({ - name: String, - age: {type: Int32, nilable: true}, - }) - - def_equals name, age - - def initialize(@name : String) - end -end - -private class StrictYAMLPerson - YAML.mapping({ - name: {type: String}, - age: {type: Int32, nilable: true}, - }, true) -end - -private class YAMLWithBool - YAML.mapping value: Bool -end - -private class YAMLWithTime - YAML.mapping({ - value: {type: Time, converter: Time::Format.new("%F %T")}, - }) -end - -private class YAMLWithKey - YAML.mapping({ - key: String, - value: Int32, - pull: Int32, - }) -end - -private class YAMLWithPropertiesKey - YAML.mapping( - properties: Hash(String, String), - ) -end - -private class YAMLWithDefaults - YAML.mapping({ - a: {type: Int32, default: 11}, - b: {type: String, default: "Haha"}, - c: {type: Bool, default: true}, - d: {type: Bool, default: false}, - e: {type: Bool, nilable: true, default: false}, - f: {type: Int32, nilable: true, default: 1}, - g: {type: Int32, nilable: true, default: nil}, - h: {type: Array(Int32), default: [1, 2, 3]}, - i: String?, - }) -end - -private class YAMLWithAny - YAML.mapping({ - obj: YAML::Any, - }) - - def initialize(@obj) - end -end - -private class YAMLWithSmallIntegers - YAML.mapping({ - foo: Int16, - bar: Int8, - }) -end - -private class YAMLWithNilableTime - YAML.mapping({ - value: {type: Time, nilable: true, converter: Time::Format.new("%F")}, - }) - - def initialize - end -end - -private class YAMLWithTimeEpoch - YAML.mapping({ - value: {type: Time, converter: Time::EpochConverter}, - }) -end - -private class YAMLWithTimeEpochMillis - YAML.mapping({ - value: {type: Time, converter: Time::EpochMillisConverter}, - }) -end - -private class YAMLWithArrayConverter - YAML.mapping({ - values: {type: Array(Time), converter: YAML::ArrayConverter(Time::EpochConverter)}, - }) -end - -private class YAMLWithPresence - YAML.mapping({ - first_name: {type: String?, presence: true, nilable: true}, - last_name: {type: String?, presence: true, nilable: true}, - }) -end - -private class YAMLWithString - YAML.mapping({ - value: String, - }) -end - -class YAMLRecursive - YAML.mapping({ - name: String, - other: YAMLRecursive, - }) -end - -class YAMLRecursiveNilable - YAML.mapping({ - name: String, - other: YAMLRecursiveNilable?, - }) -end - -class YAMLRecursiveArray - YAML.mapping({ - name: String, - other: Array(YAMLRecursiveArray), - }) -end - -class YAMLRecursiveHash - YAML.mapping({ - name: String, - other: Hash(String, YAMLRecursiveHash), - }) -end - -private class YAMLWithQueryAttributes - YAML.mapping({ - foo?: Bool, - bar?: {type: Bool, default: false, presence: true, key: "is_bar"}, - }) -end - -private class YAMLWithOverwritingQueryAttributes - property foo : Symbol? - property bar : Symbol? - YAML.mapping({ - foo?: Bool, - bar?: {type: Bool, default: false, presence: true, key: "is_bar"}, - }) -end - -private class YAMLWithFinalize - YAML.mapping({ - value: YAML::Any, - }) - - property key : Symbol? - - def finalize - if key = self.key - State.inc(key) - end - end -end - -describe "YAML mapping" do - it "parses person" do - person = YAMLPerson.from_yaml("---\nname: John\nage: 30\n") - person.should be_a(YAMLPerson) - person.name.should eq("John") - person.age.should eq(30) - end - - it "parses person without age" do - person = YAMLPerson.from_yaml("---\nname: John\n") - person.should be_a(YAMLPerson) - person.name.should eq("John") - person.name.size.should eq(4) # This verifies that name is not nilable - person.age.should be_nil - end - - it "parses person with blank age" do - person = YAMLPerson.from_yaml("---\nname: John\nage:\n") - person.should be_a(YAMLPerson) - person.name.should eq("John") - person.name.size.should eq(4) # This verifies that name is not nilable - person.age.should be_nil - end - - it "parses array of people" do - people = Array(YAMLPerson).from_yaml("---\n- name: John\n- name: Doe\n") - people.size.should eq(2) - people[0].name.should eq("John") - people[1].name.should eq("Doe") - end - - it "parses array of people with merge" do - yaml = <<-YAML - - &1 - name: foo - age: 1 - - - <<: *1 - age: 2 - YAML - - people = Array(YAMLPerson).from_yaml(yaml) - people[1].name.should eq("foo") - people[1].age.should eq(2) - end - - it "parses array of people with merge, doesn't hang on infinite recursion" do - yaml = <<-YAML - - &1 - name: foo - <<: *1 - <<: [ *1, *1 ] - age: 1 - YAML - - people = Array(YAMLPerson).from_yaml(yaml) - people[0].name.should eq("foo") - people[0].age.should eq(1) - end - - it "parses person with unknown attributes" do - person = YAMLPerson.from_yaml("---\nname: John\nunknown: [1, 2, 3]\nage: 30\n") - person.should be_a(YAMLPerson) - person.name.should eq("John") - person.age.should eq(30) - end - - it "parses strict person with unknown attributes" do - ex = expect_raises YAML::ParseException, "Unknown yaml attribute: foo" do - StrictYAMLPerson.from_yaml <<-YAML - --- - name: John - foo: [1, 2, 3] - age: 30 - YAML - end - ex.location.should eq({3, 1}) - end - - it "does to_yaml" do - person = YAMLPerson.from_yaml("---\nname: John\nage: 30\n") - person2 = YAMLPerson.from_yaml(person.to_yaml) - person2.should eq(person) - end - - it "doesn't emit null when doing to_yaml" do - person = YAMLPerson.from_yaml("---\nname: John\n") - (person.to_yaml =~ /age/).should be_falsey - end - - it "raises if non-nilable attribute is nil" do - ex = expect_raises YAML::ParseException, "Missing yaml attribute: name" do - YAMLPerson.from_yaml <<-YAML - --- - age: 30 - YAML - end - ex.location.should eq({2, 1}) - end - - it "doesn't raises on false value when not-nil" do - yaml = YAMLWithBool.from_yaml("---\nvalue: false\n") - yaml.value.should be_false - end - - it "parses yaml with Time::Format converter" do - yaml = YAMLWithTime.from_yaml("---\nvalue: 2014-10-31 23:37:16\n") - yaml.value.should be_a(Time) - yaml.value.to_s.should eq("2014-10-31 23:37:16 UTC") - yaml.value.should eq(Time.utc(2014, 10, 31, 23, 37, 16)) - yaml.to_yaml.should eq("---\nvalue: 2014-10-31 23:37:16\n") - end - - it "parses YAML with mapping key named 'key'" do - yaml = YAMLWithKey.from_yaml("---\nkey: foo\nvalue: 1\npull: 2") - yaml.key.should eq("foo") - yaml.value.should eq(1) - yaml.pull.should eq(2) - end - - it "outputs YAML with properties key" do - input = { - properties: {"foo" => "bar"}, - }.to_yaml - yaml = YAMLWithPropertiesKey.from_yaml(input) - yaml.to_yaml.should eq(input) - end - - it "allows small types of integer" do - yaml = YAMLWithSmallIntegers.from_yaml(%({"foo": 21, "bar": 7})) - - yaml.foo.should eq(21) - typeof(yaml.foo).should eq(Int16) - - yaml.bar.should eq(7) - typeof(yaml.bar).should eq(Int8) - end - - it "parses recursive" do - yaml = <<-YAML - --- &1 - name: foo - other: *1 - YAML - - rec = YAMLRecursive.from_yaml(yaml) - rec.name.should eq("foo") - rec.other.should be(rec) - end - - it "parses recursive nilable (1)" do - yaml = <<-YAML - --- &1 - name: foo - other: *1 - YAML - - rec = YAMLRecursiveNilable.from_yaml(yaml) - rec.name.should eq("foo") - rec.other.should be(rec) - end - - it "parses recursive nilable (2)" do - yaml = <<-YAML - --- &1 - name: foo - YAML - - rec = YAMLRecursiveNilable.from_yaml(yaml) - rec.name.should eq("foo") - rec.other.should be_nil - end - - it "parses recursive array" do - yaml = <<-YAML - --- - name: foo - other: &1 - - name: bar - other: *1 - YAML - - rec = YAMLRecursiveArray.from_yaml(yaml) - rec.other[0].other.should be(rec.other) - end - - it "parses recursive hash" do - yaml = <<-YAML - --- - name: foo - other: &1 - foo: - name: bar - other: *1 - YAML - - rec = YAMLRecursiveHash.from_yaml(yaml) - rec.other["foo"].other.should be(rec.other) - end - - describe "parses YAML with defaults" do - it "mixed" do - json = YAMLWithDefaults.from_yaml(%({"a":1,"b":"bla"})) - json.a.should eq 1 - json.b.should eq "bla" - - json = YAMLWithDefaults.from_yaml(%({"a":1})) - json.a.should eq 1 - json.b.should eq "Haha" - - json = YAMLWithDefaults.from_yaml(%({"b":"bla"})) - json.a.should eq 11 - json.b.should eq "bla" - - json = YAMLWithDefaults.from_yaml(%({})) - json.a.should eq 11 - json.b.should eq "Haha" - end - - it "mixes with all defaults (#2873)" do - yaml = YAMLWithDefaults.from_yaml("") - yaml.a.should eq 11 - yaml.b.should eq "Haha" - end - - it "raises when not a mapping or empty scalar" do - expect_raises(YAML::ParseException) do - YAMLWithDefaults.from_yaml("1") - end - - expect_raises(YAML::ParseException) do - YAMLWithDefaults.from_yaml("[1]") - end - end - - it "bool" do - json = YAMLWithDefaults.from_yaml(%({})) - json.c.should eq true - typeof(json.c).should eq Bool - json.d.should eq false - typeof(json.d).should eq Bool - - json = YAMLWithDefaults.from_yaml(%({"c":false})) - json.c.should eq false - json = YAMLWithDefaults.from_yaml(%({"c":true})) - json.c.should eq true - - json = YAMLWithDefaults.from_yaml(%({"d":false})) - json.d.should eq false - json = YAMLWithDefaults.from_yaml(%({"d":true})) - json.d.should eq true - end - - it "with nilable" do - json = YAMLWithDefaults.from_yaml(%({})) - - json.e.should eq false - typeof(json.e).should eq(Bool | Nil) - - json.f.should eq 1 - typeof(json.f).should eq(Int32 | Nil) - - json.g.should eq nil - typeof(json.g).should eq(Int32 | Nil) - - json = YAMLWithDefaults.from_yaml(%({"e":false})) - json.e.should eq false - json = YAMLWithDefaults.from_yaml(%({"e":true})) - json.e.should eq true - - json = YAMLWithDefaults.from_yaml(%({})) - json.i.should be_nil - - json = YAMLWithDefaults.from_yaml(%({"i":"bla"})) - json.i.should eq("bla") - end - - it "create new array every time" do - json = YAMLWithDefaults.from_yaml(%({})) - json.h.should eq [1, 2, 3] - json.h << 4 - json.h.should eq [1, 2, 3, 4] - - json = YAMLWithDefaults.from_yaml(%({})) - json.h.should eq [1, 2, 3] - end - end - - it "parses YAML with any" do - yaml = YAMLWithAny.from_yaml("obj: hello") - yaml.obj.as_s.should eq("hello") - - yaml = YAMLWithAny.from_yaml({:obj => %w(foo bar)}.to_yaml) - yaml.obj[1].as_s.should eq("bar") - - yaml = YAMLWithAny.from_yaml({:obj => {:foo => :bar}}.to_yaml) - yaml.obj["foo"].as_s.should eq("bar") - end - - it "outputs with converter when nilable" do - yaml = YAMLWithNilableTime.new - yaml.to_yaml.should eq("--- {}\n") - end - - it "uses Time::EpochConverter" do - string = %({"value":1459859781}) - yaml = YAMLWithTimeEpoch.from_yaml(string) - yaml.value.should be_a(Time) - yaml.value.should eq(Time.unix(1459859781)) - yaml.to_yaml.should eq("---\nvalue: 1459859781\n") - end - - it "uses Time::EpochMillisConverter" do - string = %({"value":1459860483856}) - yaml = YAMLWithTimeEpochMillis.from_yaml(string) - yaml.value.should be_a(Time) - yaml.value.should eq(Time.unix_ms(1459860483856)) - yaml.to_yaml.should eq("---\nvalue: 1459860483856\n") - end - - it "uses YAML::ArrayConverter" do - string = %({"values":[1459859781,1567628762]}) - yaml = YAMLWithArrayConverter.from_yaml(string) - yaml.values.should be_a(Array(Time)) - yaml.values.should eq([Time.unix(1459859781), Time.unix(1567628762)]) - yaml.to_yaml.should eq(%(---\nvalues:\n- 1459859781\n- 1567628762\n)) - end - - describe "parses YAML with presence markers" do - it "parses person with absent attributes" do - yaml = YAMLWithPresence.from_yaml("---\nfirst_name:\n") - yaml.first_name.should be_nil - yaml.first_name_present?.should be_true - yaml.last_name.should be_nil - yaml.last_name_present?.should be_false - end - end - - describe "with query attributes" do - it "defines query getter" do - yaml = YAMLWithQueryAttributes.from_yaml(%({"foo": true})) - yaml.foo?.should be_true - yaml.bar?.should be_false - end - - it "defines non-query setter and presence methods" do - yaml = YAMLWithQueryAttributes.from_yaml(%({"foo": false})) - yaml.bar_present?.should be_false - yaml.bar = true - yaml.bar?.should be_true - end - - it "maps non-query attributes" do - yaml = YAMLWithQueryAttributes.from_yaml(%({"foo": false, "is_bar": false})) - yaml.bar_present?.should be_true - yaml.bar?.should be_false - yaml.bar = true - yaml.to_yaml.should eq(%(---\nfoo: false\nis_bar: true\n)) - end - - it "raises if non-nilable attribute is nil" do - ex = expect_raises YAML::ParseException, "Missing yaml attribute: foo" do - YAMLWithQueryAttributes.from_yaml(%({"is_bar": true})) - end - ex.location.should eq({1, 1}) - end - - it "overwrites non-query attributes" do - yaml = YAMLWithOverwritingQueryAttributes.from_yaml(%({"foo": true})) - typeof(yaml.@foo).should eq(Bool) - typeof(yaml.@bar).should eq(Bool) - end - end - - it "calls #finalize" do - assert_finalizes(:yaml) { YAMLWithFinalize.from_yaml("---\nvalue: 1\n") } - end - - it "parses string even if it looks like a number" do - yaml = YAMLWithString.from_yaml <<-YAML - --- - value: 12.34 - YAML - yaml.value.should eq("12.34") - end -end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index 2f5fd9df0a99..864f82dda974 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -225,7 +225,6 @@ require "./std/xml/xml_spec.cr" require "./std/xml/xpath_spec.cr" require "./std/yaml/any_spec.cr" require "./std/yaml/builder_spec.cr" -require "./std/yaml/mapping_spec.cr" require "./std/yaml/nodes/builder_spec.cr" require "./std/yaml/schema/core_spec.cr" require "./std/yaml/schema/fail_safe_spec.cr" diff --git a/src/yaml/mapping.cr b/src/yaml/mapping.cr deleted file mode 100644 index ea85141223e1..000000000000 --- a/src/yaml/mapping.cr +++ /dev/null @@ -1,225 +0,0 @@ -module YAML - # The `YAML.mapping` macro defines how an object is mapped to YAML. - # - # It takes named arguments, a named tuple literal or a hash literal as argument, - # in which attributes and types are defined. - # Once defined, `Object#from_yaml` populates properties of the class from the - # YAML document. - # - # ``` - # require "yaml" - # - # class Employee - # YAML.mapping( - # title: String, - # name: String, - # ) - # end - # - # employee = Employee.from_yaml("title: Manager\nname: John") - # employee.title # => "Manager" - # employee.name # => "John" - # - # employee.name = "Jenny" - # employee.name # => "Jenny" - # ``` - # - # Attributes not mapped with `YAML.mapping` are not defined as properties. - # Also, missing attributes raise a `ParseException`. - # - # ``` - # employee = Employee.from_yaml("title: Manager\nname: John\nage: 30") - # employee.age # undefined method 'age'. (compile error) - # - # Employee.from_yaml("title: Manager") # raises YAML::ParseException - # ``` - # - # You can also define attributes for each property. - # - # ``` - # class Employer - # YAML.mapping( - # title: String, - # name: { - # type: String, - # nilable: true, - # key: "firstname", - # }, - # ) - # end - # ``` - # - # Available attributes: - # - # * *type* (required) defines its type. In the example above, *title: String* is a shortcut to *title: {type: String}*. - # * *nilable* defines if a property can be a `Nil`. Passing `T?` as a type has the same effect. - # * **default**: value to use if the property is missing in the YAML document, or if it's `null` and `nilable` was not set to `true`. If the default value creates a new instance of an object (for example `[1, 2, 3]` or `SomeObject.new`), a different instance will be used each time a YAML document is parsed. - # * *key* defines which key to read from a YAML document. It defaults to the name of the property. - # * *converter* takes an alternate type for parsing. It requires a `#from_yaml` method in that class, and returns an instance of the given type. Examples of converters are `Time::Format` and `Time::EpochConverter` for `Time`. - # * **setter**: if `true`, will generate a setter for the variable, `true` by default - # * **getter**: if `true`, will generate a getter for the variable, `true` by default - # * **presence**: if `true`, a `{{key}}_present?` method will be generated when the key was present (even if it has a `null` value), `false` by default - # - # This macro by default defines getters and setters for each variable (this can be overrided with *setter* and *getter*). - # The mapping doesn't define a constructor accepting these variables as arguments, but you can provide an overload. - # - # The macro basically defines a constructor accepting a `YAML::PullParser` that reads from - # it and initializes this type's instance variables. - # - # This macro also declares instance variables of the types given in the mapping. - @[Deprecated("use YAML::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/yaml_mapping.cr)")] - macro mapping(_properties_, strict = false) - {% for key, value in _properties_ %} - {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} - {% end %} - - {% for key, value in _properties_ %} - {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %} - {% end %} - - {% for key, value in _properties_ %} - @{{value[:key_id]}} : {{value[:type]}} {{ (value[:nilable] ? "?" : "").id }} - - {% if value[:setter] == nil ? true : value[:setter] %} - def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}} {{ (value[:nilable] ? "?" : "").id }}) - @{{value[:key_id]}} = _{{value[:key_id]}} - end - {% end %} - - {% if value[:getter] == nil ? true : value[:getter] %} - def {{key.id}} - @{{value[:key_id]}} - end - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present : Bool = false - - def {{value[:key_id]}}_present? - @{{value[:key_id]}}_present - end - {% end %} - {% end %} - - def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) - ctx.read_alias(node, \{{@type}}) do |obj| - return obj - end - - instance = allocate - - ctx.record_anchor(node, instance) - - instance.initialize(ctx, node, nil) - GC.add_finalizer(instance) if instance.responds_to?(:finalize) - instance - end - - # `new` and `initialize` with just `pull` as an argument collide - # and the compiler just sees the last one. This is why we add a - # dummy argument. - # - # FIXME: remove the dummy argument if we ever fix this. - - def initialize(ctx : YAML::ParseContext, node : ::YAML::Nodes::Node, _dummy : Nil) - {% for key, value in _properties_ %} - %var{key.id} = nil - %found{key.id} = false - {% end %} - - case node - when YAML::Nodes::Mapping - YAML::Schema::Core.each(node) do |key_node, value_node| - unless key_node.is_a?(YAML::Nodes::Scalar) - key_node.raise "Expected scalar as key for mapping" - end - - key = key_node.value - - case key - {% for key, value in _properties_ %} - when {{value[:key] || value[:key_id].stringify}} - %found{key.id} = true - - %var{key.id} = - {% if value[:nilable] || value[:default] != nil %} YAML::Schema::Core.parse_null_or(value_node) { {% end %} - - {% if value[:converter] %} - {{value[:converter]}}.from_yaml(ctx, value_node) - {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %} - {{value[:type]}}.new(ctx, value_node) - {% else %} - ::Union({{value[:type]}}).new(ctx, value_node) - {% end %} - - {% if value[:nilable] || value[:default] != nil %} } {% end %} - {% end %} - else - {% if strict %} - key_node.raise "Unknown yaml attribute: #{key}" - {% end %} - end - end - when YAML::Nodes::Scalar - if node.value.empty? && node.style.plain? && !node.tag - # We consider an empty scalar as an empty mapping - else - node.raise "Expected mapping, not #{node.class}" - end - else - node.raise "Expected mapping, not #{node.class}" - end - - {% for key, value in _properties_ %} - {% unless value[:nilable] || value[:default] != nil %} - if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable? - node.raise "Missing yaml attribute: {{(value[:key] || value[:key_id]).id}}" - end - {% end %} - - {% if value[:nilable] %} - {% if value[:default] != nil %} - @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : {{value[:default]}} - {% else %} - @{{value[:key_id]}} = %var{key.id} - {% end %} - {% elsif value[:default] != nil %} - @{{value[:key_id]}} = %var{key.id}.nil? ? {{value[:default]}} : %var{key.id} - {% else %} - @{{value[:key_id]}} = %var{key.id}.as({{value[:type]}}) - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present = %found{key.id} - {% end %} - {% end %} - end - - def to_yaml(%yaml : ::YAML::Nodes::Builder) - %yaml.mapping(reference: self) do - {% for key, value in _properties_ %} - _{{value[:key_id]}} = @{{value[:key_id]}} - - unless _{{value[:key_id]}}.nil? - # Key - {{value[:key] || value[:key_id].stringify}}.to_yaml(%yaml) - - # Value - {% if value[:converter] %} - {{ value[:converter] }}.to_yaml(_{{value[:key_id]}}, %yaml) - {% else %} - _{{value[:key_id]}}.to_yaml(%yaml) - {% end %} - end - {% end %} - end - end - end - - # This is a convenience method to allow invoking `YAML.mapping` - # with named arguments instead of with a hash/named-tuple literal. - @[Deprecated("use YAML::Serializable instead (the legacy behaviour is also available in a shard at github:crystal-lang/yaml_mapping.cr)")] - macro mapping(**_properties_) - ::YAML.mapping({{_properties_}}) - end -end From cb7b030da4de912e115a9c7e0ad66f7de8549fc7 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Mon, 3 Aug 2020 11:20:06 -0300 Subject: [PATCH 206/263] Drop deprecated logger (#9525) * Drop deprecated logger Use github:crystal-lang/logger.cr * Remove entry in win32_std_spec.cr * Remove entry in docs_main.cr --- spec/std/logger_spec.cr | 96 -------------------- spec/win32_std_spec.cr | 1 - src/docs_main.cr | 1 - src/logger.cr | 192 ---------------------------------------- 4 files changed, 290 deletions(-) delete mode 100644 spec/std/logger_spec.cr delete mode 100644 src/logger.cr diff --git a/spec/std/logger_spec.cr b/spec/std/logger_spec.cr deleted file mode 100644 index d0b147032cb9..000000000000 --- a/spec/std/logger_spec.cr +++ /dev/null @@ -1,96 +0,0 @@ -require "spec" -require "logger" - -describe "Logger" do - it "logs messages" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.debug "debug:skip" - logger.info "info:show" - - logger.level = Logger::DEBUG - logger.debug "debug:show" - - logger.level = Logger::WARN - logger.debug "debug:skip:again" - logger.info "info:skip" - logger.error "error:show" - - r.gets.should match(/info:show/) - r.gets.should match(/debug:show/) - r.gets.should match(/error:show/) - end - end - - it "logs any object" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.info 12345 - - r.gets.should match(/12345/) - end - end - - it "formats message" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.progname = "crystal" - logger.warn "message" - - r.gets(chomp: false).should match(/W, \[.+? #\d+\] WARN -- crystal: message\n/) - end - end - - it "uses custom formatter" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.formatter = Logger::Formatter.new do |severity, datetime, progname, message, io| - io << severity.to_s[0] << ' ' << progname << ": " << message - end - logger.warn "message", "prog" - - r.gets(chomp: false).should eq("W prog: message\n") - end - end - - it "yields message" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.error { "message" } - logger.unknown { "another message" } - - r.gets(chomp: false).should match(/ERROR -- : message\n/) - r.gets(chomp: false).should match(/ ANY -- : another message\n/) - end - end - - it "yields message with progname" do - IO.pipe do |r, w| - logger = Logger.new(w) - logger.error("crystal") { "message" } - logger.unknown("shard") { "another message" } - - r.gets(chomp: false).should match(/ERROR -- crystal: message\n/) - r.gets(chomp: false).should match(/ ANY -- shard: another message\n/) - end - end - - it "can create a logger with nil (#3065)" do - logger = Logger.new(nil) - logger.error("ouch") - end - - it "doesn't yield to the block with nil" do - a = 0 - logger = Logger.new(nil) - logger.info { a = 1 } - a.should eq(0) - end - - it "closes" do - IO.pipe do |r, w| - Logger.new(w).close - w.closed?.should be_true - end - end -end diff --git a/spec/win32_std_spec.cr b/spec/win32_std_spec.cr index 864f82dda974..935236a300af 100644 --- a/spec/win32_std_spec.cr +++ b/spec/win32_std_spec.cr @@ -126,7 +126,6 @@ require "./std/log/env_config_spec.cr" require "./std/log/io_backend_spec.cr" require "./std/log/log_spec.cr" require "./std/log/main_spec.cr" -require "./std/logger_spec.cr" require "./std/match_data_spec.cr" # require "./std/math_spec.cr" (failed linking) require "./std/mime/media_type_spec.cr" diff --git a/src/docs_main.cr b/src/docs_main.cr index 57d80d4fbc85..58a335520065 100644 --- a/src/docs_main.cr +++ b/src/docs_main.cr @@ -17,7 +17,6 @@ require "./http/**" require "./io/**" require "./json" require "./llvm" -require "./logger" require "./macros" require "./math/**" require "./oauth" diff --git a/src/logger.cr b/src/logger.cr deleted file mode 100644 index 99bb513c06ed..000000000000 --- a/src/logger.cr +++ /dev/null @@ -1,192 +0,0 @@ -# The `Logger` class provides a simple but sophisticated logging utility that you can use to output messages. -# -# The messages have associated levels, such as `INFO` or `ERROR` that indicate their importance. -# You can then give the `Logger` a level, and only messages at that level or higher will be printed. -# -# For instance, in a production system, you may have your `Logger` set to `INFO` or even `WARN`. -# When you are developing the system, however, you probably want to know about the program’s internal state, -# and would set the `Logger` to `DEBUG`. -# -# ### Example -# -# ``` -# require "logger" -# -# log = Logger.new(STDOUT) -# log.level = Logger::WARN -# -# # or -# log = Logger.new(STDOUT, level: Logger::WARN) -# -# log.debug("Created logger") -# log.info("Program started") -# log.warn("Nothing to do!") -# -# begin -# File.each_line("/foo/bar.log") do |line| -# unless line =~ /^(\w+) = (.*)$/ -# log.error("Line in wrong format: #{line}") -# end -# end -# rescue err -# log.fatal("Caught exception; exiting") -# log.fatal(err) -# end -# ``` -# -# If logging to multiple locations is required, an `IO::MultiWriter` can be -# used. -# -# ``` -# file = File.new("production.log", "a") -# writer = IO::MultiWriter.new(file, STDOUT) -# -# log = Logger.new(writer) -# log.level = Logger::DEBUG -# log.debug("Created logger") -# ``` -@[Deprecated("Use `Log` module instead")] -class Logger - property level : Severity - property progname : String - - # Customizable `Proc` (with a reasonable default) - # which the `Logger` uses to format and print its entries. - # - # Use this setter to provide a custom formatter. - # The `Logger` will invoke it with the following arguments: - # - severity: a `Logger::Severity` - # - datetime: `Time`, the entry's timestamp - # - progname: `String`, the program name, if set when the logger was built - # - message: `String`, the body of a message - # - io: `IO`, the Logger's stream, to which you must write the final output - # - # Example: - # - # ``` - # require "logger" - # - # logger = Logger.new(STDOUT) - # logger.progname = "YodaBot" - # - # logger.formatter = Logger::Formatter.new do |severity, datetime, progname, message, io| - # label = severity.unknown? ? "ANY" : severity.to_s - # io << label[0] << ", [" << datetime << " #" << Process.pid << "] " - # io << label.rjust(5) << " -- " << progname << ": " << message - # end - # - # logger.warn("Fear leads to anger. Anger leads to hate. Hate leads to suffering.") - # - # # Prints to the console: - # # "W, [2017-05-06 18:00:41 -0300 #11927] WARN -- - # # YodaBot: Fear leads to anger. Anger leads to hate. Hate leads to suffering." - # ``` - property formatter - - # A logger severity level. - enum Severity - # Low-level information for developers - DEBUG - - # Generic (useful) information about system operation - INFO - - # A warning - WARN - - # A handleable error condition - ERROR - - # An unhandleable error that results in a program crash - FATAL - - UNKNOWN - end - - alias Formatter = Severity, Time, String, String, IO -> - - private DEFAULT_FORMATTER = Formatter.new do |severity, datetime, progname, message, io| - label = severity.unknown? ? "ANY" : severity.to_s - io << label[0] << ", [" << datetime << " #" << Process.pid << "] " - io << label.rjust(5) << " -- " << progname << ": " << message - end - - # :nodoc: - record Message, - severity : Severity, - datetime : Time, - progname : String, - message : String - - # Creates a new logger that will log to the given *io*. - # If *io* is `nil` then all log calls will be silently ignored. - @[Deprecated("Use `Log` module instead")] - def initialize(@io : IO?, @level = Severity::INFO, @formatter = DEFAULT_FORMATTER, @progname = "") - @closed = false - @mutex = Mutex.new(:unchecked) - end - - # Calls the *close* method on the object passed to `initialize`. - def close - return if @closed - return unless io = @io - @closed = true - - @mutex.synchronize do - io.close - end - end - - {% for name in Severity.constants %} - {{name.id}} = Severity::{{name.id}} - - # Returns `true` if the logger's current severity is lower or equal to `{{name.id}}`. - def {{name.id.downcase}}? - level <= Severity::{{name.id}} - end - - # Logs *message* if the logger's current severity is lower or equal to `{{name.id}}`. - # *progname* overrides a default progname set in this logger. - def {{name.id.downcase}}(message, progname = nil) - log(Severity::{{name.id}}, message, progname) - end - - # Logs the message as returned from the given block if the logger's current severity - # is lower or equal to `{{name.id}}`. The block is not run if the severity is higher. - # *progname* overrides a default progname set in this logger. - def {{name.id.downcase}}(progname = nil) - log(Severity::{{name.id}}, progname) { yield } - end - {% end %} - - # Logs *message* if *severity* is higher or equal with the logger's current - # severity. *progname* overrides a default progname set in this logger. - @[Deprecated("Use `Log` module instead")] - def log(severity, message, progname = nil) - return if severity < level || !@io - write(severity, Time.local, progname || @progname, message) - end - - # Logs the message as returned from the given block if *severity* - # is higher or equal with the loggers current severity. The block is not run - # if *severity* is lower. *progname* overrides a default progname set in this logger. - @[Deprecated("Use `Log` module instead")] - def log(severity, progname = nil) - return if severity < level || !@io - write(severity, Time.local, progname || @progname, yield) - end - - private def write(severity, datetime, progname, message) - io = @io - return unless io - - progname_to_s = progname.to_s - message_to_s = message.to_s - - @mutex.synchronize do - formatter.call(severity, datetime, progname_to_s, message_to_s, io) - io.puts - io.flush - end - end -end From 1a85151e9a3a0df3e5e1d8f4a9d9d31fc861605d Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Mon, 3 Aug 2020 16:31:22 -0300 Subject: [PATCH 207/263] Check abstract def implementations with double splats (#9633) --- spec/compiler/semantic/abstract_def_spec.cr | 104 ++++++++++++++++++ .../crystal/semantic/abstract_def_checker.cr | 24 +++- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/spec/compiler/semantic/abstract_def_spec.cr b/spec/compiler/semantic/abstract_def_spec.cr index 068c0f6e4f38..a5e82cc88fd1 100644 --- a/spec/compiler/semantic/abstract_def_spec.cr +++ b/spec/compiler/semantic/abstract_def_spec.cr @@ -877,4 +877,108 @@ describe "Semantic: abstract def" do end ) end + + it "errors if keyword argument doesn't have the same default value" do + assert_error %( + abstract class Foo + abstract def foo(*, foo = 1) + end + + class Bar < Foo + def foo(*, foo = 2) + end + end + ), "abstract `def Foo#foo(*, foo = 1)` must be implemented by Bar" + end + + it "allow double splat argument" do + semantic %( + abstract class Foo + abstract def foo(**kargs) + end + + class Bar < Foo + def foo(**kargs) + end + end + ) + end + + it "allow double splat when abstract doesn't have it" do + semantic %( + abstract class Foo + abstract def foo + end + + class Bar < Foo + def foo(**kargs) + end + end + ) + end + + it "errors if implementation misses the double splat" do + assert_error %( + abstract class Foo + abstract def foo(**kargs) + end + + class Bar < Foo + def foo + end + end + ), "abstract `def Foo#foo(**kargs)` must be implemented by Bar" + end + + it "errors if double splat type doesn't match" do + assert_error %( + abstract class Foo + abstract def foo(**kargs : Int32) + end + + class Bar < Foo + def foo(**kargs : String) + end + end + ), "abstract `def Foo#foo(**kargs : Int32)` must be implemented by Bar" + end + + it "allow splat instead of keyword argument" do + semantic %( + abstract class Foo + abstract def foo(*, foo) + end + + class Bar < Foo + def foo(**kargs) + end + end + ) + end + + it "extra keyword arguments must have compatible type to double splat" do + assert_error %( + abstract class Foo + abstract def foo(**kargs : String) + end + + class Bar < Foo + def foo(*, foo : Int32 = 0, **kargs) + end + end + ), "abstract `def Foo#foo(**kargs : String)` must be implemented by Bar" + end + + it "double splat must match keyword argument type" do + assert_error %( + abstract class Foo + abstract def foo(*, foo : Int32) + end + + class Bar < Foo + def foo(**kargs : String) + end + end + ), "abstract `def Foo#foo(*, foo : Int32)` must be implemented by Bar" + end end diff --git a/src/compiler/crystal/semantic/abstract_def_checker.cr b/src/compiler/crystal/semantic/abstract_def_checker.cr index f7ead4d50411..cea1dbee6ec2 100644 --- a/src/compiler/crystal/semantic/abstract_def_checker.cr +++ b/src/compiler/crystal/semantic/abstract_def_checker.cr @@ -187,17 +187,33 @@ class Crystal::AbstractDefChecker {a1.name, a1} end + # Check double splat + if m2_double_splat = m2.double_splat + if m1_double_splat = m1.double_splat + return false unless check_arg(t1, m1_double_splat, t2, m2_double_splat) + else + return false + end + end + # Check keyword arguments + # They must either exist in the implementation or match with the double splat m2_kargs.each do |i| a2 = m2.args[i] - a1 = kargs.delete(a2.name) - return false unless a1 - return false unless check_arg(t1, a1, t2, a2) + if a1 = kargs.delete(a2.name) || m1.double_splat + return false unless check_arg(t1, a1, t2, a2) + else + return false + end end - # Remaining keyword arguments must have a default value + # Check remaining keyword arguments + # They must have a default value and match the double splat in the abstract (if it exists) kargs.each_value do |a1| return false unless a1.default_value + if m2_double_splat = m2.double_splat + return false unless check_arg(t1, a1, t2, m2_double_splat) + end end true From d9fbdd05e527b95f47e678929d3343d78bd9ba87 Mon Sep 17 00:00:00 2001 From: Jamie Gaskins Date: Tue, 4 Aug 2020 23:37:11 -0400 Subject: [PATCH 208/263] Add specs for String.new(Pointer) --- spec/std/string_spec.cr | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index f059ff41ad9f..ca7afa1b30b8 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -1851,16 +1851,34 @@ describe "String" do "ぜんぶ".chars.should eq(['ぜ', 'ん', 'ぶ']) end - it "allows creating a string with zeros" do - p = Pointer(UInt8).malloc(3) - p[0] = 'a'.ord.to_u8 - p[1] = '\0'.ord.to_u8 - p[2] = 'b'.ord.to_u8 - s = String.new(p, 3) - s[0].should eq('a') - s[1].should eq('\0') - s[2].should eq('b') - s.bytesize.should eq(3) + describe "creating from a pointer" do + it "allows creating a string with zeros" do + p = Pointer(UInt8).malloc(3) + p[0] = 'a'.ord.to_u8 + p[1] = '\0'.ord.to_u8 + p[2] = 'b'.ord.to_u8 + s = String.new(p, 3) + s[0].should eq('a') + s[1].should eq('\0') + s[2].should eq('b') + s.bytesize.should eq(3) + end + + it "raises an exception when creating a string with a null pointer and no size" do + expect_raises ArgumentError do + String.new(Pointer(UInt8).null) + end + end + + it "raises when creating from a null pointer with a nonzero size" do + expect_raises ArgumentError do + String.new(Pointer(UInt8).null, 3) + end + end + + it "returns an empty string when creating from a null pointer with size 0" do + String.new(Pointer(UInt8).null, 0).should eq "" + end end describe "tr" do From b52c610ae9903f7d52c39fa932ccfd770bd2b40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dorian=20Mari=C3=A9?= Date: Wed, 5 Aug 2020 21:15:02 +0200 Subject: [PATCH 209/263] Fetch git config correctly (#9640) for instance user.name config was not fetched git --config user.name was not a valid command --- spec/compiler/crystal/tools/init_spec.cr | 20 +++++++++++++++++++ src/compiler/crystal/tools/init.cr | 2 +- .../crystal/tools/init/template/license.ecr | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/spec/compiler/crystal/tools/init_spec.cr b/spec/compiler/crystal/tools/init_spec.cr index 7d59a6586282..c82016d1a57d 100644 --- a/spec/compiler/crystal/tools/init_spec.cr +++ b/spec/compiler/crystal/tools/init_spec.cr @@ -6,6 +6,8 @@ require "ini" require "spec" require "yaml" require "../../../support/tempfile" +require "../../../support/env" +require "../../../support/win32" private def exec_init(project_name, project_dir = nil, type = "lib", force = false, skip_existing = false) args = [type, project_name] @@ -41,6 +43,24 @@ end module Crystal describe Init::InitProject do + it "correctly uses git config" do + within_temporary_directory do + File.write(".gitconfig", <<-CONTENT) + [user] + email = dorian@dorianmarie.fr + name = Dorian Marié + CONTENT + + with_env("GIT_CONFIG": "#{FileUtils.pwd}/.gitconfig") do + exec_init("example", "example", "app") + end + + with_file "example/LICENSE" do |file| + file.should contain("Dorian Marié") + end + end + end + it "has proper contents" do within_temporary_directory do run_init_project("lib", "example", "John Smith", "john@smith.com", "jsmith") diff --git a/src/compiler/crystal/tools/init.cr b/src/compiler/crystal/tools/init.cr index 2b6486612764..1c97b0bc79a7 100644 --- a/src/compiler/crystal/tools/init.cr +++ b/src/compiler/crystal/tools/init.cr @@ -98,7 +98,7 @@ module Crystal private def self.git_config(key) String.build do |io| - Process.run("git", ["--config", key], output: io) + Process.run("git", ["config", "--get", key], output: io) end.strip.presence end diff --git a/src/compiler/crystal/tools/init/template/license.ecr b/src/compiler/crystal/tools/init/template/license.ecr index 64585cbc4244..3f72be5b86ac 100644 --- a/src/compiler/crystal/tools/init/template/license.ecr +++ b/src/compiler/crystal/tools/init/template/license.ecr @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) <%= Time.local.year %> <%= config.author %> +Copyright (c) <%= Time.local.year %> <%= config.author %> <<%= config.email %>> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From b4659a183f4890e460935762e457f5deddb369a9 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 6 Aug 2020 18:13:28 -0300 Subject: [PATCH 210/263] Update distribution-scripts and publish nightly packages to bintray (#9663) * Add package branch filter to perform only dist package buliding * Update distribution-scripts * crystal-lang/distribution-scripts#72 Bintray publishing scripts * crystal-lang/distribution-scripts#73 Update shards v0.12.0 * Publish nightlies to bintray nightlies .tar.gz .deb .rpm will be branded as x.y.z-dev --- .circleci/config.yml | 66 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59c9ab0cc2a3..0bad7067d7cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - run: | git clone https://github.com/crystal-lang/distribution-scripts.git ~/distribution-scripts cd ~/distribution-scripts - git checkout cb8f2c51a042609d7c45707ba8369b37d011037f + git checkout 44172615fa196fb8592046582471fdfc8d69472c # persist relevant information for build process - run: | cd ~/distribution-scripts @@ -225,12 +225,17 @@ jobs: steps: - attach_workspace: at: /tmp/workspace + - checkout + - run: | + # We need CRYSTAL_VERSION in prepare_nightly to use src/VERSION so we publish them as x.y.z-dev in apt/rpm + # + # How to brand it + echo "export CRYSTAL_VERSION=$(cat src/VERSION)" >> /tmp/workspace/distribution-scripts/build.env + # + # TODO: We might want to do that on docker images also to support updates on multiple development versions the same date. - run: | cd /tmp/workspace/distribution-scripts - # How to brand it - export VERSION=nightly-$(date '+%Y%m%d') - echo "export CRYSTAL_VERSION=$VERSION" >> build.env echo "export DOCKER_TAG=nightly" >> build.env # Build from working directory (needed for omnibus and when version does not match branch/tag) @@ -358,6 +363,24 @@ jobs: paths: - build + dist_bintray_nightly: + machine: true + steps: + - attach_workspace: + at: /tmp/workspace + - run: + no_output_timeout: 20m + command: | + cd /tmp/workspace/distribution-scripts + source build.env + cd bintray + ./publish-nightly.sh $CRYSTAL_VERSION $(date '+%Y-%m-%d') \ + /tmp/workspace/build/crystal-*-linux-x86_64.tar.gz \ + /tmp/workspace/build/crystal-*-linux-i686.tar.gz + - store_artifacts: + path: /tmp/workspace/distribution-scripts/bintray/build/unsigned + destination: unsigned + dist_docker: machine: true steps: @@ -522,6 +545,7 @@ workflows: branches: ignore: - /release\/.+/ + - /package\/.+/ - /.*\bci\b.*/ - test_linux32_std: filters: *unless_maintenance @@ -644,6 +668,10 @@ workflows: - dist_darwin: requires: - prepare_nightly + - dist_bintray_nightly: + requires: + - dist_linux + - dist_linux32 - dist_docker: requires: - dist_linux @@ -752,3 +780,33 @@ workflows: - dist_darwin - dist_snap - dist_docs + + package_build: + jobs: + - prepare_common: + filters: &package + branches: + only: + - /package\/.+/ + - prepare_maintenance: + filters: *package + requires: + - prepare_common + - dist_linux: + filters: *package + requires: + - prepare_maintenance + - dist_linux32: + filters: *package + requires: + - prepare_maintenance + - dist_darwin: + filters: *package + requires: + - prepare_maintenance + - dist_artifacts: + filters: *package + requires: + - dist_linux + - dist_linux32 + - dist_darwin From afcb3f46f10da3b540dda6ab5cce05765a866acf Mon Sep 17 00:00:00 2001 From: Sijawusz Pur Rahnama Date: Thu, 13 Aug 2020 23:01:47 +0200 Subject: [PATCH 211/263] Add timestamp as an argument in Log::Entry initializer (#9570) --- src/log/entry.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log/entry.cr b/src/log/entry.cr index 817706b4d310..e9d7098faa94 100644 --- a/src/log/entry.cr +++ b/src/log/entry.cr @@ -39,11 +39,11 @@ struct Log::Entry getter source : String getter severity : Severity getter message : String - getter timestamp = Time.local + getter timestamp : Time getter context : Metadata = Log.context.metadata getter data : Metadata getter exception : Exception? - def initialize(@source : String, @severity : Severity, @message : String, @data : Log::Metadata, @exception : Exception?) + def initialize(@source : String, @severity : Severity, @message : String, @data : Log::Metadata, @exception : Exception?, *, @timestamp = Time.local) end end From 7d2b41c9ac100b2367f644992e4f80c52edec2d5 Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Thu, 13 Aug 2020 23:02:46 +0200 Subject: [PATCH 212/263] Add HTTP::Cookie::SameSite::None (#9262) --- src/http/cookie.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/http/cookie.cr b/src/http/cookie.cr index 53ce31cb0472..e867e68cac2f 100644 --- a/src/http/cookie.cr +++ b/src/http/cookie.cr @@ -5,9 +5,12 @@ module HTTP class Cookie # Possible values for the `SameSite` cookie as described in the [Same-site Cookies Draft](https://tools.ietf.org/html/draft-west-first-party-cookies-07#section-4.1.1). enum SameSite + # The browser will send cookies with both cross-site requests and same-site requests. + # + # The `None` directive requires the `secure` attribute to be `true` to mitigate risks associated with cross-site access. + None # Prevents the cookie from being sent by the browser in all cross-site browsing contexts. Strict - # Allows the cookie to be sent by the browser during top-level navigations that use a [safe](https://tools.ietf.org/html/rfc7231#section-4.2.1) HTTP method. Lax end From 49dec67171d45f8473e6dd4fd44780211e7efad3 Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 19 Aug 2020 11:04:24 -0300 Subject: [PATCH 213/263] `def_equals` compare first by reference (#9650) --- spec/std/object_spec.cr | 23 +++++++++++++++++++++++ src/object.cr | 6 ++++++ 2 files changed, 29 insertions(+) diff --git a/spec/std/object_spec.cr b/spec/std/object_spec.cr index a84557ff3484..65868789abce 100644 --- a/spec/std/object_spec.cr +++ b/spec/std/object_spec.cr @@ -154,6 +154,20 @@ private class HashedTestObject def_hash :a, :b end +private struct NonReflexive + def ==(other) + false + end +end + +private class DefEquals + def initialize + @x = NonReflexive.new + end + + def_equals @x +end + describe Object do describe "delegate" do it "delegates" do @@ -496,4 +510,13 @@ describe Object do it "applies annotation to lazy property (#9139)" do TestObject.test_annotation_count.should eq(1) end + + describe "def_equals" do + it "compares by reference" do + x = DefEquals.new + y = DefEquals.new + (x == x).should be_true + (x == y).should be_false + end + end end diff --git a/src/object.cr b/src/object.cr index 4adfaab51a2c..e6f3a777dd43 100644 --- a/src/object.cr +++ b/src/object.cr @@ -1287,6 +1287,9 @@ class Object # Defines an `==` method by comparing the given fields. # # The generated `==` method has a `self` restriction. + # For classes it will first compare by reference and return `true` + # when an object instance is compared with itself, without comparing + # any of the fields. # # ``` # class Person @@ -1299,6 +1302,9 @@ class Object # ``` macro def_equals(*fields) def ==(other : self) + {% if @type.class? %} + return true if same?(other) + {% end %} {% for field in fields %} return false unless {{field.id}} == other.{{field.id}} {% end %} From d5d79dd1a445dd271a35ecae3223acdce2bce0be Mon Sep 17 00:00:00 2001 From: Juan Wajnerman Date: Wed, 19 Aug 2020 11:10:22 -0300 Subject: [PATCH 214/263] HTTP::StaticFileHandler serve pre-gzipped content (#9626) * HTTP::StaticFileHandler serve pre-gzipped content * Add some docs about serving .gz content * Change check order to avoid unnecessary syscalls --- spec/std/data/static_file_handler/test.txt.gz | Bin 0 -> 34 bytes .../handlers/static_file_handler_spec.cr | 27 ++++++++++++++++++ .../server/handlers/static_file_handler.cr | 21 +++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 spec/std/data/static_file_handler/test.txt.gz diff --git a/spec/std/data/static_file_handler/test.txt.gz b/spec/std/data/static_file_handler/test.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..25923f999dbdb3de37e554b0e51e914f02eda8d6 GIT binary patch literal 34 qcmb2|=HSpcvnY;%xg@o?M6aZxgyFNt*^?*P8JapTerIK1U;qHr6byL) literal 0 HcmV?d00001 diff --git a/spec/std/http/server/handlers/static_file_handler_spec.cr b/spec/std/http/server/handlers/static_file_handler_spec.cr index 8ed0fca44ca4..60603f75427c 100644 --- a/spec/std/http/server/handlers/static_file_handler_spec.cr +++ b/spec/std/http/server/handlers/static_file_handler_spec.cr @@ -250,4 +250,31 @@ describe HTTP::StaticFileHandler do response = handle HTTP::Request.new("GET", "/test.txt%0A") response.status_code.should eq(404) end + + it "serve compressed content" do + modification_time = File.info(datapath("static_file_handler", "test.txt")).modification_time + File.touch datapath("static_file_handler", "test.txt.gz"), modification_time + 1.second + + headers = HTTP::Headers{"Accept-Encoding" => "gzip"} + response = handle HTTP::Request.new("GET", "/test.txt", headers) + response.headers["Content-Encoding"].should eq("gzip") + end + + it "still serve compressed content when modification time is very close" do + modification_time = File.info(datapath("static_file_handler", "test.txt")).modification_time + File.touch datapath("static_file_handler", "test.txt.gz"), modification_time - 1.microsecond + + headers = HTTP::Headers{"Accept-Encoding" => "gzip"} + response = handle HTTP::Request.new("GET", "/test.txt", headers) + response.headers["Content-Encoding"].should eq("gzip") + end + + it "doesn't serve compressed content if older than raw file" do + modification_time = File.info(datapath("static_file_handler", "test.txt")).modification_time + File.touch datapath("static_file_handler", "test.txt.gz"), modification_time - 1.second + + headers = HTTP::Headers{"Accept-Encoding" => "gzip"} + response = handle HTTP::Request.new("GET", "/test.txt", headers) + response.headers["Content-Encoding"]?.should be_nil + end end diff --git a/src/http/server/handlers/static_file_handler.cr b/src/http/server/handlers/static_file_handler.cr index 0416f0cef056..e70b694cb34f 100644 --- a/src/http/server/handlers/static_file_handler.cr +++ b/src/http/server/handlers/static_file_handler.cr @@ -3,7 +3,11 @@ require "html" require "uri" require "mime" -# A simple handler that lists directories and serves files under a given public directory. +# A handler that lists directories and serves files under a given public directory. +# +# This handler can send precompressed content, if the client accepts it, and a file +# with the same name and `.gz` extension appended is found in the same directory. +# Precompressed files are only served if they are newer than the original file. class HTTP::StaticFileHandler include HTTP::Handler @@ -76,6 +80,21 @@ class HTTP::StaticFileHandler end context.response.content_type = MIME.from_filename(file_path.to_s, "application/octet-stream") + + # Checks if pre-gzipped file can be served + if context.request.headers.includes_word?("Accept-Encoding", "gzip") + gz_file_path = "#{file_path}.gz" + + if File.exists?(gz_file_path) && + # Allow small time drift. In some file systems, using `gz --keep` to + # compress the file will keep the modification time of the original file + # but truncating some decimals + last_modified - modification_time(gz_file_path) < 1.millisecond + file_path = gz_file_path + context.response.headers["Content-Encoding"] = "gzip" + end + end + context.response.content_length = File.size(file_path) File.open(file_path) do |file| IO.copy(file, context.response) From 007dcfc783681858504ec6e3920c4dc9d30cb328 Mon Sep 17 00:00:00 2001 From: PhilAtWysdom Date: Wed, 19 Aug 2020 15:11:01 +0100 Subject: [PATCH 215/263] force-peer: VerifyMode::FAIL_IF_NO_PEER_CERT is only valid when used together VerifyMode::PEER (#9668) --- src/openssl/ssl/context.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openssl/ssl/context.cr b/src/openssl/ssl/context.cr index 5a867bc6267c..031e44d57b27 100644 --- a/src/openssl/ssl/context.cr +++ b/src/openssl/ssl/context.cr @@ -410,7 +410,7 @@ abstract class OpenSSL::SSL::Context when "peer" context.verify_mode = OpenSSL::SSL::VerifyMode::PEER when "force-peer" - context.verify_mode = OpenSSL::SSL::VerifyMode::FAIL_IF_NO_PEER_CERT + context.verify_mode = OpenSSL::SSL::VerifyMode::PEER | OpenSSL::SSL::VerifyMode::FAIL_IF_NO_PEER_CERT when "none" context.verify_mode = OpenSSL::SSL::VerifyMode::NONE when nil From 203de4ed212d3310035380f807315e7b852d3da5 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 24 Aug 2020 10:41:24 -0300 Subject: [PATCH 216/263] Parser: fix `a-b -c` incorrectly parsed as `a - (b - c)` (#9652) --- spec/compiler/parser/parser_spec.cr | 12 ++++++++++++ src/compiler/crystal/syntax/parser.cr | 27 ++++++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index cebb4295b14d..3a535c24c918 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -111,6 +111,18 @@ module Crystal it_parses "1 / -2", Call.new(1.int32, "/", -2.int32) it_parses "2 / 3 + 4 / 5", Call.new(Call.new(2.int32, "/", 3.int32), "+", Call.new(4.int32, "/", 5.int32)) it_parses "2 * (3 + 4)", Call.new(2.int32, "*", Expressions.new([Call.new(3.int32, "+", 4.int32)] of ASTNode)) + it_parses "a = 1; b = 2; c = 3; a-b-c", Expressions.new([ + Assign.new("a".var, 1.int32), + Assign.new("b".var, 2.int32), + Assign.new("c".var, 3.int32), + Call.new(Call.new("a".var, "-", "b".var), "-", "c".var), + ]) + it_parses "a = 1; b = 2; c = 3; a-b -c", Expressions.new([ + Assign.new("a".var, 1.int32), + Assign.new("b".var, 2.int32), + Assign.new("c".var, 3.int32), + Call.new(Call.new("a".var, "-", "b".var), "-", "c".var), + ]) it_parses "1/2", Call.new(1.int32, "/", [2.int32] of ASTNode) it_parses "1 + /foo/", Call.new(1.int32, "+", regex("foo")) it_parses "1+0", Call.new(1.int32, "+", 0.int32) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index f687ea3c4be4..4a7a95832124 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -4056,6 +4056,16 @@ module Crystal is_var = var?(name) + # If the name is a var and '+' or '-' follow, never treat the name as a call + if is_var && next_comes_plus_or_minus? + var = Var.new(name) + var.doc = doc + var.location = name_location + var.end_location = name_location + next_token + return var + end + @wants_regex = false next_token @@ -4127,13 +4137,6 @@ module Crystal maybe_var = !force_call && is_var && !has_parentheses if maybe_var && args.size == 0 Var.new(name) - elsif maybe_var && args.size == 1 && (num = args[0]) && (num.is_a?(NumberLiteral) && num.has_sign?) - sign = num.value[0].to_s - num.value = num.value.byte_slice(1) - Call.new(Var.new(name), sign, args) - elsif maybe_var && args.size == 1 && (arg = args[0]) && arg.is_a?(Call) && !arg.obj.nil? && - arg.name.in?("+", "-") && (!arg.args || arg.args.size == 0) - Call.new(Var.new(name), arg.name, arg.obj.not_nil!) else call = Call.new(nil, name, args, nil, block_arg, named_args, global) call.name_location = name_location @@ -4168,6 +4171,16 @@ module Crystal node end + def next_comes_plus_or_minus? + pos = current_pos + while current_char.ascii_whitespace? + next_char_no_column_increment + end + comes_plus_or_minus = current_char == '+' || current_char == '-' + self.current_pos = pos + comes_plus_or_minus + end + def preserve_stop_on_do(new_value = false) old_stop_on_do = @stop_on_do @stop_on_do = new_value From f44129814e304452dbc71d109df72d016e99efba Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Mon, 24 Aug 2020 22:43:48 +0900 Subject: [PATCH 217/263] Fix String#rindex default offset for matching empty regex correctly (#9690) Fixed #9679 --- spec/std/string_spec.cr | 6 ++++++ src/string.cr | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index f059ff41ad9f..0ec443895cc0 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -910,6 +910,12 @@ describe "String" do it { "a43b53".rindex(/\d+/).should eq(4) } it { "bbbb".rindex(/\d/).should be_nil } + describe "which matches empty string" do + it { "foo".rindex(/o*/).should eq(3) } + it { "foo".rindex(//).should eq(3) } + it { "foo".rindex(/\b/).should eq(3) } + end + describe "with offset" do it { "bbbb".rindex(/b/, 2).should eq(2) } it { "abbbb".rindex(/b/, 0).should be_nil } diff --git a/src/string.cr b/src/string.cr index e9b70b1e3f4f..08414bdd5056 100644 --- a/src/string.cr +++ b/src/string.cr @@ -3180,7 +3180,7 @@ class String end # :ditto: - def rindex(search : Regex, offset = size - 1) + def rindex(search : Regex, offset = size) offset += size if offset < 0 return nil unless 0 <= offset <= size From b2b4a8bdc7df81d2a37985b3cc0aefce0a1f336b Mon Sep 17 00:00:00 2001 From: Sijawusz Pur Rahnama Date: Mon, 24 Aug 2020 16:18:48 +0200 Subject: [PATCH 218/263] Marks else branch of exhaustive case as unreachable (#9659) --- spec/compiler/semantic/cast_spec.cr | 10 ++++++++++ src/compiler/crystal/semantic/main_visitor.cr | 1 + 2 files changed, 11 insertions(+) diff --git a/spec/compiler/semantic/cast_spec.cr b/spec/compiler/semantic/cast_spec.cr index d9568c7058b2..881d042eefdd 100644 --- a/spec/compiler/semantic/cast_spec.cr +++ b/spec/compiler/semantic/cast_spec.cr @@ -359,4 +359,14 @@ describe "Semantic: cast" do x.foo if x )) { nil_type } end + + it "considers else to be unreachable (#9658)" do + assert_type(%( + case 1 + in Int32 + v = 1 + end + v + )) { int32 } + end end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index bf5ddb6bfb65..db8fb8d856ac 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -3165,6 +3165,7 @@ module Crystal def visit(node : Unreachable) node.type = @program.no_return + @unreachable = true end # # Helpers From 0be0505b6b000c36baba7e9d95a1eb329c403b91 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 28 Aug 2020 12:04:02 -0300 Subject: [PATCH 219/263] Always raise ArgumentError with null pointer --- spec/std/string_spec.cr | 6 ++++-- src/string.cr | 10 +++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index ca7afa1b30b8..6dc23e7dcc32 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -1876,8 +1876,10 @@ describe "String" do end end - it "returns an empty string when creating from a null pointer with size 0" do - String.new(Pointer(UInt8).null, 0).should eq "" + it "raises when creating from a null pointer with size 0" do + expect_raises ArgumentError do + String.new(Pointer(UInt8).null, 0).should eq "" + end end end diff --git a/src/string.cr b/src/string.cr index 7baf1b854a0f..9ea58718e21d 100644 --- a/src/string.cr +++ b/src/string.cr @@ -191,9 +191,7 @@ class String # String.new(ptr) # => "abcd" # ``` def self.new(chars : UInt8*) - if chars.null? - raise ArgumentError.new("Cannot create a string with a null pointer") - end + raise ArgumentError.new("Cannot create a string with a null pointer") if chars.null? new(chars, LibC.strlen(chars)) end @@ -210,13 +208,11 @@ class String # String.new(ptr, 2) # => "ab" # ``` def self.new(chars : UInt8*, bytesize, size = 0) + raise ArgumentError.new("Cannot create a string with a null pointer") if chars.null? + # Avoid allocating memory for the empty string return "" if bytesize == 0 - if chars.null? - raise ArgumentError.new("Cannot create a string with a null pointer") - end - new(bytesize) do |buffer| buffer.copy_from(chars, bytesize) {bytesize, size} From 853002557dab697f35ed5957a16b096136484769 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Sun, 30 Aug 2020 12:57:54 -0300 Subject: [PATCH 220/263] Compiler: make inherited hook work through generic instances (#9701) --- spec/compiler/semantic/hooks_spec.cr | 20 +++++++++++++++++++ .../crystal/semantic/top_level_visitor.cr | 14 +++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/spec/compiler/semantic/hooks_spec.cr b/spec/compiler/semantic/hooks_spec.cr index dd471027bbd1..a2ff7ee7d0aa 100644 --- a/spec/compiler/semantic/hooks_spec.cr +++ b/spec/compiler/semantic/hooks_spec.cr @@ -228,4 +228,24 @@ describe "Semantic: hooks" do {a1.x, a2.y} ), inject_primitives: false) { tuple_of([string, int32]) } end + + it "does inherited macro through generic instance type (#9693)" do + assert_type(" + class Foo(X) + macro inherited + def self.{{@type.name.downcase.id}} + 1 + end + end + end + + class Bar < Foo(Int32) + end + + class Baz < Bar + end + + Baz.baz + ") { int32 } + end end diff --git a/src/compiler/crystal/semantic/top_level_visitor.cr b/src/compiler/crystal/semantic/top_level_visitor.cr index 35a41277519a..3ce013f6bc43 100644 --- a/src/compiler/crystal/semantic/top_level_visitor.cr +++ b/src/compiler/crystal/semantic/top_level_visitor.cr @@ -1013,8 +1013,18 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor node.add_hook_expansion(expansion) end - if kind == :inherited && (superclass = type_with_hooks.instance_type.superclass) - run_hooks(superclass.metaclass, current_type, kind, node) + if kind == :inherited + # In the case of: + # + # class A(X); end + # clsss B < A(Int32);end + # + # we need to go from A(Int32) to A(X) to go up the hierarchy. + if type_with_hooks.is_a?(GenericClassInstanceMetaclassType) + run_hooks(type_with_hooks.instance_type.generic_type.metaclass, current_type, kind, node) + elsif (superclass = type_with_hooks.instance_type.superclass) + run_hooks(superclass.metaclass, current_type, kind, node) + end end end From 79ded4abeaf41ba82243a7dc7c1c30d53c399bf1 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Sun, 30 Aug 2020 12:58:05 -0300 Subject: [PATCH 221/263] Disallow keywords as block argument name (#9704) * Parser: disallow `self` as an argument name * Parser: disallow keywords as block argument names * Parser: disallow `in` as argument name * Rename `in` argument to `input` --- spec/compiler/parser/parser_spec.cr | 5 +- src/compiler/crystal/syntax/parser.cr | 15 ++- src/digest/md5.cr | 130 +++++++++++++------------- 3 files changed, 81 insertions(+), 69 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 3a535c24c918..73b9730a7f27 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -209,13 +209,16 @@ module Crystal extend class struct module enum while until return next break lib fun alias pointerof sizeof instance_sizeof offsetof typeof private protected asm out - end + end self in ).each do |kw| assert_syntax_error "def foo(#{kw}); end", "cannot use '#{kw}' as an argument name", 1, 9 assert_syntax_error "def foo(foo #{kw}); end", "cannot use '#{kw}' as an argument name", 1, 13 it_parses "def foo(#{kw} foo); end", Def.new("foo", [Arg.new("foo", external_name: kw.to_s)]) it_parses "def foo(@#{kw}); end", Def.new("foo", [Arg.new("__arg0", external_name: kw.to_s)], [Assign.new("@#{kw}".instance_var, "__arg0".var)] of ASTNode) it_parses "def foo(@@#{kw}); end", Def.new("foo", [Arg.new("__arg0", external_name: kw.to_s)], [Assign.new("@@#{kw}".class_var, "__arg0".var)] of ASTNode) + + assert_syntax_error "foo { |#{kw})| }", "cannot use '#{kw}' as a block argument name", 1, 8 + assert_syntax_error "foo { |(#{kw}))| }", "cannot use '#{kw}' as a block argument name", 1, 9 end it_parses "def self.foo\n1\nend", Def.new("foo", body: 1.int32, receiver: "self".var) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 4a7a95832124..b71c27e4c65b 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3917,8 +3917,7 @@ module Crystal :extend, :class, :struct, :module, :enum, :while, :until, :return, :next, :break, :lib, :fun, :alias, :pointerof, :sizeof, :offsetof, :instance_sizeof, :typeof, :private, :protected, :asm, :out, - # `end` is also invalid because it maybe terminate `def` block. - :end + :self, :in, :end true else false @@ -3930,7 +3929,7 @@ module Crystal "extend", "class", "struct", "module", "enum", "while", "until", "return", "next", "break", "lib", "fun", "alias", "pointerof", "sizeof", "offsetof", "instance_sizeof", "typeof", "private", "protected", "asm", "out", - "end" + "self", "in", "end" true else false @@ -4242,7 +4241,12 @@ module Crystal case @token.type when :IDENT + if @token.keyword? && invalid_internal_name?(@token.value) + raise "cannot use '#{@token}' as a block argument name", @token + end + arg_name = @token.value.to_s + if all_names.includes?(arg_name) raise "duplicated block argument name: #{arg_name}", @token end @@ -4258,7 +4262,12 @@ module Crystal while true case @token.type when :IDENT + if @token.keyword? && invalid_internal_name?(@token.value) + raise "cannot use '#{@token}' as a block argument name", @token + end + sub_arg_name = @token.value.to_s + if all_names.includes?(sub_arg_name) raise "duplicated block argument name: #{sub_arg_name}", @token end diff --git a/src/digest/md5.cr b/src/digest/md5.cr index 69b7ddefa3f0..e7c4699eb085 100644 --- a/src/digest/md5.cr +++ b/src/digest/md5.cr @@ -131,80 +131,80 @@ class Digest::MD5 < Digest::Base a &+= b end - private def transform(in) + private def transform(input) a, b, c, d = @buf # Round 1 - a = ff(a, b, c, d, in[0], S11, 3614090360) # 1 - d = ff(d, a, b, c, in[1], S12, 3905402710) # 2 - c = ff(c, d, a, b, in[2], S13, 606105819) # 3 - b = ff(b, c, d, a, in[3], S14, 3250441966) # 4 - a = ff(a, b, c, d, in[4], S11, 4118548399) # 5 - d = ff(d, a, b, c, in[5], S12, 1200080426) # 6 - c = ff(c, d, a, b, in[6], S13, 2821735955) # 7 - b = ff(b, c, d, a, in[7], S14, 4249261313) # 8 - a = ff(a, b, c, d, in[8], S11, 1770035416) # 9 - d = ff(d, a, b, c, in[9], S12, 2336552879) # 10 - c = ff(c, d, a, b, in[10], S13, 4294925233) # 11 - b = ff(b, c, d, a, in[11], S14, 2304563134) # 12 - a = ff(a, b, c, d, in[12], S11, 1804603682) # 13 - d = ff(d, a, b, c, in[13], S12, 4254626195) # 14 - c = ff(c, d, a, b, in[14], S13, 2792965006) # 15 - b = ff(b, c, d, a, in[15], S14, 1236535329) # 16 + a = ff(a, b, c, d, input[0], S11, 3614090360) # 1 + d = ff(d, a, b, c, input[1], S12, 3905402710) # 2 + c = ff(c, d, a, b, input[2], S13, 606105819) # 3 + b = ff(b, c, d, a, input[3], S14, 3250441966) # 4 + a = ff(a, b, c, d, input[4], S11, 4118548399) # 5 + d = ff(d, a, b, c, input[5], S12, 1200080426) # 6 + c = ff(c, d, a, b, input[6], S13, 2821735955) # 7 + b = ff(b, c, d, a, input[7], S14, 4249261313) # 8 + a = ff(a, b, c, d, input[8], S11, 1770035416) # 9 + d = ff(d, a, b, c, input[9], S12, 2336552879) # 10 + c = ff(c, d, a, b, input[10], S13, 4294925233) # 11 + b = ff(b, c, d, a, input[11], S14, 2304563134) # 12 + a = ff(a, b, c, d, input[12], S11, 1804603682) # 13 + d = ff(d, a, b, c, input[13], S12, 4254626195) # 14 + c = ff(c, d, a, b, input[14], S13, 2792965006) # 15 + b = ff(b, c, d, a, input[15], S14, 1236535329) # 16 # Round 2 - a = gg(a, b, c, d, in[1], S21, 4129170786) # 17 - d = gg(d, a, b, c, in[6], S22, 3225465664) # 18 - c = gg(c, d, a, b, in[11], S23, 643717713) # 19 - b = gg(b, c, d, a, in[0], S24, 3921069994) # 20 - a = gg(a, b, c, d, in[5], S21, 3593408605) # 21 - d = gg(d, a, b, c, in[10], S22, 38016083) # 22 - c = gg(c, d, a, b, in[15], S23, 3634488961) # 23 - b = gg(b, c, d, a, in[4], S24, 3889429448) # 24 - a = gg(a, b, c, d, in[9], S21, 568446438) # 25 - d = gg(d, a, b, c, in[14], S22, 3275163606) # 26 - c = gg(c, d, a, b, in[3], S23, 4107603335) # 27 - b = gg(b, c, d, a, in[8], S24, 1163531501) # 28 - a = gg(a, b, c, d, in[13], S21, 2850285829) # 29 - d = gg(d, a, b, c, in[2], S22, 4243563512) # 30 - c = gg(c, d, a, b, in[7], S23, 1735328473) # 31 - b = gg(b, c, d, a, in[12], S24, 2368359562) # 32 + a = gg(a, b, c, d, input[1], S21, 4129170786) # 17 + d = gg(d, a, b, c, input[6], S22, 3225465664) # 18 + c = gg(c, d, a, b, input[11], S23, 643717713) # 19 + b = gg(b, c, d, a, input[0], S24, 3921069994) # 20 + a = gg(a, b, c, d, input[5], S21, 3593408605) # 21 + d = gg(d, a, b, c, input[10], S22, 38016083) # 22 + c = gg(c, d, a, b, input[15], S23, 3634488961) # 23 + b = gg(b, c, d, a, input[4], S24, 3889429448) # 24 + a = gg(a, b, c, d, input[9], S21, 568446438) # 25 + d = gg(d, a, b, c, input[14], S22, 3275163606) # 26 + c = gg(c, d, a, b, input[3], S23, 4107603335) # 27 + b = gg(b, c, d, a, input[8], S24, 1163531501) # 28 + a = gg(a, b, c, d, input[13], S21, 2850285829) # 29 + d = gg(d, a, b, c, input[2], S22, 4243563512) # 30 + c = gg(c, d, a, b, input[7], S23, 1735328473) # 31 + b = gg(b, c, d, a, input[12], S24, 2368359562) # 32 # Round 3 - a = hh(a, b, c, d, in[5], S31, 4294588738) # 33 - d = hh(d, a, b, c, in[8], S32, 2272392833) # 34 - c = hh(c, d, a, b, in[11], S33, 1839030562) # 35 - b = hh(b, c, d, a, in[14], S34, 4259657740) # 36 - a = hh(a, b, c, d, in[1], S31, 2763975236) # 37 - d = hh(d, a, b, c, in[4], S32, 1272893353) # 38 - c = hh(c, d, a, b, in[7], S33, 4139469664) # 39 - b = hh(b, c, d, a, in[10], S34, 3200236656) # 40 - a = hh(a, b, c, d, in[13], S31, 681279174) # 41 - d = hh(d, a, b, c, in[0], S32, 3936430074) # 42 - c = hh(c, d, a, b, in[3], S33, 3572445317) # 43 - b = hh(b, c, d, a, in[6], S34, 76029189) # 44 - a = hh(a, b, c, d, in[9], S31, 3654602809) # 45 - d = hh(d, a, b, c, in[12], S32, 3873151461) # 46 - c = hh(c, d, a, b, in[15], S33, 530742520) # 47 - b = hh(b, c, d, a, in[2], S34, 3299628645) # 48 + a = hh(a, b, c, d, input[5], S31, 4294588738) # 33 + d = hh(d, a, b, c, input[8], S32, 2272392833) # 34 + c = hh(c, d, a, b, input[11], S33, 1839030562) # 35 + b = hh(b, c, d, a, input[14], S34, 4259657740) # 36 + a = hh(a, b, c, d, input[1], S31, 2763975236) # 37 + d = hh(d, a, b, c, input[4], S32, 1272893353) # 38 + c = hh(c, d, a, b, input[7], S33, 4139469664) # 39 + b = hh(b, c, d, a, input[10], S34, 3200236656) # 40 + a = hh(a, b, c, d, input[13], S31, 681279174) # 41 + d = hh(d, a, b, c, input[0], S32, 3936430074) # 42 + c = hh(c, d, a, b, input[3], S33, 3572445317) # 43 + b = hh(b, c, d, a, input[6], S34, 76029189) # 44 + a = hh(a, b, c, d, input[9], S31, 3654602809) # 45 + d = hh(d, a, b, c, input[12], S32, 3873151461) # 46 + c = hh(c, d, a, b, input[15], S33, 530742520) # 47 + b = hh(b, c, d, a, input[2], S34, 3299628645) # 48 # Round 4 - a = ii(a, b, c, d, in[0], S41, 4096336452) # 49 - d = ii(d, a, b, c, in[7], S42, 1126891415) # 50 - c = ii(c, d, a, b, in[14], S43, 2878612391) # 51 - b = ii(b, c, d, a, in[5], S44, 4237533241) # 52 - a = ii(a, b, c, d, in[12], S41, 1700485571) # 53 - d = ii(d, a, b, c, in[3], S42, 2399980690) # 54 - c = ii(c, d, a, b, in[10], S43, 4293915773) # 55 - b = ii(b, c, d, a, in[1], S44, 2240044497) # 56 - a = ii(a, b, c, d, in[8], S41, 1873313359) # 57 - d = ii(d, a, b, c, in[15], S42, 4264355552) # 58 - c = ii(c, d, a, b, in[6], S43, 2734768916) # 59 - b = ii(b, c, d, a, in[13], S44, 1309151649) # 60 - a = ii(a, b, c, d, in[4], S41, 4149444226) # 61 - d = ii(d, a, b, c, in[11], S42, 3174756917) # 62 - c = ii(c, d, a, b, in[2], S43, 718787259) # 63 - b = ii(b, c, d, a, in[9], S44, 3951481745) # 64 + a = ii(a, b, c, d, input[0], S41, 4096336452) # 49 + d = ii(d, a, b, c, input[7], S42, 1126891415) # 50 + c = ii(c, d, a, b, input[14], S43, 2878612391) # 51 + b = ii(b, c, d, a, input[5], S44, 4237533241) # 52 + a = ii(a, b, c, d, input[12], S41, 1700485571) # 53 + d = ii(d, a, b, c, input[3], S42, 2399980690) # 54 + c = ii(c, d, a, b, input[10], S43, 4293915773) # 55 + b = ii(b, c, d, a, input[1], S44, 2240044497) # 56 + a = ii(a, b, c, d, input[8], S41, 1873313359) # 57 + d = ii(d, a, b, c, input[15], S42, 4264355552) # 58 + c = ii(c, d, a, b, input[6], S43, 2734768916) # 59 + b = ii(b, c, d, a, input[13], S44, 1309151649) # 60 + a = ii(a, b, c, d, input[4], S41, 4149444226) # 61 + d = ii(d, a, b, c, input[11], S42, 3174756917) # 62 + c = ii(c, d, a, b, input[2], S43, 718787259) # 63 + b = ii(b, c, d, a, input[9], S44, 3951481745) # 64 @buf[0] &+= a @buf[1] &+= b From c928cdb5315d7c51d5b234177714c305f087bc81 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 31 Aug 2020 09:57:54 -0300 Subject: [PATCH 222/263] Fix `String#index` not working well for broken UTF-8 sequences (#9713) * String: fix index for broken UTF-8 sequences * Put some specs inside `it` * Invalid byte sequence always consumes 1 byte * Add a spec of `String#index` with offset and broken UTF-8 * Fix utf16 spec * Fix `String#ascii_only?` * Fix `String#rchop?` bug * String optimized paths now use `single_byte_optimizable?` * Faster char_bytesize_at using pointer This is safe because a string always ends with a null char * Faster access to string byte in Char::Reader This is safe because strings ends with a null char --- spec/std/char/reader_spec.cr | 30 ++++---- spec/std/string/utf16_spec.cr | 2 +- spec/std/string_spec.cr | 113 ++++++++++++++++------------- src/char/reader.cr | 44 +++++------- src/path.cr | 2 +- src/string.cr | 132 ++++++++++++++++++++-------------- 6 files changed, 176 insertions(+), 147 deletions(-) diff --git a/spec/std/char/reader_spec.cr b/spec/std/char/reader_spec.cr index a395db44b640..0d34beac6d45 100644 --- a/spec/std/char/reader_spec.cr +++ b/spec/std/char/reader_spec.cr @@ -1,10 +1,10 @@ require "spec" require "char/reader" -private def assert_invalid_byte_sequence(bytes, width) +private def assert_invalid_byte_sequence(bytes) reader = Char::Reader.new(String.new bytes) reader.current_char.should eq(Char::REPLACEMENT) - reader.current_char_width.should eq(width) + reader.current_char_width.should eq(1) reader.error.should eq(bytes[0]) end @@ -130,51 +130,51 @@ describe "Char::Reader" do end it "errors if 0x80 <= first_byte < 0xC2" do - assert_invalid_byte_sequence Bytes[0x80], 1 - assert_invalid_byte_sequence Bytes[0xC1], 1 + assert_invalid_byte_sequence Bytes[0x80] + assert_invalid_byte_sequence Bytes[0xC1] end it "errors if (second_byte & 0xC0) != 0x80" do - assert_invalid_byte_sequence Bytes[0xd0], 1 + assert_invalid_byte_sequence Bytes[0xd0] end it "errors if first_byte == 0xE0 && second_byte < 0xA0" do - assert_invalid_byte_sequence Bytes[0xe0, 0x9F, 0xA0], 3 + assert_invalid_byte_sequence Bytes[0xe0, 0x9F, 0xA0] end it "errors if first_byte == 0xED && second_byte >= 0xA0" do - assert_invalid_byte_sequence Bytes[0xed, 0xB0, 0xA0], 3 + assert_invalid_byte_sequence Bytes[0xed, 0xB0, 0xA0] end it "errors if first_byte < 0xF0 && (third_byte & 0xC0) != 0x80" do - assert_invalid_byte_sequence Bytes[0xe0, 0xA0, 0], 2 + assert_invalid_byte_sequence Bytes[0xe0, 0xA0, 0] end it "errors if first_byte == 0xF0 && second_byte < 0x90" do - assert_invalid_byte_sequence Bytes[0xf0, 0x8F, 0xA0], 3 + assert_invalid_byte_sequence Bytes[0xf0, 0x8F, 0xA0] end it "errors if first_byte == 0xF4 && second_byte >= 0x90" do - assert_invalid_byte_sequence Bytes[0xf4, 0x90, 0xA0], 3 + assert_invalid_byte_sequence Bytes[0xf4, 0x90, 0xA0] end it "errors if first_byte < 0xF5 && (fourth_byte & 0xC0) != 0x80" do - assert_invalid_byte_sequence Bytes[0xf4, 0x8F, 0xA0, 0], 4 + assert_invalid_byte_sequence Bytes[0xf4, 0x8F, 0xA0, 0] end it "errors if first_byte >= 0xF5" do - assert_invalid_byte_sequence Bytes[0xf5, 0x8F, 0xA0, 0xA0], 4 + assert_invalid_byte_sequence Bytes[0xf5, 0x8F, 0xA0, 0xA0] end it "errors if second_byte is out of bounds" do - assert_invalid_byte_sequence Bytes[0xf4], 1 + assert_invalid_byte_sequence Bytes[0xf4] end it "errors if third_byte is out of bounds" do - assert_invalid_byte_sequence Bytes[0xf4, 0x8f], 2 + assert_invalid_byte_sequence Bytes[0xf4, 0x8f] end it "errors if fourth_byte is out of bounds" do - assert_invalid_byte_sequence Bytes[0xf4, 0x8f, 0xa0], 3 + assert_invalid_byte_sequence Bytes[0xf4, 0x8f, 0xa0] end end diff --git a/spec/std/string/utf16_spec.cr b/spec/std/string/utf16_spec.cr index ef1b78be1bcf..912772ea050b 100644 --- a/spec/std/string/utf16_spec.cr +++ b/spec/std/string/utf16_spec.cr @@ -28,7 +28,7 @@ describe "String UTF16" do it "in the range U+D800..U+DFFF" do encoded = "\u{D800}\u{DFFF}".to_utf16 - encoded.should eq(Slice[0xFFFD_u16, 0xFFFD_u16]) + encoded.should eq(Slice[0xFFFD_u16, 0xFFFD_u16, 0xFFFD_u16, 0xFFFD_u16, 0xFFFD_u16, 0xFFFD_u16]) encoded.unsafe_fetch(encoded.size).should eq 0_u16 end end diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index 0ec443895cc0..8e88d321c348 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -699,6 +699,7 @@ describe "String" do describe "rchop?" do it { "".rchop?.should be_nil } + it { "\n".rchop?.should eq("") } it { "foo".rchop?.should eq("fo") } it { "foo\n".rchop?.should eq("foo") } it { "foo\r".rchop?.should eq("foo") } @@ -831,6 +832,8 @@ describe "String" do it { "foo".index("").should eq(0) } it { "foo".index("foo").should eq(0) } it { "日本語日本語".index("本語").should eq(1) } + it { "\xFF\xFFcrystal".index("crystal").should eq(2) } + it { "\xFD\x9A\xAD\x50NG".index("PNG").should eq(3) } describe "with offset" do it { "foobarbaz".index("ba", 4).should eq(6) } @@ -840,6 +843,8 @@ describe "String" do it { "foo".index("", 3).should eq(3) } it { "foo".index("", 4).should be_nil } it { "日本語日本語".index("本語", 2).should eq(4) } + it { "\xFD\x9A\xAD\x50NG".index("PNG", 2).should eq(3) } + it { "\xFD\x9A\xAD\x50NG".index("PNG", 4).should be_nil } end end @@ -930,57 +935,57 @@ describe "String" do describe "partition" do describe "by char" do - "hello".partition('h').should eq ({"", "h", "ello"}) - "hello".partition('o').should eq ({"hell", "o", ""}) - "hello".partition('l').should eq ({"he", "l", "lo"}) - "hello".partition('x').should eq ({"hello", "", ""}) + it { "hello".partition('h').should eq ({"", "h", "ello"}) } + it { "hello".partition('o').should eq ({"hell", "o", ""}) } + it { "hello".partition('l').should eq ({"he", "l", "lo"}) } + it { "hello".partition('x').should eq ({"hello", "", ""}) } end describe "by string" do - "hello".partition("h").should eq ({"", "h", "ello"}) - "hello".partition("o").should eq ({"hell", "o", ""}) - "hello".partition("l").should eq ({"he", "l", "lo"}) - "hello".partition("ll").should eq ({"he", "ll", "o"}) - "hello".partition("x").should eq ({"hello", "", ""}) + it { "hello".partition("h").should eq ({"", "h", "ello"}) } + it { "hello".partition("o").should eq ({"hell", "o", ""}) } + it { "hello".partition("l").should eq ({"he", "l", "lo"}) } + it { "hello".partition("ll").should eq ({"he", "ll", "o"}) } + it { "hello".partition("x").should eq ({"hello", "", ""}) } end describe "by regex" do - "hello".partition(/h/).should eq ({"", "h", "ello"}) - "hello".partition(/o/).should eq ({"hell", "o", ""}) - "hello".partition(/l/).should eq ({"he", "l", "lo"}) - "hello".partition(/ll/).should eq ({"he", "ll", "o"}) - "hello".partition(/.l/).should eq ({"h", "el", "lo"}) - "hello".partition(/.h/).should eq ({"hello", "", ""}) - "hello".partition(/h./).should eq ({"", "he", "llo"}) - "hello".partition(/o./).should eq ({"hello", "", ""}) - "hello".partition(/.o/).should eq ({"hel", "lo", ""}) - "hello".partition(/x/).should eq ({"hello", "", ""}) + it { "hello".partition(/h/).should eq ({"", "h", "ello"}) } + it { "hello".partition(/o/).should eq ({"hell", "o", ""}) } + it { "hello".partition(/l/).should eq ({"he", "l", "lo"}) } + it { "hello".partition(/ll/).should eq ({"he", "ll", "o"}) } + it { "hello".partition(/.l/).should eq ({"h", "el", "lo"}) } + it { "hello".partition(/.h/).should eq ({"hello", "", ""}) } + it { "hello".partition(/h./).should eq ({"", "he", "llo"}) } + it { "hello".partition(/o./).should eq ({"hello", "", ""}) } + it { "hello".partition(/.o/).should eq ({"hel", "lo", ""}) } + it { "hello".partition(/x/).should eq ({"hello", "", ""}) } end end describe "rpartition" do describe "by char" do - "hello".rpartition('l').should eq ({"hel", "l", "o"}) - "hello".rpartition('o').should eq ({"hell", "o", ""}) - "hello".rpartition('h').should eq ({"", "h", "ello"}) + it { "hello".rpartition('l').should eq ({"hel", "l", "o"}) } + it { "hello".rpartition('o').should eq ({"hell", "o", ""}) } + it { "hello".rpartition('h').should eq ({"", "h", "ello"}) } end describe "by string" do - "hello".rpartition("l").should eq ({"hel", "l", "o"}) - "hello".rpartition("x").should eq ({"", "", "hello"}) - "hello".rpartition("o").should eq ({"hell", "o", ""}) - "hello".rpartition("h").should eq ({"", "h", "ello"}) - "hello".rpartition("ll").should eq ({"he", "ll", "o"}) - "hello".rpartition("lo").should eq ({"hel", "lo", ""}) - "hello".rpartition("he").should eq ({"", "he", "llo"}) + it { "hello".rpartition("l").should eq ({"hel", "l", "o"}) } + it { "hello".rpartition("x").should eq ({"", "", "hello"}) } + it { "hello".rpartition("o").should eq ({"hell", "o", ""}) } + it { "hello".rpartition("h").should eq ({"", "h", "ello"}) } + it { "hello".rpartition("ll").should eq ({"he", "ll", "o"}) } + it { "hello".rpartition("lo").should eq ({"hel", "lo", ""}) } + it { "hello".rpartition("he").should eq ({"", "he", "llo"}) } end describe "by regex" do - "hello".rpartition(/.l/).should eq ({"he", "ll", "o"}) - "hello".rpartition(/ll/).should eq ({"he", "ll", "o"}) - "hello".rpartition(/.o/).should eq ({"hel", "lo", ""}) - "hello".rpartition(/.e/).should eq ({"", "he", "llo"}) - "hello".rpartition(/l./).should eq ({"hel", "lo", ""}) + it { "hello".rpartition(/.l/).should eq ({"he", "ll", "o"}) } + it { "hello".rpartition(/ll/).should eq ({"he", "ll", "o"}) } + it { "hello".rpartition(/.o/).should eq ({"hel", "lo", ""}) } + it { "hello".rpartition(/.e/).should eq ({"", "he", "llo"}) } + it { "hello".rpartition(/l./).should eq ({"hel", "lo", ""}) } end end @@ -1972,25 +1977,31 @@ describe "String" do end end - it "answers ascii_only?" do - "a".ascii_only?.should be_true - "あ".ascii_only?.should be_false + describe "ascii_only?" do + it "answers ascii_only?" do + "a".ascii_only?.should be_true + "あ".ascii_only?.should be_false - str = String.new(1) do |buffer| - buffer.value = 'a'.ord.to_u8 - {1, 0} - end - str.ascii_only?.should be_true + str = String.new(1) do |buffer| + buffer.value = 'a'.ord.to_u8 + {1, 0} + end + str.ascii_only?.should be_true - str = String.new(4) do |buffer| - count = 0 - 'あ'.each_byte do |byte| - buffer[count] = byte - count += 1 + str = String.new(4) do |buffer| + count = 0 + 'あ'.each_byte do |byte| + buffer[count] = byte + count += 1 + end + {count, 0} end - {count, 0} + str.ascii_only?.should be_false + end + + it "broken UTF-8 is not ascii_only" do + "\xED\xA0\x80\xED\xBF\xBF".ascii_only?.should be_false end - str.ascii_only?.should be_false end describe "scan" do @@ -2695,9 +2706,9 @@ describe "String" do it "scrubs" do string = String.new(Bytes[255, 129, 97, 255, 97]) - string.scrub.bytes.should eq([239, 191, 189, 97, 239, 191, 189, 97]) + string.scrub.bytes.should eq([239, 191, 189, 239, 191, 189, 97, 239, 191, 189, 97]) - string.scrub("?").should eq("?a?a") + string.scrub("?").should eq("??a?a") "hello".scrub.should eq("hello") end diff --git a/src/char/reader.cr b/src/char/reader.cr index 6673c814bc20..edc0450b2bf3 100644 --- a/src/char/reader.cr +++ b/src/char/reader.cr @@ -185,65 +185,63 @@ struct Char end private def decode_char_at(pos) - first = byte_at?(pos) || 0u32 + first = byte_at(pos) if first < 0x80 return yield first, 1, nil end if first < 0xc2 - invalid_byte_sequence 1 + invalid_byte_sequence end - second = byte_at?(pos + 1) - if second.nil? || (second & 0xc0) != 0x80 - invalid_byte_sequence 1 + second = byte_at(pos + 1) + if (second & 0xc0) != 0x80 + invalid_byte_sequence end if first < 0xe0 return yield (first << 6) &+ (second &- 0x3080), 2, nil end - third = byte_at?(pos + 2) - if third.nil? || (third & 0xc0) != 0x80 - invalid_byte_sequence 2 + third = byte_at(pos + 2) + if (third & 0xc0) != 0x80 + invalid_byte_sequence end if first < 0xf0 if first == 0xe0 && second < 0xa0 - invalid_byte_sequence 3 + invalid_byte_sequence end if first == 0xed && second >= 0xa0 - invalid_byte_sequence 3 + invalid_byte_sequence end return yield (first << 12) &+ (second << 6) &+ (third &- 0xE2080), 3, nil end if first == 0xf0 && second < 0x90 - invalid_byte_sequence 3 + invalid_byte_sequence end if first == 0xf4 && second >= 0x90 - invalid_byte_sequence 3 + invalid_byte_sequence end - fourth = byte_at?(pos + 3) - if fourth.nil? - invalid_byte_sequence 3 - elsif (fourth & 0xc0) != 0x80 - invalid_byte_sequence 4 + fourth = byte_at(pos + 3) + if (fourth & 0xc0) != 0x80 + invalid_byte_sequence end if first < 0xf5 return yield (first << 18) &+ (second << 12) &+ (third << 6) &+ (fourth &- 0x3C82080), 4, nil end - invalid_byte_sequence 4 + invalid_byte_sequence end - private macro invalid_byte_sequence(width) - return yield Char::REPLACEMENT.ord, {{width}}, first.to_u8 + private macro invalid_byte_sequence + return yield Char::REPLACEMENT.ord, 1, first.to_u8 end @[AlwaysInline] @@ -274,11 +272,7 @@ struct Char end private def byte_at(i) - @string.byte_at(i).to_u32 - end - - private def byte_at?(i) - @string.byte_at?(i).try(&.to_u32) + @string.to_unsafe[i].to_u32 end end end diff --git a/src/path.cr b/src/path.cr index 03b817cde050..f17878cc80f5 100644 --- a/src/path.cr +++ b/src/path.cr @@ -828,7 +828,7 @@ struct Path # Copy the part buffer.copy_from(part_ptr, part_bytesize) - {bytesize, @name.ascii_only? && part.ascii_only? ? bytesize : 0} + {bytesize, @name.single_byte_optimizable? && part.single_byte_optimizable? ? bytesize : 0} end new_instance new_name diff --git a/src/string.cr b/src/string.cr index 08414bdd5056..13108b1b54c3 100644 --- a/src/string.cr +++ b/src/string.cr @@ -784,7 +784,7 @@ class String # Like `#[Int, Int]` but returns `nil` if the *start* index is out of bounds. def []?(start : Int, count : Int) raise ArgumentError.new "Negative count: #{count}" if count < 0 - return byte_slice?(start, count) if ascii_only? + return byte_slice?(start, count) if single_byte_optimizable? start += size if start < 0 @@ -863,7 +863,7 @@ class String # "hello".char_at(-6) { 'x' } # => 'x' # ``` def char_at(index : Int, &) - if ascii_only? + if single_byte_optimizable? byte = byte_at?(index) if byte return byte < 0x80 ? byte.unsafe_chr : Char::REPLACEMENT @@ -988,7 +988,7 @@ class String when size return "" else - if ascii_only? + if single_byte_optimizable? byte_delete_at(index, count, count) else unicode_delete_at(index, count) @@ -1095,7 +1095,7 @@ class String raise ArgumentError.new "Negative count" if count < 0 start += bytesize if start < 0 - single_byte_optimizable = ascii_only? + single_byte_optimizable = single_byte_optimizable? if 0 <= start < bytesize count = bytesize - start if start + count > bytesize @@ -1214,7 +1214,7 @@ class String def downcase(options : Unicode::CaseOptions = :none) : String return self if empty? - if ascii_only? && (options.none? || options.ascii?) + if single_byte_optimizable? && (options.none? || options.ascii?) return String.new(bytesize) do |buffer| bytesize.times do |i| buffer[i] = unsafe_byte_at(i).unsafe_chr.downcase.ord.to_u8 @@ -1249,7 +1249,7 @@ class String def upcase(options : Unicode::CaseOptions = :none) : String return self if empty? - if ascii_only? && (options.none? || options.ascii?) + if single_byte_optimizable? && (options.none? || options.ascii?) return String.new(bytesize) do |buffer| bytesize.times do |i| buffer[i] = unsafe_byte_at(i).unsafe_chr.upcase.ord.to_u8 @@ -1285,7 +1285,7 @@ class String def capitalize(options : Unicode::CaseOptions = :none) : String return self if empty? - if ascii_only? && (options.none? || options.ascii?) + if single_byte_optimizable? && (options.none? || options.ascii?) return String.new(bytesize) do |buffer| bytesize.times do |i| byte = if i.zero? @@ -1331,7 +1331,7 @@ class String def titleize(options : Unicode::CaseOptions = :none) : String return self if empty? - if ascii_only? && (options.none? || options.ascii?) + if single_byte_optimizable? && (options.none? || options.ascii?) upcase_next = true return String.new(bytesize) do |buffer| @@ -1458,7 +1458,7 @@ class String def lchop? : String? return if empty? - if ascii_only? + if single_byte_optimizable? unsafe_byte_slice_string(1, bytesize - 1) else reader = Char::Reader.new(self) @@ -1516,9 +1516,9 @@ class String # "".rchop? # => nil # ``` def rchop? : String? - return if bytesize <= 1 + return if empty? - if to_unsafe[bytesize - 1] < 128 || ascii_only? + if to_unsafe[bytesize - 1] < 0x80 || single_byte_optimizable? return unsafe_byte_slice_string(0, bytesize - 1) end @@ -1646,7 +1646,7 @@ class String bytes, count = String.char_bytes_and_bytesize(other) new_bytesize = bytesize + count - new_size = (ascii_only? && other.ascii?) ? new_bytesize : 0 + new_size = (single_byte_optimizable? && other.ascii?) ? new_bytesize : 0 insert_impl(byte_index, bytes.to_unsafe, count, new_bytesize, new_size) end @@ -1675,7 +1675,7 @@ class String raise IndexError.new unless byte_index new_bytesize = bytesize + other.bytesize - new_size = ascii_only? && other.ascii_only? ? new_bytesize : 0 + new_size = single_byte_optimizable? && other.single_byte_optimizable? ? new_bytesize : 0 insert_impl(byte_index, other.to_unsafe, other.bytesize, new_bytesize, new_size) end @@ -2280,7 +2280,7 @@ class String buffer.value = byte buffer += 1 end - {buffer, ascii_only? ? bytesize - (to_index - from_index) + 1 : 0} + {buffer, single_byte_optimizable? ? bytesize - (to_index - from_index) + 1 : 0} end end @@ -2823,7 +2823,7 @@ class String def compare(other : String, case_insensitive = false, options = Unicode::CaseOptions::None) return self <=> other unless case_insensitive - if ascii_only? && other.ascii_only? + if single_byte_optimizable? && other.single_byte_optimizable? position = 0 while position < bytesize && position < other.bytesize @@ -2992,7 +2992,7 @@ class String # ``` def index(search : Char, offset = 0) # If it's ASCII we can delegate to slice - if search.ascii? && ascii_only? + if search.ascii? && single_byte_optimizable? return to_slice.index(search.ord.to_u8, offset) end @@ -3030,16 +3030,8 @@ class String pointer = to_unsafe end_pointer = pointer + bytesize while char_index < offset && pointer < end_pointer - byte = pointer.value - if byte < 0x80 - pointer += 1 - elsif byte < 0xe0 - pointer += 2 - elsif byte < 0xf0 - pointer += 3 - else - pointer += 4 - end + char_bytesize = String.char_bytesize_at(pointer) + pointer += char_bytesize char_index += 1 end @@ -3063,18 +3055,14 @@ class String return if pointer >= end_pointer byte = head_pointer.value - - # update a rolling hash of this text (heystack) - # thanks @MaxLap for suggesting this loop reduction - if byte < 0x80 - update_hash 1 - elsif byte < 0xe0 - update_hash 2 - elsif byte < 0xf0 - update_hash 3 - else - update_hash 4 + char_bytesize = String.char_bytesize_at(head_pointer) + case char_bytesize + when 1 then update_hash 1 + when 2 then update_hash 2 + when 3 then update_hash 3 + else update_hash 4 end + char_index += 1 end end @@ -3099,7 +3087,7 @@ class String # ``` def rindex(search : Char, offset = size - 1) # If it's ASCII we can delegate to slice - if search.ascii? && ascii_only? + if search.ascii? && single_byte_optimizable? return to_slice.rindex(search.ord.to_u8, offset) end @@ -3387,7 +3375,7 @@ class String # "こんにちは".char_index_to_byte_index(5) # => 15 # ``` def char_index_to_byte_index(index) - if ascii_only? + if single_byte_optimizable? return 0 <= index <= bytesize ? index : nil end @@ -3403,7 +3391,7 @@ class String # It is valid to pass `#bytesize` to *index*, and in this case the answer # will be the size of this string. def byte_index_to_char_index(index) - if ascii_only? + if single_byte_optimizable? return 0 <= index <= bytesize ? index : nil end @@ -3475,7 +3463,7 @@ class String end yielded = 0 - single_byte_optimizable = ascii_only? + single_byte_optimizable = single_byte_optimizable? index = 0 i = 0 looking_for_space = false @@ -3666,7 +3654,7 @@ class String byte_offset = 0 separator_bytesize = separator.bytesize - single_byte_optimizable = ascii_only? + single_byte_optimizable = single_byte_optimizable? i = 0 stop = bytesize - separator.bytesize + 1 @@ -3988,7 +3976,7 @@ class String def reverse return self if bytesize <= 1 - if ascii_only? + if single_byte_optimizable? String.new(bytesize) do |buffer| bytesize.times do |i| buffer[i] = self.to_unsafe[bytesize - i - 1] @@ -4371,7 +4359,7 @@ class String # array # => ['a', 'b', '☃'] # ``` def each_char : Nil - if ascii_only? + if single_byte_optimizable? each_byte do |byte| yield (byte < 0x80 ? byte.unsafe_chr : Char::REPLACEMENT) end @@ -4752,7 +4740,7 @@ class String def ends_with?(char : Char) : Bool return false unless bytesize > 0 - if char.ascii? || ascii_only? + if char.ascii? || single_byte_optimizable? return to_unsafe[bytesize - 1] == char.ord end @@ -4819,6 +4807,18 @@ class String # "你好".ascii_only? # => false # ``` def ascii_only? + if @bytesize == size + each_byte do |byte| + return false unless byte < 0x80 + end + true + else + false + end + end + + # :nodoc: + def single_byte_optimizable? @bytesize == size end @@ -4858,43 +4858,67 @@ class String end protected def char_bytesize_at(byte_index) - first = unsafe_byte_at(byte_index) + String.char_bytesize_at(to_unsafe + byte_index) + end + + protected def self.char_bytesize_at(bytes : Pointer(UInt8)) + first = bytes.value if first < 0x80 return 1 end if first < 0xc2 - return 1 + return 1 # Invalid end - second = unsafe_byte_at(byte_index + 1) + second = bytes[1] + if (second & 0xc0) != 0x80 - return 1 + return 1 # Invalid end if first < 0xe0 return 2 end - third = unsafe_byte_at(byte_index + 2) + third = bytes[2] + if (third & 0xc0) != 0x80 - return 2 + return 1 # Invalid end if first < 0xf0 + if first == 0xe0 && second < 0xa0 + return 1 # Invalid + end + + if first == 0xed && second >= 0xa0 + return 1 # Invalid + end + return 3 end if first == 0xf0 && second < 0x90 - return 3 + return 1 # Invalid end if first == 0xf4 && second >= 0x90 - return 3 + return 1 # Invalid + end + + fourth = bytes[3] + + if (fourth & 0xc0) != 0x80 + return 1 # Invalid + end + + if first < 0xf5 + return 4 end - return 4 + 1 # Invalid end # :nodoc: From 2cb80ce39957f2efeb1b526f7bbc5bba46237a7d Mon Sep 17 00:00:00 2001 From: proxy <51172302+3n-k1@users.noreply.github.com> Date: Mon, 31 Aug 2020 17:13:21 -0400 Subject: [PATCH 223/263] fix spelling mistake in http/server.cr (#9717) --- src/http/server.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/server.cr b/src/http/server.cr index ee19e61666e8..ca92675a673f 100644 --- a/src/http/server.cr +++ b/src/http/server.cr @@ -73,7 +73,7 @@ require "log" # # ## Binding to sockets # -# The server can be bound to one ore more server sockets (see `#bind`) +# The server can be bound to one or more server sockets (see `#bind`) # # Supported types: # From 0a628699d673d01ab9cc0fc3720fabead1435d2e Mon Sep 17 00:00:00 2001 From: proxy <51172302+3n-k1@users.noreply.github.com> Date: Mon, 31 Aug 2020 17:14:24 -0400 Subject: [PATCH 224/263] make __crystal_raise_overflow error more clear (#9686) * make __crystal_raise_overflow error more clear * use LocationlessException --- src/compiler/crystal/codegen/codegen.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index 09eef8806dc0..e0caadad5f37 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -2036,7 +2036,7 @@ module Crystal if raise_overflow_fun = @raise_overflow_fun check_main_fun RAISE_OVERFLOW_NAME, raise_overflow_fun else - raise "BUG: __crystal_raise_overflow is not defined" + raise LocationlessException.new("Missing __crystal_raise_overflow function, either use std-lib's prelude or define it") end end From d918fc09c9c19c81c2b5378e55e1b1a28d16d673 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 1 Sep 2020 12:41:07 -0300 Subject: [PATCH 225/263] Complex: unary plus returns `self` (#9719) --- spec/std/complex_spec.cr | 2 +- src/complex.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/std/complex_spec.cr b/spec/std/complex_spec.cr index 568961d4d7f6..ddad2a571cac 100644 --- a/spec/std/complex_spec.cr +++ b/spec/std/complex_spec.cr @@ -118,7 +118,7 @@ describe "Complex" do describe "+" do it "+ complex" do - (+Complex.new(-5.43, -27.12)).should eq(Complex.new(5.43, 27.12)) + (+Complex.new(-5.43, -27.12)).should eq(Complex.new(-5.43, -27.12)) end it "complex + complex" do diff --git a/src/complex.cr b/src/complex.cr index 44772543fb7b..292cbf6f733c 100644 --- a/src/complex.cr +++ b/src/complex.cr @@ -225,9 +225,9 @@ struct Complex log / Math::LOG10 end - # Returns absolute value of `self`. + # Returns `self`. def + - Complex.new(@real.abs, @imag.abs) + self end # Adds the value of `self` to *other*. From 208140f702877140d5850d6f5a2b3124b91b51fb Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Wed, 2 Sep 2020 10:12:18 -0300 Subject: [PATCH 226/263] Parser: fix `def self./` and `def self.%` (#9721) * Parser: fix `def self.\` and `def self.%` * Make sure def with and without parens are tests for all operators --- spec/compiler/parser/parser_spec.cr | 6 +++--- src/compiler/crystal/syntax/parser.cr | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index 73b9730a7f27..c9a4b7d2c0d8 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -469,11 +469,11 @@ module Crystal ["/", "<", "<=", "==", "!=", "=~", "!~", ">", ">=", "+", "-", "*", "/", "~", "%", "&", "|", "^", "**", "==="].each do |op| it_parses "def #{op}; end;", Def.new(op) + it_parses "def #{op}(); end;", Def.new(op) + it_parses "def self.#{op}; end;", Def.new(op, receiver: "self".var) + it_parses "def self.#{op}(); end;", Def.new(op, receiver: "self".var) end - it_parses "def %(); end;", Def.new("%") - it_parses "def /(); end;", Def.new("/") - ["<<", "<", "<=", "==", ">>", ">", ">=", "+", "-", "*", "/", "//", "%", "|", "&", "^", "**", "===", "=~", "!~", "&+", "&-", "&*", "&**"].each do |op| it_parses "1 #{op} 2", Call.new(1.int32, op, 2.int32) it_parses "n #{op} 2", Call.new("n".call, op, 2.int32) diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index b71c27e4c65b..bdc3ab64e817 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -3416,7 +3416,8 @@ module Crystal raise "shouldn't reach this line" end end - next_token_skip_space + + consume_def_or_macro_name if @token.type == :IDENT check_valid_def_name From 5c536e7d8cc4c98319d976920b8d2f24daf5e4f7 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 3 Sep 2020 15:39:24 -0300 Subject: [PATCH 227/263] MachO: Handle missing LC_UUID (#9706) --- src/crystal/mach_o.cr | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/crystal/mach_o.cr b/src/crystal/mach_o.cr index 02d906cc7ef3..8209452fd3b4 100644 --- a/src/crystal/mach_o.cr +++ b/src/crystal/mach_o.cr @@ -441,11 +441,15 @@ module Crystal end def uuid - @uuid ||= seek_to(LoadCommand::UUID) do - bytes = uninitialized UInt8[16] - @io.read_fully(bytes.to_slice) - UUID.new(bytes) - end.not_nil! + @uuid ||= begin + # ld has a -no_version_load_command options to suppress the LC_UUID entry. + # The Dwarf file generated in such scenario has a LC_UUID = 00000000-0000-0000-0000-000000000000. + seek_to(LoadCommand::UUID) do + bytes = uninitialized UInt8[16] + @io.read_fully(bytes.to_slice) + UUID.new(bytes) + end || UUID.new(StaticArray(UInt8, 16).new(0)) + end end def symbols From 03259ea21ba218f1b6613d6e7108a6c85db92ff5 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 4 Sep 2020 14:22:48 -0300 Subject: [PATCH 228/263] Bug/formatter newline after comment before end (#9722) * Formatter: don't add extra newlines after comment before end * Run crystal tool format --- spec/compiler/formatter/formatter_spec.cr | 66 +++++++++++++++++++++-- src/compiler/crystal/tools/formatter.cr | 34 ++++++------ src/spec.cr | 1 - 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 2d7a0423278d..bb07ee7dad48 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -219,12 +219,12 @@ describe Crystal::Formatter do assert_format "def foo(a : T) forall T \n #\nend", "def foo(a : T) forall T\n #\nend" assert_format "def foo(a : T, b : U) forall T, U\n #\nend", "def foo(a : T, b : U) forall T, U\n #\nend" assert_format "def foo(a : T, b : U) forall T, U #\n #\nend", "def foo(a : T, b : U) forall T, U #\n #\nend" - assert_format "def foo(a : T) forall T\n #\n\nend", "def foo(a : T) forall T\n #\n\nend" - assert_format "def foo(a : T) forall T\n #\n\n\nend", "def foo(a : T) forall T\n #\n\nend" + assert_format "def foo(a : T) forall T\n #\n\nend", "def foo(a : T) forall T\n #\nend" + assert_format "def foo(a : T) forall T\n #\n\n\nend", "def foo(a : T) forall T\n #\nend" assert_format "def foo\n 1\n #\nrescue\nend" - assert_format "def foo\n 1 #\n\nrescue\nend" assert_format "def foo\n 1 #\nrescue\nend" - assert_format "def foo\n 1\n #\n\n\nrescue\nend", "def foo\n 1\n #\n\nrescue\nend" + assert_format "def foo\n 1 #\nrescue\nend" + assert_format "def foo\n 1\n #\n\n\nrescue\nend", "def foo\n 1\n #\nrescue\nend" assert_format "loop do\n 1\nrescue\n 2\nend" assert_format "loop do\n 1\n loop do\n 2\n rescue\n 3\n end\n 4\nend" @@ -1667,4 +1667,62 @@ describe Crystal::Formatter do 1 # foo / #{1} / CODE + + assert_format <<-BEFORE, + def foo + # Comment + + + end + BEFORE + <<-AFTER + def foo + # Comment + end + AFTER + + assert_format <<-BEFORE, + def foo + 1 + # Comment + + + end + BEFORE + <<-AFTER + def foo + 1 + # Comment + end + AFTER + + assert_format <<-CODE + def foo + 1 + end + + # Comment + + def bar + 2 + end + CODE + + assert_format <<-CODE + require "foo" + + @x : Int32 + + class Bar + end + CODE + + assert_format <<-CODE + x = <<-FOO + hello + FOO + + def bar + end + CODE end diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 60c3858f3441..b98a6797fb73 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -287,8 +287,14 @@ module Crystal else if needs_two_lines unless found_comment - skip_space_write_line - found_comment = skip_space_or_newline last: true, at_least_one: true + if @wrote_newline + write_line unless @wrote_double_newlines + elsif !@wrote_double_newlines + write_line + write_line + end + @wrote_double_newlines = true + found_comment = skip_space_or_newline write_line unless found_comment || @wrote_double_newlines end else @@ -1374,7 +1380,7 @@ module Crystal def format_nested(node, indent = @indent, write_end_line = true, write_indent = true) slash_is_regex! if node.is_a?(Nop) - skip_nop(indent + 2) + skip_space_write_line else if write_indent indent(indent + 2) do @@ -4420,7 +4426,7 @@ module Crystal indent(indent) { skip_space(write_comma) } end - def skip_space_or_newline(last : Bool = false, at_least_one : Bool = false) + def skip_space_or_newline(last : Bool = false, at_least_one : Bool = false, next_comes_end : Bool = false) just_wrote_line = @wrote_newline base_column = @column has_space = false @@ -4453,15 +4459,15 @@ module Crystal write_line end end - write_comment(needs_indent: !needs_space) + write_comment(needs_indent: !needs_space, next_comes_end: last) true else false end end - def skip_space_or_newline(indent : Int32, last : Bool = false, at_least_one : Bool = false) - indent(indent) { skip_space_or_newline(last, at_least_one) } + def skip_space_or_newline(indent : Int32, last : Bool = false, at_least_one : Bool = false, next_comes_end : Bool = false) + indent(indent) { skip_space_or_newline(last, at_least_one, next_comes_end) } end def slash_is_regex! @@ -4478,11 +4484,6 @@ module Crystal found_comment end - def skip_nop(indent) - skip_space_write_line - skip_space_or_newline(indent) - end - def skip_semicolon while @token.type == :";" next_token @@ -4523,7 +4524,7 @@ module Crystal skip_space_or_newline end - def write_comment(needs_indent = true, consume_newline = true) + def write_comment(needs_indent = true, consume_newline = true, next_comes_end = false) while @token.type == :COMMENT empty_line = @line_output.to_s.strip.empty? if empty_line @@ -4576,18 +4577,18 @@ module Crystal write value next_token_skip_space if consume_newline - consume_newlines + consume_newlines(next_comes_end: next_comes_end) skip_space_or_newline end end end - def consume_newlines + def consume_newlines(next_comes_end = false) if @token.type == :NEWLINE write_line unless @wrote_newline next_token_skip_space - if @token.type == :NEWLINE + if @token.type == :NEWLINE && !next_comes_end write_line @wrote_double_newlines = true end @@ -4660,6 +4661,7 @@ module Crystal end @wrote_newline = false + @wrote_double_newlines = false @last_write = string end diff --git a/src/spec.cr b/src/spec.cr index 15d0f2121a3b..c8290f673d0a 100644 --- a/src/spec.cr +++ b/src/spec.cr @@ -28,7 +28,6 @@ require "./spec/dsl" # end # # # lots more specs -# # end # ``` # From c11be8fb71374d5e5abe1bcc9d8c5b31adde2172 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 4 Sep 2020 14:23:20 -0300 Subject: [PATCH 229/263] Compiler: don't form a closure on `typeof(@ivar)` (#9723) --- spec/compiler/semantic/closure_spec.cr | 25 +++++++++++++++++++ src/compiler/crystal/semantic/main_visitor.cr | 2 ++ 2 files changed, 27 insertions(+) diff --git a/spec/compiler/semantic/closure_spec.cr b/spec/compiler/semantic/closure_spec.cr index 69f26dec829f..809ebca9fda1 100644 --- a/spec/compiler/semantic/closure_spec.cr +++ b/spec/compiler/semantic/closure_spec.cr @@ -524,4 +524,29 @@ describe "Semantic: closure" do ), "can't send closure to C function (closured vars: x)" end + + it "doesn't closure typeof local var" do + result = assert_type("x = 1; -> { typeof(x) }; x") { int32 } + program = result.program + var = program.vars["x"] + var.closured?.should be_false + end + + it "doesn't closure typeof instance var (#9479)" do + result = assert_type(" + class Foo + @x : Int32? + + def foo + -> { typeof(@x) } + end + end + + Foo.new.foo + 1 + ") { int32 } + node = result.node.as(Expressions) + call = node.expressions[-2].as(Call) + call.target_def.self_closured?.should be_false + end end diff --git a/src/compiler/crystal/semantic/main_visitor.cr b/src/compiler/crystal/semantic/main_visitor.cr index db8fb8d856ac..720ae39a5397 100644 --- a/src/compiler/crystal/semantic/main_visitor.cr +++ b/src/compiler/crystal/semantic/main_visitor.cr @@ -3212,6 +3212,8 @@ module Crystal end def check_self_closured + return if @typeof_nest > 0 + scope = @scope return unless scope From 1ef9c845ef82ec3f557fb3c32fc0e3d9075ec777 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 4 Sep 2020 14:24:09 -0300 Subject: [PATCH 230/263] Encoding: remove unnecessary and wrong optimization (#9724) --- spec/std/io/io_spec.cr | 19 +++++++++++++++++++ src/io/encoding.cr | 8 -------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 3b5bf8d175e2..893d9ac8fb43 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -1,5 +1,6 @@ require "../spec_helper" require "../../support/channel" +require "socket" {% unless flag?(:win32) %} require "big" @@ -767,6 +768,24 @@ describe IO do io.read_string(11).should eq("Hello world") io.gets_to_end.should eq("\r\nFoo\nBar") end + + it "gets ascii from socket (#9056)" do + server = TCPServer.new "localhost", 0 + sock = TCPSocket.new "localhost", server.local_address.port + begin + sock.set_encoding("ascii") + spawn do + client = server.accept + message = client.gets + client << "#{message}\n" + end + sock << "K\n" + sock.gets.should eq("K") + ensure + server.close + sock.close + end + end end describe "encode" do diff --git a/src/io/encoding.cr b/src/io/encoding.cr index 174029fbb31c..07c7661afd73 100644 --- a/src/io/encoding.cr +++ b/src/io/encoding.cr @@ -80,14 +80,6 @@ class IO @in_buffer_left = LibC::SizeT.new(io.read(@buffer)) end - # If we just have a few bytes to decode, read more, just in case these don't produce a character - if @in_buffer_left < 16 - buffer_remaining = BUFFER_SIZE - @in_buffer_left - (@in_buffer - @buffer.to_unsafe) - @buffer.copy_from(@in_buffer, @in_buffer_left) - @in_buffer = @buffer.to_unsafe - @in_buffer_left += LibC::SizeT.new(io.read(Slice.new(@in_buffer + @in_buffer_left, buffer_remaining))) - end - # If, after refilling the buffer, we couldn't read new bytes # it means we reached the end break if @in_buffer_left == 0 From dfbd6afbe291a63913eff86fff4f6c8f2be0d679 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 4 Sep 2020 14:24:40 -0300 Subject: [PATCH 231/263] Fix sprintf for zero left padding a negative number (#9725) --- spec/std/string_spec.cr | 14 ++++++++++++++ src/string/formatter.cr | 13 ++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/spec/std/string_spec.cr b/spec/std/string_spec.cr index aba4c6bb5929..9ba90a5b0284 100644 --- a/spec/std/string_spec.cr +++ b/spec/std/string_spec.cr @@ -1722,17 +1722,31 @@ describe "String" do ("%+i" % -123).should eq("-123") ("% i" % 123).should eq(" 123") ("%20d" % 123).should eq(" 123") + ("%20d" % -123).should eq(" -123") + ("%20d" % 0).should eq(" 0") ("%+20d" % 123).should eq(" +123") ("%+20d" % -123).should eq(" -123") + ("%+20d" % 0).should eq(" +0") ("% 20d" % 123).should eq(" 123") ("%020d" % 123).should eq("00000000000000000123") + ("%020d" % -123).should eq("0000000000000000-123") + ("%020d" % 0).should eq("00000000000000000000") ("%+020d" % 123).should eq("+0000000000000000123") + ("%+020d" % -123).should eq("-0000000000000000123") + ("%+020d" % 0).should eq("+0000000000000000000") ("% 020d" % 123).should eq(" 0000000000000000123") + ("% 020d" % 0).should eq(" 0000000000000000000") ("%-d" % 123).should eq("123") + ("%-d" % 0).should eq("0") ("%-20d" % 123).should eq("123 ") + ("%-20d" % -123).should eq("-123 ") + ("%-20d" % 0).should eq("0 ") ("%-+20d" % 123).should eq("+123 ") ("%-+20d" % -123).should eq("-123 ") + ("%-+20d" % 0).should eq("+0 ") ("%- 20d" % 123).should eq(" 123 ") + ("%- 20d" % -123).should eq("-123 ") + ("%- 20d" % 0).should eq(" 0 ") ("%s" % 'a').should eq("a") ("%-s" % 'a').should eq("a") ("%20s" % 'a').should eq(" a") diff --git a/src/string/formatter.cr b/src/string/formatter.cr index 34a16cb44907..39cf6250f7ce 100644 --- a/src/string/formatter.cr +++ b/src/string/formatter.cr @@ -229,14 +229,21 @@ struct String::Formatter(A) if flags.left_padding? if flags.padding_char == '0' - @io << '+' if flags.plus + if flags.plus + if int >= 0 + @io << '+' + else + @io << '-' + int = int.abs + end + end @io << ' ' if flags.space end pad_int int, flags end - if int > 0 + if int >= 0 unless flags.padding_char == '0' @io << '+' if flags.plus @io << ' ' if flags.space @@ -312,7 +319,7 @@ struct String::Formatter(A) def pad_int(int, flags) size = int.to_s(flags.base).bytesize - size += 1 if int > 0 && (flags.plus || flags.space) + size += 1 if int >= 0 && (flags.plus || flags.space) pad size, flags end From 634af9fcb1d8cc115763679d450950814fc8c839 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 4 Sep 2020 14:25:08 -0300 Subject: [PATCH 232/263] HTTP::Server: don't override content-lenght if already set (#9726) --- spec/std/http/server/response_spec.cr | 8 ++++++++ src/http/server/response.cr | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/std/http/server/response_spec.cr b/spec/std/http/server/response_spec.cr index 33bd0aa70ada..0042f40c008f 100644 --- a/spec/std/http/server/response_spec.cr +++ b/spec/std/http/server/response_spec.cr @@ -91,6 +91,14 @@ describe HTTP::Server::Response do io.to_s.should eq("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n1234567890") end + it "doesn't override content-length when there's no body" do + io = IO::Memory.new + response = Response.new(io) + response.content_length = 10 + response.close + io.to_s.should eq("HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n") + end + it "adds header" do io = IO::Memory.new response = Response.new(io) diff --git a/src/http/server/response.cr b/src/http/server/response.cr index 86268cabd95b..a2651339fcca 100644 --- a/src/http/server/response.cr +++ b/src/http/server/response.cr @@ -223,7 +223,7 @@ class HTTP::Server def close return if closed? - unless response.wrote_headers? + if !response.wrote_headers? && !response.headers.has_key?("Content-Length") response.content_length = @out_count end From e6498c49406c6b23192ec78caac2f240bd97c6e0 Mon Sep 17 00:00:00 2001 From: Julien Reichardt Date: Fri, 4 Sep 2020 19:25:46 +0200 Subject: [PATCH 233/263] Remove Enumerable#grep (#9711) --- spec/std/enumerable_spec.cr | 10 ---------- src/enumerable.cr | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/spec/std/enumerable_spec.cr b/spec/std/enumerable_spec.cr index 8cc3290968c3..1ca3cc3dace4 100644 --- a/spec/std/enumerable_spec.cr +++ b/spec/std/enumerable_spec.cr @@ -503,16 +503,6 @@ describe "Enumerable" do end end - describe "grep" do - it "works with regexes for instance" do - ["Alice", "Bob", "Cipher", "Anna"].grep(/^A/).should eq ["Alice", "Anna"] - end - - it "returns empty array if nothing matches" do - %w(Alice Bob Mallory).grep(/nothing/).should eq [] of String - end - end - describe "group_by" do it { [1, 2, 2, 3].group_by { |x| x == 2 }.should eq({true => [2, 2], false => [1, 3]}) } diff --git a/src/enumerable.cr b/src/enumerable.cr index 135ccde206e2..f80e23b6cb25 100644 --- a/src/enumerable.cr +++ b/src/enumerable.cr @@ -535,17 +535,6 @@ module Enumerable(T) ary end - # Returns an `Array` with all the elements in the collection that - # match the `RegExp` *pattern*. - # - # ``` - # ["Alice", "Bob"].grep(/^A/) # => ["Alice"] - # ``` - @[Deprecated("Use `#select` instead")] - def grep(pattern) - self.select { |elem| pattern === elem } - end - # Returns a `Hash` whose keys are each different value that the passed block # returned when run for each element in the collection, and which values are # an `Array` of the elements for which the block returned that value. From 52c750ece75211117f4d86a694d774b3317e61c6 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Sat, 5 Sep 2020 22:56:23 +0900 Subject: [PATCH 234/263] Parser: apply string pieces combination even if heredoc has no indent (#9475) Fixed #9473 It is important to simplify the formatter implementation. --- spec/compiler/formatter/formatter_spec.cr | 2 ++ src/compiler/crystal/syntax/parser.cr | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index bb07ee7dad48..0185c1cd0c5d 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -999,6 +999,8 @@ describe Crystal::Formatter do assert_format "<<-FOO\n#{"foo"}bar\nFOO" assert_format "<<-FOO\nbar#{"foo"}\nFOO" assert_format "<<-FOO\nbar#{"foo"}bar\nFOO" + assert_format "<<-FOO\nfoo\n#{"foo"}\nFOO" + assert_format "<<-FOO\nfoo\n#{1}\nFOO" assert_format "#!shebang\n1 + 2" diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index bdc3ab64e817..448b6fefc61c 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -2145,7 +2145,7 @@ module Crystal end def needs_heredoc_indent_removed?(delimiter_state) - delimiter_state.kind == :heredoc && delimiter_state.heredoc_indent > 0 + delimiter_state.kind == :heredoc && delimiter_state.heredoc_indent >= 0 end def remove_heredoc_indent(pieces : Array, indent) From 0c926c06f9db91b78cf3bdd4156ee7c4bd7a2f39 Mon Sep 17 00:00:00 2001 From: Kubo Takehiro Date: Fri, 11 Sep 2020 04:28:17 +0900 Subject: [PATCH 235/263] Enable large-file support on i386-linux-gnu (#9478) Without large-file support `File.open`, `File.info`, `Dir#read` fail for files whose size is larger than 2G or whose inode number is larger than 4G on 32-bit linux. This commit enables large-file support by using type definitions, function names and structure members when _FILE_OFFSET_BITS=64 is defined in C. --- src/lib_c/i386-linux-gnu/c/dirent.cr | 4 ++-- src/lib_c/i386-linux-gnu/c/fcntl.cr | 2 +- src/lib_c/i386-linux-gnu/c/stdlib.cr | 4 ++-- src/lib_c/i386-linux-gnu/c/sys/mman.cr | 2 +- src/lib_c/i386-linux-gnu/c/sys/stat.cr | 11 +++++------ src/lib_c/i386-linux-gnu/c/sys/types.cr | 6 +++--- src/lib_c/i386-linux-gnu/c/unistd.cr | 8 ++++---- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/lib_c/i386-linux-gnu/c/dirent.cr b/src/lib_c/i386-linux-gnu/c/dirent.cr index 3cb61b21129e..e4674e3bf0f4 100644 --- a/src/lib_c/i386-linux-gnu/c/dirent.cr +++ b/src/lib_c/i386-linux-gnu/c/dirent.cr @@ -7,7 +7,7 @@ lib LibC struct Dirent d_ino : InoT - d_off : Long + d_off : OffT d_reclen : UShort d_type : Char d_name : StaticArray(Char, 256) @@ -15,6 +15,6 @@ lib LibC fun closedir(dirp : DIR*) : Int fun opendir(name : Char*) : DIR* - fun readdir(dirp : DIR*) : Dirent* + fun readdir = readdir64(dirp : DIR*) : Dirent* fun rewinddir(dirp : DIR*) : Void end diff --git a/src/lib_c/i386-linux-gnu/c/fcntl.cr b/src/lib_c/i386-linux-gnu/c/fcntl.cr index 1a8fd2f1787b..2a12d9b4857f 100644 --- a/src/lib_c/i386-linux-gnu/c/fcntl.cr +++ b/src/lib_c/i386-linux-gnu/c/fcntl.cr @@ -28,5 +28,5 @@ lib LibC end fun fcntl(fd : Int, cmd : Int, ...) : Int - fun open(file : Char*, oflag : Int, ...) : Int + fun open = open64(file : Char*, oflag : Int, ...) : Int end diff --git a/src/lib_c/i386-linux-gnu/c/stdlib.cr b/src/lib_c/i386-linux-gnu/c/stdlib.cr index 66abbaf4743d..46ffee257b84 100644 --- a/src/lib_c/i386-linux-gnu/c/stdlib.cr +++ b/src/lib_c/i386-linux-gnu/c/stdlib.cr @@ -13,8 +13,8 @@ lib LibC fun free(ptr : Void*) : Void fun getenv(name : Char*) : Char* fun malloc(size : SizeT) : Void* - fun mkstemp(template : Char*) : Int - fun mkstemps(template : Char*, suffixlen : Int) : Int + fun mkstemp = mkstemp64(template : Char*) : Int + fun mkstemps = mkstemps64(template : Char*, suffixlen : Int) : Int fun putenv(string : Char*) : Int fun realloc(ptr : Void*, size : SizeT) : Void* fun realpath(name : Char*, resolved : Char*) : Char* diff --git a/src/lib_c/i386-linux-gnu/c/sys/mman.cr b/src/lib_c/i386-linux-gnu/c/sys/mman.cr index 8c44b210a24e..158228f6946d 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/mman.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/mman.cr @@ -24,7 +24,7 @@ lib LibC MADV_HUGEPAGE = 14 MADV_NOHUGEPAGE = 15 - fun mmap(addr : Void*, len : SizeT, prot : Int, flags : Int, fd : Int, offset : OffT) : Void* + fun mmap = mmap64(addr : Void*, len : SizeT, prot : Int, flags : Int, fd : Int, offset : OffT) : Void* fun mprotect(addr : Void*, len : SizeT, prot : Int) : Int fun munmap(addr : Void*, len : SizeT) : Int fun madvise(addr : Void*, len : SizeT, advice : Int) : Int diff --git a/src/lib_c/i386-linux-gnu/c/sys/stat.cr b/src/lib_c/i386-linux-gnu/c/sys/stat.cr index cf2edc189e71..2ad9f557dfae 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/stat.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/stat.cr @@ -29,7 +29,7 @@ lib LibC struct Stat st_dev : DevT __pad1 : UShort - st_ino : InoT + __st_ino : ULong st_mode : ModeT st_nlink : NlinkT st_uid : UidT @@ -42,16 +42,15 @@ lib LibC st_atim : Timespec st_mtim : Timespec st_ctim : Timespec - __unused4 : ULong - __unused5 : ULong + st_ino : InoT end fun chmod(file : Char*, mode : ModeT) : Int - fun fstat(fd : Int, buf : Stat*) : Int - fun lstat(file : Char*, buf : Stat*) : Int + fun fstat = fstat64(fd : Int, buf : Stat*) : Int + fun lstat = lstat64(file : Char*, buf : Stat*) : Int fun mkdir(path : Char*, mode : ModeT) : Int fun mkfifo(path : Char*, mode : ModeT) : Int fun mknod(path : Char*, mode : ModeT, dev : DevT) : Int - fun stat(file : Char*, buf : Stat*) : Int + fun stat = stat64(file : Char*, buf : Stat*) : Int fun umask(mask : ModeT) : ModeT end diff --git a/src/lib_c/i386-linux-gnu/c/sys/types.cr b/src/lib_c/i386-linux-gnu/c/sys/types.cr index 7a1618b91cf2..f89d29723e65 100644 --- a/src/lib_c/i386-linux-gnu/c/sys/types.cr +++ b/src/lib_c/i386-linux-gnu/c/sys/types.cr @@ -2,17 +2,17 @@ require "../stddef" require "../stdint" lib LibC - alias BlkcntT = Long + alias BlkcntT = LongLong alias BlksizeT = Long alias ClockT = Long alias ClockidT = Int alias DevT = ULongLong alias GidT = UInt alias IdT = UInt - alias InoT = ULong + alias InoT = ULongLong alias ModeT = UInt alias NlinkT = UInt - alias OffT = Long + alias OffT = LongLong alias PidT = Int union PthreadAttrT diff --git a/src/lib_c/i386-linux-gnu/c/unistd.cr b/src/lib_c/i386-linux-gnu/c/unistd.cr index c83a01f81ffa..3cd2f6d71cdc 100644 --- a/src/lib_c/i386-linux-gnu/c/unistd.cr +++ b/src/lib_c/i386-linux-gnu/c/unistd.cr @@ -21,7 +21,7 @@ lib LibC @[ReturnsTwice] fun fork : PidT fun fsync(fd : Int) : Int - fun ftruncate(fd : Int, length : OffT) : Int + fun ftruncate = ftruncate64(fd : Int, length : OffT) : Int fun getcwd(buf : Char*, size : SizeT) : Char* fun gethostname(name : Char*, len : SizeT) : Int fun getpgid(pid : PidT) : PidT @@ -31,11 +31,11 @@ lib LibC fun ttyname_r(fd : Int, buf : Char*, buffersize : SizeT) : Int fun lchown(file : Char*, owner : UidT, group : GidT) : Int fun link(from : Char*, to : Char*) : Int - fun lockf(fd : Int, cmd : Int, len : OffT) : Int - fun lseek(fd : Int, offset : OffT, whence : Int) : OffT + fun lockf = lockf64(fd : Int, cmd : Int, len : OffT) : Int + fun lseek = lseek64(fd : Int, offset : OffT, whence : Int) : OffT fun pipe(pipedes : StaticArray(Int, 2)) : Int fun read(fd : Int, buf : Void*, nbytes : SizeT) : SSizeT - fun pread(x0 : Int, x1 : Void*, x2 : SizeT, x3 : OffT) : SSizeT + fun pread = pread64(x0 : Int, x1 : Void*, x2 : SizeT, x3 : OffT) : SSizeT fun rmdir(path : Char*) : Int fun symlink(from : Char*, to : Char*) : Int fun readlink(path : Char*, buf : Char*, size : SizeT) : SSizeT From 4e34ca15d753d844c5093416c70776ad584a3f64 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 11 Sep 2020 08:44:36 -0300 Subject: [PATCH 236/263] Formatter: correctly format named arguments like `foo:bar` (#9740) --- spec/compiler/formatter/formatter_spec.cr | 1 + src/compiler/crystal/syntax/lexer.cr | 1 + src/compiler/crystal/tools/formatter.cr | 2 ++ 3 files changed, 4 insertions(+) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 0185c1cd0c5d..176de863ef03 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -277,6 +277,7 @@ describe Crystal::Formatter do assert_format "foo 1,\n2", "foo 1,\n 2" assert_format "foo 1, a: 1,\nb: 2,\nc: 3", "foo 1, a: 1,\n b: 2,\n c: 3" assert_format "foo 1,\na: 1,\nb: 2,\nc: 3", "foo 1,\n a: 1,\n b: 2,\n c: 3" + assert_format "foo bar:baz, qux:other", "foo bar: baz, qux: other" assert_format "foo(\n 1, 2, &block)", "foo(\n 1, 2, &block)" assert_format "foo(\n 1, 2,\n&block)", "foo(\n 1, 2,\n &block)" assert_format "foo(\n 1,\n 2\n) do\n 1\nend" diff --git a/src/compiler/crystal/syntax/lexer.cr b/src/compiler/crystal/syntax/lexer.cr index a867370815b6..0c26ff6af905 100644 --- a/src/compiler/crystal/syntax/lexer.cr +++ b/src/compiler/crystal/syntax/lexer.cr @@ -14,6 +14,7 @@ module Crystal getter token : Token property line_number : Int32 property column_number : Int32 + property wants_symbol : Bool @filename : String | VirtualFile | Nil @stacked_filename : String | VirtualFile | Nil @token_end_location : Location? diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index b98a6797fb73..c3f77c2fb72d 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -985,7 +985,9 @@ module Crystal StringLiteral.new(name).accept self else write @token + @lexer.wants_symbol = false next_token + @lexer.wants_symbol = true end end From d6db9f007f31c966bf73a7e7ba4ccf3a22f6d284 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Thu, 17 Sep 2020 11:05:44 -0300 Subject: [PATCH 237/263] Initial nix development environment. Use Nix for OSX CI instead of homebrew. (#9727) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial nix development environment Extract latest from release compiler binary and libgc.a only. That is enough to use bin/crystal with the new std-lib. Allow bin/ci to run nix-shell when CI_NIX_SHELL is set Use nix instead of brew for CI Darwin in GitHub * Update shell.nix Co-authored-by: Dorian Marié * Build GC for nix development environment. Overrides the existing boehmgc package to apply patch needed for mt support. * Build gc with libatomics + mt patch * Allow custom system to use 32 bits easily Co-authored-by: Dorian Marié Co-authored-by: Kim Burgess --- .github/workflows/macos.yml | 10 ++- bin/ci | 26 +++++-- shell.nix | 151 ++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 8 deletions(-) create mode 100644 shell.nix diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 9303f001bbfc..0f10ac993bc4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -3,10 +3,8 @@ name: macOS CI on: [push, pull_request] env: - TRAVIS_OS_NAME: osx - LLVM_CONFIG: /usr/local/opt/llvm/bin/llvm-config - PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig SPEC_SPLIT_DOTS: 160 + CI_NIX_SHELL: true jobs: test_macos: @@ -15,6 +13,12 @@ jobs: - name: Download Crystal source uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v10 + - uses: cachix/cachix-action@v6 + with: + name: crystal-ci + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + - name: Prepare System run: bin/ci prepare_system diff --git a/bin/ci b/bin/ci index 221760bf3f11..5151b9acb640 100755 --- a/bin/ci +++ b/bin/ci @@ -46,12 +46,16 @@ on_os() { os="$1" shift - verify_environment + if [ -z "$CI_NIX_SHELL" ]; then + verify_environment - if [ "$TRAVIS_OS_NAME" = "$os" ]; then - echo "${@}" - eval "${@}" - return $? + if [ "$TRAVIS_OS_NAME" = "$os" ]; then + echo "${@}" + eval "${@}" + return $? + else + return 0 + fi else return 0 fi @@ -65,6 +69,16 @@ on_osx() { fail_on_error on_os "osx" "${@}" } +on_nix_shell() { + if [ -n "$CI_NIX_SHELL" ]; then + echo "${@}" + eval "${@}" + return $? + else + return 0 + fi +} + on_github() { if [ "$GITHUB_ACTIONS" = "true" ]; then eval "${@}" @@ -182,6 +196,8 @@ with_build_env() { CRYSTAL_CACHE_DIR="/tmp/crystal" \ /bin/sh -c "'$command'" + on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --run "'TZ=$TZ $command'" + on_github echo "::endgroup::" } diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..060bc3f9cc5a --- /dev/null +++ b/shell.nix @@ -0,0 +1,151 @@ +# This nix-shell script can be used to get a complete development environment +# for the Crystal compiler. +# +# You can choose which llvm version use and, on Linux, choose to use musl. +# +# $ nix-shell --pure +# $ nix-shell --pure --arg llvm 10 +# $ nix-shell --pure --arg llvm 10 --arg musl true +# $ nix-shell --pure --arg llvm 9 +# $ nix-shell --pure --arg llvm 9 --argstr system i686-linux +# ... +# $ nix-shell --pure --arg llvm 6 +# +# If needed, you can use https://app.cachix.org/cache/crystal-ci to avoid building +# packages that are not available in Nix directly. This is mostly useful for musl. +# +# $ nix-env -iA cachix -f https://cachix.org/api/v1/install +# $ cachix use crystal-ci +# $ nix-shell --pure --arg musl true +# + +{llvm ? 10, musl ? false, system ? builtins.currentSystem}: + +let + nixpkgs = import (builtins.fetchTarball { + name = "nixpkgs-20.03"; + url = "https://github.com/NixOS/nixpkgs/archive/2d580cd2793a7b5f4b8b6b88fb2ccec700ee1ae6.tar.gz"; + sha256 = "1nbanzrir1y0yi2mv70h60sars9scwmm0hsxnify2ldpczir9n37"; + }) { + inherit system; + }; + + pkgs = if musl then nixpkgs.pkgsMusl else nixpkgs; + + genericBinary = { url, sha256 }: + pkgs.stdenv.mkDerivation rec { + name = "crystal-binary"; + src = builtins.fetchTarball { inherit url sha256; }; + + # Extract only the compiler binary + buildCommand = '' + mkdir -p $out/bin + + # Darwin packages use embedded/bin/crystal + [ ! -f "${src}/embedded/bin/crystal" ] || cp ${src}/embedded/bin/crystal $out/bin/ + + # Linux packages use lib/crystal/bin/crystal + [ ! -f "${src}/lib/crystal/bin/crystal" ] || cp ${src}/lib/crystal/bin/crystal $out/bin/ + ''; + }; + + # Hashes obtained using `nix-prefetch-url --unpack ` + latestCrystalBinary = genericBinary ({ + x86_64-darwin = { + url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-darwin-x86_64.tar.gz"; + sha256 = "sha256:0gpn42xh372hw2bqfgxc9wibpbam8gn7gx3p1b8p9adydjg0zxfm"; + }; + + x86_64-linux = { + url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-x86_64.tar.gz"; + sha256 = "sha256:077pby4ylf0z831gg0hbiwxcq3g0yl0cdlybirgg8rv24a2sa7zh"; + }; + + i686-linux = { + url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-i686.tar.gz"; + sha256 = "sha256:0nfgxjndfslyacicjy4303pvvqfg74v5fnpr4b10ss9rqakmlbgd"; + }; + }.${pkgs.stdenv.system}); + + pkgconfig = pkgs.pkgconfig; + + llvm_suite = ({ + llvm_10 = { + llvm = pkgs.llvm_10; + extra = [ pkgs.lld_10 pkgs.lldb_10 ]; + }; + llvm_9 = { + llvm = pkgs.llvm_9; + extra = [ ]; # lldb it fails to compile on Darwin + }; + llvm_8 = { + llvm = pkgs.llvm_8; + extra = [ ]; # lldb it fails to compile on Darwin + }; + llvm_7 = { + llvm = pkgs.llvm; + extra = [ pkgs.lldb ]; + }; + llvm_6 = { + llvm = pkgs.llvm_6; + extra = [ ]; # lldb it fails to compile on Darwin + }; + }."llvm_${toString llvm}"); + + libatomic_ops = builtins.fetchurl { + url = "https://github.com/ivmai/libatomic_ops/releases/download/v7.6.10/libatomic_ops-7.6.10.tar.gz"; + sha256 = "1bwry043f62pc4mgdd37zx3fif19qyrs8f5bw7qxlmkzh5hdyzjq"; + }; + + boehmgc = pkgs.stdenv.mkDerivation rec { + pname = "boehm-gc"; + version = "8.0.4"; + + src = builtins.fetchTarball { + url = "https://github.com/ivmai/bdwgc/releases/download/v${version}/gc-${version}.tar.gz"; + sha256 = "16ic5dwfw51r5lcl88vx3qrkg3g2iynblazkri3sl9brnqiyzjk7"; + }; + + patches = [ + (pkgs.fetchpatch { + url = "https://github.com/ivmai/bdwgc/commit/5668de71107022a316ee967162bc16c10754b9ce.patch"; + sha256 = "02f0rlxl4fsqk1xiq0pabkhwydnmyiqdik2llygkc6ixhxbii8xw"; + }) + ]; + + postUnpack = '' + mkdir $sourceRoot/libatomic_ops + tar -xzf ${libatomic_ops} -C $sourceRoot/libatomic_ops --strip-components 1 + ''; + + configureFlags = [ + "--disable-debug" + "--disable-dependency-tracking" + "--disable-shared" + "--enable-large-config" + ]; + + enableParallelBuilding = true; + }; + + stdLibDeps = with pkgs; [ + boehmgc gmp libevent libiconv libxml2 libyaml openssl pcre zlib + ] ++ stdenv.lib.optionals stdenv.isDarwin [ libiconv ]; + + tools = [ pkgs.hostname llvm_suite.extra ]; +in + +pkgs.stdenv.mkDerivation rec { + name = "crystal-dev"; + + buildInputs = tools ++ stdLibDeps ++ [ + latestCrystalBinary + pkgconfig + llvm_suite.llvm + ]; + + LLVM_CONFIG = "${llvm_suite.llvm}/bin/llvm-config"; + + # ld: warning: object file (.../src/ext/libcrystal.a(sigfault.o)) was built for newer OSX version (10.14) than being linked (10.12) + MACOSX_DEPLOYMENT_TARGET = "10.11"; +} From 135e9fbc335a11e39d4132267095c7640f4dafd5 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 18 Sep 2020 09:18:08 -0300 Subject: [PATCH 238/263] Disable specs added in #9724 on windows (#9758) They were failing with a In src\socket.cr:1:1 1 | require "c/arpa/inet" ^ Error: can't find file 'c/arpa/inet' --- spec/std/io/io_spec.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/std/io/io_spec.cr b/spec/std/io/io_spec.cr index 893d9ac8fb43..b740b114789a 100644 --- a/spec/std/io/io_spec.cr +++ b/spec/std/io/io_spec.cr @@ -1,8 +1,8 @@ require "../spec_helper" require "../../support/channel" -require "socket" {% unless flag?(:win32) %} + require "socket" require "big" {% end %} require "base64" @@ -769,7 +769,7 @@ describe IO do io.gets_to_end.should eq("\r\nFoo\nBar") end - it "gets ascii from socket (#9056)" do + pending_win32 "gets ascii from socket (#9056)" do server = TCPServer.new "localhost", 0 sock = TCPSocket.new "localhost", server.local_address.port begin From 400e925fa54d7d3842826ccfa3288f3998910e5b Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 18 Sep 2020 17:26:22 -0300 Subject: [PATCH 239/263] CI: Disable darwin ci on CircleCI (#9763) In GitHub Action the darwin ci is using Nix. The CircleCI alternative still uses brew and llvm@10 (10.0.1) is not usable as is. Most likely the CI for PR will be moved entirely to GitHub Actions and CircleCI will remain for nightly and releases builds, at least, for while. --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0bad7067d7cb..efa12ff9db15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -551,8 +551,8 @@ workflows: filters: *unless_maintenance - test_alpine: filters: *unless_maintenance - - test_darwin: - filters: *unless_maintenance + # - test_darwin: + # filters: *unless_maintenance - test_preview_mt: filters: *unless_maintenance - check_format: @@ -577,8 +577,8 @@ workflows: filters: *per_tag - test_alpine: filters: *per_tag - - test_darwin: - filters: *per_tag + # - test_darwin: + # filters: *per_tag - test_preview_mt: filters: *per_tag - check_format: @@ -652,7 +652,7 @@ workflows: - test_linux - test_linux32_std - test_alpine - - test_darwin + # - test_darwin - test_preview_mt - check_format - prepare_common @@ -716,8 +716,8 @@ workflows: filters: *maintenance - test_alpine: filters: *maintenance - - test_darwin: - filters: *maintenance + # - test_darwin: + # filters: *maintenance - test_preview_mt: filters: *maintenance - check_format: From d9b757adecca4a8a33fb86813cef39e3426d8006 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 25 Sep 2020 14:49:31 -0300 Subject: [PATCH 240/263] Add HTTP::Client logging and basic instrumentation (#9756) * Add HTTP::Client logging and basic instrumentation This adds protected methods HTTP::Client#before_exec and #after_exec that can be used to hook into the lifecycle of the http client. A first utility for this is adding a http.client log source to know a request is starting. This enables some basic instrumentation to monitor the activity of the application without monkey patching sensitive parts of the http client * Replace with use def_around_exec macro * Manually expand macro to allow simple usage of def_around_exec * Add docs --- spec/std/http/client/client_spec.cr | 34 ++++++++++++++++++++++ src/http.cr | 1 + src/http/client.cr | 44 +++++++++++++++++++++++++++-- src/http/log.cr | 19 +++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 src/http/log.cr diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index e7dc6443b0b7..e9802761d9de 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -3,6 +3,7 @@ require "../../socket/spec_helper" require "openssl" require "http/client" require "http/server" +require "log/spec" private def test_server(host, port, read_time = 0, content_type = "text/plain", write_response = true) server = TCPServer.new(host, port) @@ -300,5 +301,38 @@ module HTTP client.get("/") end end + + describe "logging" do + it "emit logs" do + test_server("localhost", 0, content_type: "") do |server| + client = Client.new("localhost", server.local_address.port) + Log.capture("http.client") do |logs| + client.get("/") + + logs.check(:debug, "Performing request") + logs.entry.data[:method].should eq("GET") + logs.entry.data[:host].should eq("localhost") + logs.entry.data[:port].should eq(server.local_address.port) + logs.entry.data[:resource].should eq("/") + end + end + end + + it "emit logs with block" do + test_server("localhost", 0, content_type: "") do |server| + Client.new("localhost", server.local_address.port) do |client| + Log.capture("http.client") do |logs| + client.get("/") do |response| + logs.check(:debug, "Performing request") + logs.entry.data[:method].should eq("GET") + logs.entry.data[:host].should eq("localhost") + logs.entry.data[:port].should eq(server.local_address.port) + logs.entry.data[:resource].should eq("/") + end + end + end + end + end + end end end diff --git a/src/http.cr b/src/http.cr index facbdd2f897c..33cbc7895b39 100644 --- a/src/http.cr +++ b/src/http.cr @@ -2,6 +2,7 @@ require "uri" {% unless flag?(:win32) %} require "./http/client" require "./http/server" + require "./http/log" {% end %} require "./http/common" diff --git a/src/http/client.cr b/src/http/client.cr index bc032ac0b986..822a63e43de4 100644 --- a/src/http/client.cr +++ b/src/http/client.cr @@ -577,7 +577,9 @@ class HTTP::Client # response.body # => "..." # ``` def exec(request : HTTP::Request) : HTTP::Client::Response - exec_internal(request) + around_exec(request) do + exec_internal(request) + end end private def exec_internal(request) @@ -615,8 +617,10 @@ class HTTP::Client # end # ``` def exec(request : HTTP::Request, &block) - exec_internal(request) do |response| - yield response + around_exec(request) do + exec_internal(request) do |response| + yield response + end end end @@ -862,6 +866,40 @@ class HTTP::Client yield client, path end end + + # This method is called when executing the request. Although it can be + # redefined, it is recommended to use the `def_around_exec` macro to be + # able to add new behaviors without loosing prior existing ones. + protected def around_exec(request) + yield + end + + # This macro allows injecting code to be run before and after the execution + # of the request. It should return the yielded value. It must be called with 1 + # block argument that will be used to pass the `HTTP::Request`. + # + # ``` + # class HTTP::Client + # def_around_exec do |request| + # # do something before exec + # res = yield + # # do something after exec + # res + # end + # end + # ``` + macro def_around_exec(&block) + protected def around_exec(%request) + previous_def do + {% if block.args.size != 1 %} + {% raise "Wrong number of block arguments (given #{block.args.size}, expected: 1)" %} + {% end %} + + {{ block.args.first.id }} = %request + {{ block.body }} + end + end + end end require "socket" diff --git a/src/http/log.cr b/src/http/log.cr new file mode 100644 index 000000000000..7f16c4c63e90 --- /dev/null +++ b/src/http/log.cr @@ -0,0 +1,19 @@ +require "log" + +class HTTP::Client + Log = ::Log.for(self) + + def_around_exec do |request| + emit_log(request) + yield + end + + protected def emit_log(request) + Log.debug &.emit("Performing request", + method: request.method, + host: host, + port: port, + resource: request.resource, + ) + end +end From c09544937e717dac01c55e8bfa6b683f3f49620a Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 29 Sep 2020 11:06:33 -0300 Subject: [PATCH 241/263] Detect failures in Nix build environment (#9776) * CI: properly forward failure in nix-shell * Disable failing spec in Darwin/Nix CI environment * Add git to shell.nix for --pure environments --- bin/ci | 6 +++++- shell.nix | 2 +- spec/std/http/client/client_spec.cr | 27 ++++++++++++++++----------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/bin/ci b/bin/ci index 5151b9acb640..9ca230c12048 100755 --- a/bin/ci +++ b/bin/ci @@ -69,7 +69,7 @@ on_osx() { fail_on_error on_os "osx" "${@}" } -on_nix_shell() { +on_nix_shell_eval() { if [ -n "$CI_NIX_SHELL" ]; then echo "${@}" eval "${@}" @@ -79,6 +79,10 @@ on_nix_shell() { fi } +on_nix_shell() { + fail_on_error on_nix_shell_eval "${@}" +} + on_github() { if [ "$GITHUB_ACTIONS" = "true" ]; then eval "${@}" diff --git a/shell.nix b/shell.nix index 060bc3f9cc5a..00f6bd7f695d 100644 --- a/shell.nix +++ b/shell.nix @@ -132,7 +132,7 @@ let boehmgc gmp libevent libiconv libxml2 libyaml openssl pcre zlib ] ++ stdenv.lib.optionals stdenv.isDarwin [ libiconv ]; - tools = [ pkgs.hostname llvm_suite.extra ]; + tools = [ pkgs.hostname pkgs.git llvm_suite.extra ]; in pkgs.stdenv.mkDerivation rec { diff --git a/spec/std/http/client/client_spec.cr b/spec/std/http/client/client_spec.cr index e9802761d9de..4af7437bdc43 100644 --- a/spec/std/http/client/client_spec.cr +++ b/spec/std/http/client/client_spec.cr @@ -225,19 +225,24 @@ module HTTP end end - it "tests write_timeout" do - # Here we don't want to write a response on the server side because - # it doesn't make sense to try to write because the client will already - # timeout on read. Writing a response could lead on an exception in - # the server if the socket is closed. - test_server("localhost", 0, 0, write_response: false) do |server| - client = Client.new("localhost", server.local_address.port) - expect_raises(IO::TimeoutError, "Write timed out") do - client.write_timeout = 0.001 - client.post("/", body: "a" * 5_000_000) + {% unless flag?(:darwin) %} + # TODO the following spec is failing on Nix Darwin CI when executed + # together with some other tests. If run alone it succeeds. + # The exhibit failure is a Failed to raise an exception: END_OF_STACK. + it "tests write_timeout" do + # Here we don't want to write a response on the server side because + # it doesn't make sense to try to write because the client will already + # timeout on read. Writing a response could lead on an exception in + # the server if the socket is closed. + test_server("localhost", 0, 0, write_response: false) do |server| + client = Client.new("localhost", server.local_address.port) + expect_raises(IO::TimeoutError, "Write timed out") do + client.write_timeout = 0.001 + client.post("/", body: "a" * 5_000_000) + end end end - end + {% end %} it "tests connect_timeout" do test_server("localhost", 0, 0) do |server| From e64e87215aeb3610d032bcde00e5b157bdd54661 Mon Sep 17 00:00:00 2001 From: Matthew McGarvey Date: Tue, 29 Sep 2020 09:14:50 -0500 Subject: [PATCH 242/263] Fix bug for c structs assigned to a type (#9743) * Fix bug for c structs assigned to a type * Call Type#remove_typedef instead of creating a separate method to handle typedefs * Reassign type argument --- spec/compiler/codegen/c_struct_spec.cr | 15 +++++++++++++++ src/compiler/crystal/codegen/codegen.cr | 1 + 2 files changed, 16 insertions(+) diff --git a/spec/compiler/codegen/c_struct_spec.cr b/spec/compiler/codegen/c_struct_spec.cr index 4d93e967b3f5..93e16018052c 100644 --- a/spec/compiler/codegen/c_struct_spec.cr +++ b/spec/compiler/codegen/c_struct_spec.cr @@ -379,4 +379,19 @@ describe "Code gen: struct" do foo.x.call(1) )).to_i.should eq(2) end + + it "can access member of uninitialized struct behind type (#8774)" do + run(%( + lib LibFoo + struct Foo + x : Int32 + end + + type FooT = Foo + end + + foo = uninitialized LibFoo::FooT + foo.x + )) + end end diff --git a/src/compiler/crystal/codegen/codegen.cr b/src/compiler/crystal/codegen/codegen.cr index e0caadad5f37..3a47dcc74c52 100644 --- a/src/compiler/crystal/codegen/codegen.cr +++ b/src/compiler/crystal/codegen/codegen.cr @@ -1187,6 +1187,7 @@ module Crystal end def read_instance_var(node_type, type, name, value) + type = type.remove_typedef ivar = type.lookup_instance_var(name) ivar_ptr = instance_var_ptr type, name, value @last = downcast ivar_ptr, node_type, ivar.type, false From 4c729411d7ffc0be71ae87bd167567558644642c Mon Sep 17 00:00:00 2001 From: Hugo Parente Lima Date: Tue, 29 Sep 2020 11:16:57 -0300 Subject: [PATCH 243/263] Fix example in the doc of `Crystal.main`. (#9736) Fixes #9688 --- src/crystal/main.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/main.cr b/src/crystal/main.cr index a5c7ed00b369..641d1dd1402d 100644 --- a/src/crystal/main.cr +++ b/src/crystal/main.cr @@ -79,7 +79,7 @@ module Crystal # # ``` # fun main(argc : Int32, argv : UInt8**) : Int32 - # LibFoo.init_foo_and_invoke_main(argc, argv, ->Crystal.main) + # LibFoo.init_foo_and_invoke_main(argc, argv, ->Crystal.main(Int32, UInt8**)) # end # ``` # From 31be12baa42aec4520e6a075c60b1e11cac9052f Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 29 Sep 2020 12:04:20 -0300 Subject: [PATCH 244/263] Drop RemoteAddressType private alias (#9777) --- .../http/server/handlers/log_handler_spec.cr | 32 +++++++-------- src/http/request.cr | 40 ++++++++----------- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/spec/std/http/server/handlers/log_handler_spec.cr b/spec/std/http/server/handlers/log_handler_spec.cr index 08d26e0a9300..c2a15c385626 100644 --- a/spec/std/http/server/handlers/log_handler_spec.cr +++ b/spec/std/http/server/handlers/log_handler_spec.cr @@ -5,24 +5,24 @@ require "../../../../support/io" require "../../../../support/retry" describe HTTP::LogHandler do - it "logs" do - io = IO::Memory.new - request = HTTP::Request.new("GET", "/") - {% if flag?(:win32) %} - request.remote_address = "192.168.0.1" - {% else %} + {% unless flag?(:win32) %} + # TODO: Remove this once `Socket` is working on Windows + + it "logs" do + io = IO::Memory.new + request = HTTP::Request.new("GET", "/") request.remote_address = Socket::IPAddress.new("192.168.0.1", 1234) - {% end %} - response = HTTP::Server::Response.new(io) - context = HTTP::Server::Context.new(request, response) + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) - called = false - handler = HTTP::LogHandler.new - handler.next = ->(ctx : HTTP::Server::Context) { called = true } - logs = Log.capture("http.server") { handler.call(context) } - logs.check(:info, %r(^192.168.0.1 - GET / HTTP/1.1 - 200 \(\d+(\.\d+)?[mµn]s\)$)) - called.should be_true - end + called = false + handler = HTTP::LogHandler.new + handler.next = ->(ctx : HTTP::Server::Context) { called = true } + logs = Log.capture("http.server") { handler.call(context) } + logs.check(:info, %r(^192.168.0.1 - GET / HTTP/1.1 - 200 \(\d+(\.\d+)?[mµn]s\)$)) + called.should be_true + end + {% end %} it "logs to custom logger" do request = HTTP::Request.new("GET", "/") diff --git a/src/http/request.cr b/src/http/request.cr index ab95dc6c1c2f..063ece74d0a4 100644 --- a/src/http/request.cr +++ b/src/http/request.cr @@ -2,16 +2,6 @@ require "./common" require "uri" require "http/params" -# TODO: Remove this once `Socket` is working on Windows -{% begin %} -private alias RemoteAddressType = - {% if flag?(:win32) %} - String? - {% else %} - Socket::Address? - {% end %} -{% end %} - # An HTTP request. # # It serves both to perform requests by an `HTTP::Client` and to @@ -30,14 +20,20 @@ class HTTP::Request @query_params : Params? @uri : URI? - # The network address that sent the request to an HTTP server. - # - # `HTTP::Server` will try to fill this property, and its value - # will have a format like "IP:port", but this format is not guaranteed. - # Middlewares can overwrite this value. - # - # This property is not used by `HTTP::Client`. - property remote_address : RemoteAddressType + {% unless flag?(:win32) %} + # The network address that sent the request to an HTTP server. + # + # `HTTP::Server` will try to fill this property, and its value + # will have a format like "IP:port", but this format is not guaranteed. + # Middlewares can overwrite this value. + # + # This property is not used by `HTTP::Client`. + property remote_address : Socket::Address? + {% else %} + # TODO: Remove this once `Socket` is working on Windows + + property remote_address : Nil + {% end %} def self.new(method : String, resource : String, headers : Headers? = nil, body : String | Bytes | IO | Nil = nil, version = "HTTP/1.1") # Duplicate headers to prevent the request from modifying data that the user might hold. @@ -119,11 +115,9 @@ class HTTP::Request # No need to dup headers since nobody else holds them request = new line.method, line.resource, headers, body, line.http_version, internal: nil - {% unless flag?(:win32) %} - if io.responds_to?(:remote_address) - request.remote_address = io.remote_address - end - {% end %} + if io.responds_to?(:remote_address) + request.remote_address = io.remote_address + end return request end From f095adc67efd91225419cec61da3ea0fd59f3b66 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 29 Sep 2020 12:05:34 -0300 Subject: [PATCH 245/263] Formatter: handle comment before do in a separate line (#9762) --- spec/compiler/formatter/formatter_spec.cr | 13 ++++++++++++ src/compiler/crystal/tools/formatter.cr | 25 ++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index 176de863ef03..e9b98f9d17c4 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -1728,4 +1728,17 @@ describe Crystal::Formatter do def bar end CODE + + assert_format <<-CODE + foo 1, # comment + do + end + CODE + + assert_format <<-CODE + foo 1, # comment + # bar + do + end + CODE end diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index c3f77c2fb72d..1ee111617518 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -2906,18 +2906,37 @@ module Crystal old_inside_call_or_assign = @inside_call_or_assign @inside_call_or_assign = 0 + comma_before_comment = false + if @token.type == :"," - needs_comma = true - next_token_skip_space_or_newline + next_token + next_token if @token.type == :SPACE + if @token.type == :COMMENT + write "," + needs_comma = false + comma_before_comment = true + @indent += 2 + else + needs_comma = true + end + skip_space_or_newline + @indent -= 2 if comma_before_comment end if @token.keyword?(:do) - write " do" + if comma_before_comment + @indent += 2 + write_indent + else + write " " + end + write "do" next_token_skip_space body = format_block_args node.args, node old_implicit_exception_handler_indent, @implicit_exception_handler_indent = @implicit_exception_handler_indent, @indent format_nested_with_end body @implicit_exception_handler_indent = old_implicit_exception_handler_indent + @indent -= 2 elsif @token.type == :"{" write "," if needs_comma write " {" From d8baec4ded218796804ea85d0b2ea2912330e4a1 Mon Sep 17 00:00:00 2001 From: Ibraheem Ahmed Date: Tue, 29 Sep 2020 11:31:50 -0400 Subject: [PATCH 246/263] Add documentation for HTTP::WebSocket (#9761) * Added docs for HTTP::WebSocket public methods * fix typo Co-authored-by: Julien Reichardt * fix typo Co-authored-by: Julien Reichardt Co-authored-by: Julien Reichardt --- src/http/web_socket.cr | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/http/web_socket.cr b/src/http/web_socket.cr index 18ddca94433d..7950d11422a0 100644 --- a/src/http/web_socket.cr +++ b/src/http/web_socket.cr @@ -48,18 +48,23 @@ class HTTP::WebSocket new(Protocol.new(host, path, port, tls, headers)) end + # Called when the server sends a ping to a client. def on_ping(&@on_ping : String ->) end + # Called when the server receives a pong from a client. def on_pong(&@on_pong : String ->) end + # Called when the server receives a text message from a client. def on_message(&@on_message : String ->) end + # Called when the server receives a binary message from a client. def on_binary(&@on_binary : Bytes ->) end + # Called when the server closes a client's connection. def on_close(&@on_close : CloseCode, String ->) end @@ -67,6 +72,7 @@ class HTTP::WebSocket raise IO::Error.new "Closed socket" if closed? end + # Sends a message payload (message) to the client. def send(message) check_open @ws.send(message) @@ -102,12 +108,29 @@ class HTTP::WebSocket close(nil, message) end + # Sends a close frame to the client, and closes the connection. + # The close frame may contain a body (message) that indicates the reason for closing. def close(code : CloseCode | Int? = nil, message = nil) return if closed? @closed = true @ws.close(code, message) end + # Continuously receives messages and calls previously set callbacks until the websocket is closed. + # Ping and pong messages are automatically handled. + # + # ``` + # # Open websocket connection + # ws = WebSocket.new(uri) + # + # # Set callback + # ws.on_message do |msg| + # ws.send "response" + # end + # + # # Start infinite loop + # ws.run + # ``` def run loop do begin From bdbd386858bd020768ca74eb8b75ecf88103b908 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 29 Sep 2020 12:32:10 -0300 Subject: [PATCH 247/263] Codegen: promote C variadic args as needed (#9747) --- spec/compiler/codegen/c_abi/c_abi_spec.cr | 46 +++++++++++++++++++++++ src/compiler/crystal/codegen/call.cr | 19 +++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/spec/compiler/codegen/c_abi/c_abi_spec.cr b/spec/compiler/codegen/c_abi/c_abi_spec.cr index 17e885628c32..0cbeb5ecd765 100644 --- a/spec/compiler/codegen/c_abi/c_abi_spec.cr +++ b/spec/compiler/codegen/c_abi/c_abi_spec.cr @@ -199,4 +199,50 @@ describe "Code gen: C ABI" do (str.x + str.y + str.z).to_i32 ), &.to_i.should eq(6)) end + + it "promotes variadic args (float to double)" do + test_c( + %( + #include + + double foo(int n, ...) { + va_list args; + va_start(args, n); + return va_arg(args, double); + } + ), + %( + lib LibFoo + fun foo(n : Int32, ...) : Float64 + end + + LibFoo.foo(1, 1.0_f32) + ), &.to_f64.should eq(1.0)) + end + + [{"i8", -123}, + {"u8", 255}, + {"i16", -123}, + {"u16", 65535}, + ].each do |int_kind, int_value| + it "promotes variadic args (#{int_kind} to i32) (#9742)" do + test_c( + %( + #include + + int foo(int n, ...) { + va_list args; + va_start(args, n); + return va_arg(args, int); + } + ), + %( + lib LibFoo + fun foo(n : Int32, ...) : Int32 + end + + LibFoo.foo(1, #{int_value}_#{int_kind}) + ), &.to_i.should eq(int_value)) + end + end end diff --git a/src/compiler/crystal/codegen/call.cr b/src/compiler/crystal/codegen/call.cr index 4016827f50d5..65f58123d89c 100644 --- a/src/compiler/crystal/codegen/call.cr +++ b/src/compiler/crystal/codegen/call.cr @@ -166,6 +166,8 @@ class Crystal::CodeGenVisitor call_args << sret_value end + target_def_args_size = target_def.args.size + node.args.each_with_index do |arg, i| if arg.is_a?(Out) has_out = true @@ -217,13 +219,26 @@ class Crystal::CodeGenVisitor case abi_arg_type.kind when LLVM::ABI::ArgKind::Direct call_arg = codegen_direct_abi_call(call_arg, abi_arg_type) unless arg.type.nil_type? - call_args << call_arg when LLVM::ABI::ArgKind::Indirect # Pass argument as is (will be passed byval) - call_args << call_arg when LLVM::ABI::ArgKind::Ignore # Ignore + next + end + + # If we are passing variadic arguments there are some special rules + if i >= target_def_args_size + arg_type = arg.type.remove_indirection + if arg_type.is_a?(FloatType) && arg_type.kind == :f32 + # Floats must be passed as doubles (there are no float varargs) + call_arg = extend_float @program.float64, call_arg + elsif arg_type.is_a?(IntegerType) && arg_type.kind.in?(:i8, :u8, :i16, :u16) + # Integer with a size less that `int` must be converted to `int` + call_arg = extend_int arg_type, @program.int32, call_arg + end end + + call_args << call_arg end @needs_value = old_needs_value From 5d1a7224256c08e62098f3cd4186ca2407a61678 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 29 Sep 2020 14:40:30 -0300 Subject: [PATCH 248/263] Fix formatting issue (#9778) Follow-up #9761 --- src/http/web_socket.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/http/web_socket.cr b/src/http/web_socket.cr index 7950d11422a0..1770c42e9d8a 100644 --- a/src/http/web_socket.cr +++ b/src/http/web_socket.cr @@ -118,11 +118,11 @@ class HTTP::WebSocket # Continuously receives messages and calls previously set callbacks until the websocket is closed. # Ping and pong messages are automatically handled. - # + # # ``` # # Open websocket connection # ws = WebSocket.new(uri) - # + # # # Set callback # ws.on_message do |msg| # ws.send "response" From b34dd43ce8a5759e036a4986256592d398d6c2e9 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Tue, 29 Sep 2020 14:40:48 -0300 Subject: [PATCH 249/263] Update-distribution scripts (#9710) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index efa12ff9db15..dd53ff928afb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - run: | git clone https://github.com/crystal-lang/distribution-scripts.git ~/distribution-scripts cd ~/distribution-scripts - git checkout 44172615fa196fb8592046582471fdfc8d69472c + git checkout e1704da2a215264ea8e101eb15d4ec1baae98222 # persist relevant information for build process - run: | cd ~/distribution-scripts From d97fe3f41461acc6bdeca4485e1f3c1ced9853de Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Wed, 30 Sep 2020 10:30:57 -0300 Subject: [PATCH 250/263] YAML: correctly serialize infinity and NaN (#9780) --- spec/std/yaml/serialization_spec.cr | 34 +++++++++++++++++++++++++++++ src/yaml/to_yaml.cr | 15 +++++++++++++ 2 files changed, 49 insertions(+) diff --git a/spec/std/yaml/serialization_spec.cr b/spec/std/yaml/serialization_spec.cr index 85dec4579c20..8f3b0ed42805 100644 --- a/spec/std/yaml/serialization_spec.cr +++ b/spec/std/yaml/serialization_spec.cr @@ -72,13 +72,19 @@ describe "YAML serialization" do it "does Float32#from_yaml" do Float32.from_yaml("1.5").should eq(1.5_f32) + Float32.from_yaml(".nan").nan?.should be_true Float32.from_yaml(".inf").should eq(Float32::INFINITY) + Float32.from_yaml("-.inf").should eq(-Float32::INFINITY) end it "does Float64#from_yaml" do value = Float64.from_yaml("1.5") value.should eq(1.5) value.should be_a(Float64) + + Float64.from_yaml(".nan").nan?.should be_true + Float64.from_yaml(".inf").should eq(Float64::INFINITY) + Float64.from_yaml("-.inf").should eq(-Float64::INFINITY) end it "does Array#from_yaml" do @@ -276,10 +282,38 @@ describe "YAML serialization" do Int32.from_yaml(1.to_yaml).should eq(1) end + it "does for Float32" do + Float32.from_yaml(1.5_f32.to_yaml).should eq(1.5_f32) + end + + it "does for Float32 (infinity)" do + Float32.from_yaml(Float32::INFINITY.to_yaml).should eq(Float32::INFINITY) + end + + it "does for Float32 (-infinity)" do + Float32.from_yaml((-Float32::INFINITY).to_yaml).should eq(-Float32::INFINITY) + end + + it "does for Float32 (nan)" do + Float32.from_yaml(Float32::NAN.to_yaml).nan?.should be_true + end + it "does for Float64" do Float64.from_yaml(1.5.to_yaml).should eq(1.5) end + it "does for Float64 (infinity)" do + Float64.from_yaml(Float64::INFINITY.to_yaml).should eq(Float64::INFINITY) + end + + it "does for Float64 (-infinity)" do + Float64.from_yaml((-Float64::INFINITY).to_yaml).should eq(-Float64::INFINITY) + end + + it "does for Float64 (nan)" do + Float64.from_yaml(Float64::NAN.to_yaml).nan?.should be_true + end + it "does for String" do String.from_yaml("hello".to_yaml).should eq("hello") end diff --git a/src/yaml/to_yaml.cr b/src/yaml/to_yaml.cr index db488be72bb7..a6ef21a00cef 100644 --- a/src/yaml/to_yaml.cr +++ b/src/yaml/to_yaml.cr @@ -80,6 +80,21 @@ struct Number end end +struct Float + def to_yaml(yaml : YAML::Nodes::Builder) + infinite = self.infinite? + if infinite == 1 + yaml.scalar(".inf") + elsif infinite == -1 + yaml.scalar("-.inf") + elsif nan? + yaml.scalar(".nan") + else + yaml.scalar self.to_s + end + end +end + struct Nil def to_yaml(yaml : YAML::Nodes::Builder) yaml.scalar "" From 670bf33e012aedec43477484454378d6953b6022 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 30 Sep 2020 10:31:13 -0300 Subject: [PATCH 251/263] Fix overflows in md5 and sha1 (#9781) --- spec/manual/digest_large_file_spec.cr | 27 +++++++++++++++++++++++++++ src/digest/md5.cr | 6 +++--- src/digest/sha1.cr | 4 ++-- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 spec/manual/digest_large_file_spec.cr diff --git a/spec/manual/digest_large_file_spec.cr b/spec/manual/digest_large_file_spec.cr new file mode 100644 index 000000000000..419deea1b589 --- /dev/null +++ b/spec/manual/digest_large_file_spec.cr @@ -0,0 +1,27 @@ +require "spec" +require "digest/sha1" +require "digest/md5" + +private DATA = "a" * 1024 +private TOTAL_SIZE_GB = 1 +private TOTAL_SIZE = TOTAL_SIZE_GB * 1024 * 1024 * 1024 + +describe Digest::SHA1 do + it "does digest for large file" do + Digest::SHA1.digest do |ctx| + (TOTAL_SIZE / DATA.size).ceil.to_i.times do + ctx.update DATA + end + end + end +end + +describe Digest::MD5 do + it "does digest for large file" do + Digest::MD5.digest do |ctx| + (TOTAL_SIZE / DATA.size).ceil.to_i.times do + ctx.update DATA + end + end + end +end diff --git a/src/digest/md5.cr b/src/digest/md5.cr index e7c4699eb085..90dfdaa93ea0 100644 --- a/src/digest/md5.cr +++ b/src/digest/md5.cr @@ -35,9 +35,9 @@ class Digest::MD5 < Digest::Base mdi = (@i[0] >> 3) & 0x3F # update number of bits - @i[1] += 1 if (@i[0] + (inLen << 3)) < @i[0] - @i[0] += (inLen << 3) - @i[1] += (inLen >> 29) + @i[1] &+= 1 if (@i[0] &+ (inLen << 3)) < @i[0] + @i[0] &+= (inLen << 3) + @i[1] &+= (inLen >> 29) inLen.times do # add new character to buffer, increment mdi diff --git a/src/digest/sha1.cr b/src/digest/sha1.cr index c75de4c07738..b188530088fb 100644 --- a/src/digest/sha1.cr +++ b/src/digest/sha1.cr @@ -35,10 +35,10 @@ class Digest::SHA1 < Digest::Base data.each do |byte| @message_block[@message_block_index] = byte & 0xFF_u8 @message_block_index += 1 - @length_low += 8 + @length_low &+= 8 if @length_low == 0 - @length_high += 1 + @length_high &+= 1 if @length_high == 0 raise ArgumentError.new "Crypto.sha1: message too long" end From aed3d6f5afa5bfc33600545ad49eba2f631263ac Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Wed, 30 Sep 2020 15:31:55 +0200 Subject: [PATCH 252/263] Windows: pass a flag to allow creating symlinks without admin rights (#9767) As far as I can tell, passing this flag just has no downsides. Without it, symlinks can be created only as administrator; with it, symlinks still can't be created by default, but can be if only "developer mode" is enabled in Windows. But I'm guessing that'll be a requirement for most things anyway. --- src/crystal/system/win32/file.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index a1574733e602..716c285030e1 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -193,7 +193,7 @@ module Crystal::System::File def self.symlink(old_path : String, new_path : String) : Nil # TODO: support directory symlinks (copy Go's stdlib logic here) - if LibC.CreateSymbolicLinkW(to_windows_path(new_path), to_windows_path(old_path), 0) == 0 + if LibC.CreateSymbolicLinkW(to_windows_path(new_path), to_windows_path(old_path), LibC::SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE) == 0 raise ::File::Error.from_winerror("Error creating symbolic link", file: old_path, other: new_path) end end From d509f8e601cbaa1a03d967705572883ac789061e Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Thu, 1 Oct 2020 18:13:43 +0200 Subject: [PATCH 253/263] Fix `Process.find_executable` once and for all (#9365) * Fix `Process.find_executable` once and for all Make it work on Windows. Make it actually check that the found path is an *executable* *file*. Fix a lot of edge cases. Add specs and also verify that these expectations exactly match what Process.new would do. * fixup! Fix `Process.find_executable` once and for all * Expand the comment in the manual spec * Factor out a function listing possibilities of a path to check * Address review comments + refactor * Simplify compilation of test executables, doesn't have to be parallel * Add spec involving `..` * Resolve review comments --- spec/manual/find_executable_spec.cr | 58 ++++++++++ spec/std/process/find_executable_spec.cr | 141 +++++++++++++++++++++++ spec/std/process_spec.cr | 21 ---- src/process/executable_path.cr | 54 +++++++-- 4 files changed, 242 insertions(+), 32 deletions(-) create mode 100644 spec/manual/find_executable_spec.cr create mode 100644 spec/std/process/find_executable_spec.cr diff --git a/spec/manual/find_executable_spec.cr b/spec/manual/find_executable_spec.cr new file mode 100644 index 000000000000..ed7781b033cc --- /dev/null +++ b/spec/manual/find_executable_spec.cr @@ -0,0 +1,58 @@ +# Verifies that find_executable's specs match the behavior of Process.run. +# This doesn't actually test find_executable, only takes all the test cases +# directly from spec/std/process/find_executable_spec.cr and checks that +# *they* match what the OS actually does when finding an executable for the +# purpose of running it. + +require "spec" +require "digest/sha1" +require "../support/env" +require "../support/tempfile" +require "../std/process/find_executable_spec" + +describe "Process.run" do + test_dir = Path[SPEC_TEMPFILE_PATH] / "manual_find_executable" + base_dir = Path[test_dir] / "base" + path_dir = Path[test_dir] / "path" + + around_all do |all| + Dir.mkdir_p(test_dir) + + exe_names, non_exe_names = FIND_EXECUTABLE_TEST_FILES + exe_names.each do |name| + src_fn = test_dir / "self_printer.cr" + exe_fn = test_dir / "self_printer.exe" + File.write(src_fn, "print #{name.inspect}") + Process.run("bin/crystal", ["build", "-o", exe_fn.to_s, src_fn.to_s]) + Dir.mkdir_p((base_dir / name).parent) + File.rename(exe_fn, base_dir / name) + end + non_exe_names.each do |name| + File.write(base_dir / name, "") + end + + with_env "PATH": {ENV["PATH"], path_dir}.join(Process::PATH_DELIMITER) do + Dir.cd(base_dir) do + all.run + end + end + + FileUtils.rm_r(test_dir.to_s) + end + + find_executable_test_cases(base_dir).each do |(command, exp)| + if exp + it "runs '#{command}' as '#{exp}'" do + output = Process.run command, &.output.gets_to_end + $?.success?.should be_true + output.should eq exp + end + else + it "fails to run '#{command}'" do + expect_raises IO::Error do + Process.run(command) + end + end + end + end +end diff --git a/spec/std/process/find_executable_spec.cr b/spec/std/process/find_executable_spec.cr new file mode 100644 index 000000000000..cb5033f13ccd --- /dev/null +++ b/spec/std/process/find_executable_spec.cr @@ -0,0 +1,141 @@ +require "spec" +require "../../support/tempfile" + +{% if flag?(:win32) %} + FIND_EXECUTABLE_TEST_FILES = { + [ + "inbase.exe", + "not_exe", + ".exe", + "inboth.exe", + + "inbasebat.bat", + "inbase.foo.exe", + ".inbase.exe", + + "sub/insub.exe", + "sub/not_exe", + "sub/.exe", + + "../path/inpath.exe", + "../path/not_exe", + "../path/.exe", + "../path/inboth.exe", + ], [] of String, + } + + def find_executable_test_cases(pwd) + pwd_nodrive = "\\#{pwd.relative_to(pwd.anchor.not_nil!)}" + { + "inbase.exe" => "inbase.exe", + "inbase" => "inbase.exe", + "sub\\insub.exe" => "sub/insub.exe", + "sub/insub" => "sub/insub.exe", + "inpath.exe" => "../path/inpath.exe", + "inpath" => "../path/inpath.exe", + "sub/.exe" => "sub/.exe", + "sub\\" => "sub/.exe", + "sub/" => "sub/.exe", + ".exe" => ".exe", + "not_exe" => nil, + "sub\\not_exe" => nil, + "inbasebat" => nil, + "inbase.foo.exe" => "inbase.foo.exe", + "inbase.foo" => nil, + ".inbase.exe" => ".inbase.exe", + ".inbase" => nil, + "" => nil, + "." => nil, + "inboth.exe" => "inboth.exe", + "inboth" => "inboth.exe", + "./inbase" => "inbase.exe", + "../base/inbase" => "inbase.exe", + "./inpath" => nil, + "sub" => nil, + "#{pwd}\\sub" => nil, + "#{pwd}\\sub\\" => nil, + # 'C:\Temp\base\inbase', 'C:\Temp\base\.exe', 'C:\Temp\base\' + "#{pwd}\\inbase" => "inbase.exe", + "#{pwd}\\.exe" => ".exe", + "#{pwd}\\" => nil, + # 'C:inbase', 'C:.exe', 'C:' + "#{pwd.drive}inbase" => "inbase.exe", + "#{pwd.drive}.exe" => ".exe", + "#{pwd.drive}" => nil, + "#{pwd.drive}sub\\" => nil, + # '\Temp\base\inbase', '\Temp\base\.exe', '\Temp\base\' + "#{pwd_nodrive}\\inbase" => "inbase.exe", + "#{pwd_nodrive}\\.exe" => ".exe", + "#{pwd_nodrive}\\" => nil, + } + end +{% else %} + FIND_EXECUTABLE_TEST_FILES = { + [ + "inbase", + "sub/insub", + "../path/inpath", + ], [ + "not_exe", + "sub/not_exe", + "../path/not_exe", + ], + } + + def find_executable_test_cases(pwd) + { + "./inbase" => "inbase", + "../base/inbase" => "inbase", + "inbase" => nil, + "sub/insub" => "sub/insub", + "inpath" => "../path/inpath", + "./inpath" => nil, + "inbase/" => nil, + "sub/insub/" => nil, + "./not_exe" => nil, + "not_exe" => nil, + "sub/not_exe" => nil, + "" => nil, + "." => nil, + "#{pwd}/inbase" => "inbase", + "#{pwd}/inbase/" => nil, + "#{pwd}/sub" => nil, + "./sub" => nil, + "sub" => nil, + } + end +{% end %} + +describe "Process.find_executable" do + test_dir = Path[SPEC_TEMPFILE_PATH] / "find_executable" + base_dir = Path[test_dir] / "base" + path_dir = Path[test_dir] / "path" + + around_all do |all| + exe_names, non_exe_names = FIND_EXECUTABLE_TEST_FILES + (exe_names + non_exe_names).each do |name| + Dir.mkdir_p((base_dir / name).parent) + File.write(base_dir / name, "") + end + exe_names.each do |name| + File.chmod(base_dir / name, 0o755) + end + + all.run + + FileUtils.rm_r(test_dir.to_s) + end + + find_executable_test_cases(base_dir).each do |(command, exp)| + if exp + exp_path = File.expand_path(exp, base_dir) + it "finds '#{command}' as '#{exp}'" do + Process.find_executable(command, path: path_dir.to_s, pwd: base_dir).should eq exp_path + end + else + it "fails to find '#{command}'" do + Process.find_executable(command, path: path_dir.to_s, pwd: base_dir).should be_nil + end + end + end +end diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 968fc2559aa1..77ac0da19944 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -402,27 +402,6 @@ describe Process do end end - describe "find_executable" do - pwd = Process::INITIAL_PWD - crystal_path = Path.new(pwd, "bin", "crystal").to_s - - pending_win32 "resolves absolute executable" do - Process.find_executable(Path.new(pwd, "bin", "crystal")).should eq(crystal_path) - end - - pending_win32 "resolves relative executable" do - Process.find_executable(Path.new("bin", "crystal")).should eq(crystal_path) - Process.find_executable(Path.new("..", File.basename(pwd), "bin", "crystal")).should eq(crystal_path) - end - - pending_win32 "searches within PATH" do - (path = Process.find_executable("ls")).should_not be_nil - path.not_nil!.should match(/#{File::SEPARATOR}ls$/) - - Process.find_executable("some_very_unlikely_file_to_exist").should be_nil - end - end - describe "quote_posix" do it { Process.quote_posix("").should eq "''" } it { Process.quote_posix(" ").should eq "' '" } diff --git a/src/process/executable_path.cr b/src/process/executable_path.cr index b9c7498413da..5485a921bb73 100644 --- a/src/process/executable_path.cr +++ b/src/process/executable_path.cr @@ -28,13 +28,47 @@ class Process end end + private def self.is_executable_file?(path) + unless File.info?(path, follow_symlinks: true).try &.file? + return false + end + {% if flag?(:win32) %} + # This is *not* a temporary stub. + # Windows doesn't have "executable" metadata for files, so it also doesn't have files that are "not executable". + true + {% else %} + File.executable?(path) + {% end %} + end + # Searches an executable, checking for an absolute path, a path relative to # *pwd* or absolute path, then eventually searching in directories declared # in *path*. def self.find_executable(name : Path | String, path : String? = ENV["PATH"]?, pwd : Path | String = Dir.current) : String? - name = Path.new(name) + find_executable_possibilities(Path.new(name), path, pwd) do |p| + if is_executable_file?(p) + return p.to_s + end + end + nil + end + + private def self.find_executable_possibilities(name, path, pwd) + return if name.to_s.empty? + + {% if flag?(:win32) %} + # https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw#parameters + # > If the file name does not contain an extension, .exe is appended. + # See find_executable_spec.cr for cases this needs to match, based on CreateProcessW behavior. + basename = name.ends_with_separator? ? "" : name.basename + basename = "" if basename == name.anchor.to_s + if (basename.empty? ? !name.anchor : !basename.includes?(".")) + name = Path.new("#{name}.exe") + end + {% end %} + if name.absolute? - return name.to_s + yield name end # check if the name includes a separator @@ -43,19 +77,17 @@ class Process count_parts += 1 break if count_parts > 1 end + has_separator = (count_parts > 1) - if count_parts > 1 - return name.expand(pwd).to_s + if {{ flag?(:win32) }} || has_separator + yield name.expand(pwd) end - return unless path - - path.split(PATH_DELIMITER).each do |path_entry| - executable = Path.new(path_entry, name) - return executable.to_s if File.exists?(executable) + if path && !has_separator + path.split(PATH_DELIMITER).each do |path_entry| + yield Path.new(path_entry, name) + end end - - nil end end From f94e5f788e03f79d0d964731374cdddd79f7e02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Cla=C3=9Fen?= Date: Fri, 2 Oct 2020 12:19:37 +0200 Subject: [PATCH 254/263] Typo fixed in example (#9786) --- src/option_parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/option_parser.cr b/src/option_parser.cr index 62fa89c19f65..c9f500f2c955 100644 --- a/src/option_parser.cr +++ b/src/option_parser.cr @@ -58,7 +58,7 @@ # welcome = true # parser.banner = "Usage: example welcome" # end -# parser.on("-v", "--verbose", "Enabled servose output") { verbose = true } +# parser.on("-v", "--verbose", "Enabled verbose output") { verbose = true } # parser.on("-h", "--help", "Show this help") do # puts parser # exit From 5be111affd18206610a129c78ad8fbe7b93af687 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Sat, 3 Oct 2020 10:25:02 -0300 Subject: [PATCH 255/263] Fix `Iterator#cons_pair` return type (#9788) * Fix Iterator#cons_pair return type * Removed redundant return check wrapped_next already checks for stop --- spec/std/iterator_spec.cr | 4 ++++ src/iterator.cr | 11 +++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/spec/std/iterator_spec.cr b/spec/std/iterator_spec.cr index 5aa4a9e0e899..4cb73cd91e51 100644 --- a/spec/std/iterator_spec.cr +++ b/spec/std/iterator_spec.cr @@ -177,6 +177,10 @@ describe Iterator do iter.next.should eq({4, 5}) iter.next.should be_a(Iterator::Stop) end + + it "doesn't include stop in return type" do + (1..3).each.cons_pair.to_a.should eq([{1, 2}, {2, 3}]) + end end describe "cycle" do diff --git a/src/iterator.cr b/src/iterator.cr index da5906bc6520..5da4299a060b 100644 --- a/src/iterator.cr +++ b/src/iterator.cr @@ -338,18 +338,17 @@ module Iterator(T) def initialize(@iterator : I) end - def next + def next : {T, T} | Iterator::Stop elem = wrapped_next - return elem if elem.is_a?(Iterator::Stop) + last_elem = @last_elem - if @last_elem.is_a?(Iterator::Stop) + if last_elem.is_a?(Iterator::Stop) @last_elem = elem - self.next else + value = {last_elem, elem} @last_elem, elem = elem, @last_elem - - {elem, @last_elem} + value end end end From 719d314879719eb98099459199d3493f09694841 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Mon, 5 Oct 2020 15:04:11 -0300 Subject: [PATCH 256/263] Make `**` be right associative (#9684) --- spec/compiler/parser/parser_spec.cr | 1 + src/compiler/crystal/syntax/parser.cr | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/compiler/parser/parser_spec.cr b/spec/compiler/parser/parser_spec.cr index c9a4b7d2c0d8..cafa5246704a 100644 --- a/spec/compiler/parser/parser_spec.cr +++ b/spec/compiler/parser/parser_spec.cr @@ -136,6 +136,7 @@ module Crystal it_parses "foo[] /2", Call.new(Call.new("foo".call, "[]"), "/", 2.int32) it_parses "foo[1] /2", Call.new(Call.new("foo".call, "[]", 1.int32), "/", 2.int32) it_parses "[1] /2", Call.new(([1.int32] of ASTNode).array, "/", 2.int32) + it_parses "2**3**4", Call.new(2.int32, "**", Call.new(3.int32, "**", 4.int32)) it_parses "!1", Not.new(1.int32) it_parses "- 1", Call.new(1.int32, "-") diff --git a/src/compiler/crystal/syntax/parser.cr b/src/compiler/crystal/syntax/parser.cr index 448b6fefc61c..5ddd2d007be9 100644 --- a/src/compiler/crystal/syntax/parser.cr +++ b/src/compiler/crystal/syntax/parser.cr @@ -504,7 +504,7 @@ module Crystal RangeLiteral.new(exp, right, exclusive).at(location).at_end(right) end - macro parse_operator(name, next_operator, node, operators) + macro parse_operator(name, next_operator, node, operators, right_associative = false) def parse_{{name.id}} location = @token.location @@ -521,7 +521,7 @@ module Crystal slash_is_regex! next_token_skip_space_or_newline - right = parse_{{next_operator.id}} + right = parse_{{(right_associative ? name : next_operator).id}} left = ({{node.id}}).at(location).at_end(right) left.name_location = name_location if left.is_a?(Call) else @@ -579,7 +579,7 @@ module Crystal end parse_operator :mul_or_div, :pow, "Call.new left, method, right", %(:"*", :"/", :"//", :"%", :"&*") - parse_operator :pow, :prefix, "Call.new left, method, right", %(:"**", :"&**") + parse_operator :pow, :prefix, "Call.new left, method, right", %(:"**", :"&**"), right_associative: true def parse_prefix name_location = @token.location From 5251f2a3a72e11dee45a127aa01fcb044cd4c738 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Tue, 6 Oct 2020 09:36:07 -0300 Subject: [PATCH 257/263] Codegen: don't use init check for consts that are declared before read (#9801) --- src/compiler/crystal/codegen/const.cr | 40 +++++++++++++++++++++++++-- src/compiler/crystal/codegen/types.cr | 9 ++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/codegen/const.cr b/src/compiler/crystal/codegen/const.cr index 893c18a52b43..e64c6e3d98fb 100644 --- a/src/compiler/crystal/codegen/const.cr +++ b/src/compiler/crystal/codegen/const.cr @@ -96,9 +96,43 @@ class Crystal::CodeGenVisitor end end + def initialize_no_init_flag_const(const) + global = declare_const(const) + + with_cloned_context do + # "self" in a constant is the constant's namespace + context.type = const.namespace + + # Start with fresh variables + context.vars = LLVMVars.new + + alloca_vars const.vars + request_value do + accept const.value + end + end + + const_type = const.value.type + if const_type.passed_by_value? + @last = load @last + end + + store @last, global + + global.initializer = @last.type.null + + global + end + def initialize_const(const) - # Maybe the constant was simple and doesn't need a real initialization + # If the constant wasn't read yet, we can initialize it right now and + # avoid checking an "initialized" flag every time we read it. + unless const.read? + const.no_init_flag = true + return initialize_no_init_flag_const(const) + end + # Maybe the constant was simple and doesn't need a real initialization global, initialized_flag = declare_const_and_initialized_flag(const) return global if const.initializer @@ -173,7 +207,9 @@ class Crystal::CodeGenVisitor end def read_const_pointer(const) - if const == @program.argc || const == @program.argv || const.initializer + const.read = true + + if const == @program.argc || const == @program.argv || const.initializer || const.no_init_flag? global_name = const.llvm_name global = declare_const(const) diff --git a/src/compiler/crystal/codegen/types.cr b/src/compiler/crystal/codegen/types.cr index f156e568cd0e..5db91ee3046e 100644 --- a/src/compiler/crystal/codegen/types.cr +++ b/src/compiler/crystal/codegen/types.cr @@ -167,6 +167,15 @@ module Crystal class Const property initializer : LLVM::Value? + # Was this constant already read during the codegen phase? + # If not, and we are at the place that declares the constant, we can + # directly initialize the constant now, without checking for an `init` flag. + property? read = false + + # If true, there's no need to check whether the constant was initialized or + # not when reading it. + property? no_init_flag = false + def initialized_llvm_name "#{llvm_name}:init" end From a3f9182b20eec2d42611a6d27e1ddb4627815b3a Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 7 Oct 2020 18:27:24 -0300 Subject: [PATCH 258/263] Use Dwarf information on Exception::CallStack.print_frame (#9792) * Use dwarf if available on Exception::CallStack.print_frame Calling callee at the beginning of the program will load dwarf * Load dwarf on start up and look for it only if compiled with --debug --- src/exception/call_stack.cr | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/exception/call_stack.cr b/src/exception/call_stack.cr index 2cd1ec042a63..2d6d589faba8 100644 --- a/src/exception/call_stack.cr +++ b/src/exception/call_stack.cr @@ -123,6 +123,21 @@ struct Exception::CallStack end private def self.print_frame(repeated_frame) + {% if flag?(:debug) %} + if @@dwarf_loaded && + (name = decode_function_name(repeated_frame.ip.address)) + file, line, column = Exception::CallStack.decode_line_number(repeated_frame.ip.address) + if file && file != "??" + if repeated_frame.count == 0 + Crystal::System.print_error "[0x%lx] %s at %s:%ld:%i\n", repeated_frame.ip, name, file, line, column + else + Crystal::System.print_error "[0x%lx] %s at %s:%ld:%i (%ld times)\n", repeated_frame.ip, name, file, line, column, repeated_frame.count + 1 + end + return + end + end + {% end %} + frame = decode_frame(repeated_frame.ip) if frame offset, sname = frame @@ -202,4 +217,10 @@ struct Exception::CallStack end end end + + {% if flag?(:debug) %} + # load dwarf on start up of the program when compiled with --debug + # this will make dwarf available on print_frame that is used on __crystal_sigfault_handler + load_dwarf + {% end %} end From 8c182b8ab4445f84d08022546b400c7aa3d4d442 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Wed, 7 Oct 2020 18:27:42 -0300 Subject: [PATCH 259/263] Allow -Dgc_none to "work" with weak_ref (#9806) Otherwise it is impossible on medium projects with crystal-db to run with the gc_none flag. --- src/gc/none.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gc/none.cr b/src/gc/none.cr index 1c1690a0e368..4f5cf5f7f7e3 100644 --- a/src/gc/none.cr +++ b/src/gc/none.cr @@ -43,6 +43,9 @@ module GC def self.add_finalizer(object) end + def self.register_disappearing_link(pointer : Void**) + end + def self.stats zero = LibC::ULong.new(0) Stats.new(zero, zero, zero, zero, zero) From 0fa9550758e91f12ca3aa7fdf0ce01bc8123ddf2 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 9 Oct 2020 18:29:28 -0300 Subject: [PATCH 260/263] Make abstract def return type warning an error (#9810) * Make abstract def return type warning an error Fixes #9655 * Drop AbstractDefImplementationError * Drop skip_abstract_def_check flag --- spec/compiler/semantic/abstract_def_spec.cr | 30 +++++++++---------- src/compiler/crystal/semantic.cr | 8 ++--- .../crystal/semantic/abstract_def_checker.cr | 14 ++++----- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/spec/compiler/semantic/abstract_def_spec.cr b/spec/compiler/semantic/abstract_def_spec.cr index a5e82cc88fd1..c0b078f6b893 100644 --- a/spec/compiler/semantic/abstract_def_spec.cr +++ b/spec/compiler/semantic/abstract_def_spec.cr @@ -395,8 +395,8 @@ describe "Semantic: abstract def" do ) end - it "warning if missing return type" do - assert_warning <<-CR, + it "errors if missing return type" do + assert_error <<-CR, abstract class Foo abstract def foo : Int32 end @@ -407,11 +407,11 @@ describe "Semantic: abstract def" do end end CR - "warning in line 6\nWarning: this method overrides Foo#foo() which has an explicit return type of Int32.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." + "this method overrides Foo#foo() which has an explicit return type of Int32.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." end - it "warning if different return type" do - assert_warning <<-CR, + it "errors if different return type" do + assert_error <<-CR, abstract class Foo abstract def foo : Int32 end @@ -425,7 +425,7 @@ describe "Semantic: abstract def" do end end CR - "warning in line 9\nWarning: this method must return Int32, which is the return type of the overridden method Foo#foo(), or a subtype of it, not Bar::Int32" + "this method must return Int32, which is the return type of the overridden method Foo#foo(), or a subtype of it, not Bar::Int32" end it "can return a more specific type" do @@ -542,8 +542,8 @@ describe "Semantic: abstract def" do )) end - it "is missing a return type in subclass of generic subclass" do - assert_warning <<-CR, + it "errors if missing a return type in subclass of generic subclass" do + assert_error <<-CR, abstract class Foo(T) abstract def foo : T end @@ -553,11 +553,11 @@ describe "Semantic: abstract def" do end end CR - "warning in line 6\nWarning: this method overrides Foo(T)#foo() which has an explicit return type of T.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." + "this method overrides Foo(T)#foo() which has an explicit return type of T.\n\nPlease add an explicit return type (Int32 or a subtype of it) to this method as well." end - it "can't find parent return type" do - assert_warning <<-CR, + it "errors if can't find parent return type" do + assert_error <<-CR, abstract class Foo abstract def foo : Unknown end @@ -567,11 +567,11 @@ describe "Semantic: abstract def" do end end CR - "warning in line 2\nWarning: can't resolve return type Unknown" + "can't resolve return type Unknown" end - it "can't find child return type" do - assert_warning <<-CR, + it "errors if can't find child return type" do + assert_error <<-CR, abstract class Foo abstract def foo : Int32 end @@ -581,7 +581,7 @@ describe "Semantic: abstract def" do end end CR - "warning in line 6\nWarning: can't resolve return type Unknown" + "can't resolve return type Unknown" end it "doesn't crash when abstract method is implemented by supertype (#8031)" do diff --git a/src/compiler/crystal/semantic.cr b/src/compiler/crystal/semantic.cr index 931781631bd1..4e2ddef0993b 100644 --- a/src/compiler/crystal/semantic.cr +++ b/src/compiler/crystal/semantic.cr @@ -70,12 +70,8 @@ class Crystal::Program TypeDeclarationProcessor.new(self).process(node) end - # TODO: remove this check a couple of versions after 0.30.0 once - # we are sure it's working fine for everyone - unless has_flag?("skip_abstract_def_check") - @progress_tracker.stage("Semantic (abstract def check)") do - AbstractDefChecker.new(self).run - end + @progress_tracker.stage("Semantic (abstract def check)") do + AbstractDefChecker.new(self).run end {node, processor} diff --git a/src/compiler/crystal/semantic/abstract_def_checker.cr b/src/compiler/crystal/semantic/abstract_def_checker.cr index cea1dbee6ec2..e0dafd0b5bb6 100644 --- a/src/compiler/crystal/semantic/abstract_def_checker.cr +++ b/src/compiler/crystal/semantic/abstract_def_checker.cr @@ -261,7 +261,7 @@ class Crystal::AbstractDefChecker original_base_return_type = base_type.lookup_type?(base_return_type_node) unless original_base_return_type - report_warning(base_return_type_node, "can't resolve return type #{base_return_type_node}\n#{this_warning_will_become_an_error}") + report_error(base_return_type_node, "can't resolve return type #{base_return_type_node}") return end @@ -279,24 +279,24 @@ class Crystal::AbstractDefChecker base_return_type = base_type.lookup_type?(base_return_type_node) unless base_return_type - report_warning(base_return_type_node, "can't resolve return type #{base_return_type_node}\n#{this_warning_will_become_an_error}") + report_error(base_return_type_node, "can't resolve return type #{base_return_type_node}") return end return_type_node = method.return_type unless return_type_node - report_warning(method, "this method overrides #{Call.def_full_name(base_type, base_method)} which has an explicit return type of #{original_base_return_type}.\n#{@program.colorize("Please add an explicit return type (#{base_return_type} or a subtype of it) to this method as well.").yellow.bold}\n\n#{this_warning_will_become_an_error}") + report_error(method, "this method overrides #{Call.def_full_name(base_type, base_method)} which has an explicit return type of #{original_base_return_type}.\n#{@program.colorize("Please add an explicit return type (#{base_return_type} or a subtype of it) to this method as well.").yellow.bold}\n") return end return_type = type.lookup_type?(return_type_node) unless return_type - report_warning(return_type_node, "can't resolve return type #{return_type_node}\n#{this_warning_will_become_an_error}") + report_error(return_type_node, "can't resolve return type #{return_type_node}") return end unless return_type.implements?(base_return_type) - report_warning(return_type_node, "this method must return #{base_return_type}, which is the return type of the overridden method #{Call.def_full_name(base_type, base_method)}, or a subtype of it, not #{return_type}\n#{this_warning_will_become_an_error}") + report_error(return_type_node, "this method must return #{base_return_type}, which is the return type of the overridden method #{Call.def_full_name(base_type, base_method)}, or a subtype of it, not #{return_type}") return end end @@ -321,8 +321,8 @@ class Crystal::AbstractDefChecker @program.colorize("The above warning will become an error in a future Crystal version.").yellow.bold end - private def report_warning(node, message) - @program.report_warning(node, message) + private def report_error(node, message) + node.raise(message, nil) end class ReplacePathWithTypeVar < Visitor From 0e53f6e603b2b021597cfd6724fba35be05ff221 Mon Sep 17 00:00:00 2001 From: "Brian J. Cardiff" Date: Fri, 16 Oct 2020 12:06:51 -0300 Subject: [PATCH 261/263] Force secure renegotiation on server (#9815) Prevent Secure Client-Initiated Renegotiation vulnerability attack by default on servers Ref: https://www.openssl.org/docs/man1.1.0/man3/SSL_CTX_set_options.html#SECURE-RENEGOTIATION --- spec/std/openssl/ssl/context_spec.cr | 3 +++ src/openssl/lib_ssl.cr | 3 +++ src/openssl/ssl/context.cr | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/spec/std/openssl/ssl/context_spec.cr b/spec/std/openssl/ssl/context_spec.cr index df71c4dfa127..68cf7d1076b4 100644 --- a/spec/std/openssl/ssl/context_spec.cr +++ b/spec/std/openssl/ssl/context_spec.cr @@ -41,6 +41,9 @@ describe OpenSSL::SSL::Context do (context.options & OpenSSL::SSL::Options::SINGLE_ECDH_USE).should eq(OpenSSL::SSL::Options::SINGLE_ECDH_USE) (context.options & OpenSSL::SSL::Options::SINGLE_DH_USE).should eq(OpenSSL::SSL::Options::SINGLE_DH_USE) (context.options & OpenSSL::SSL::Options::CIPHER_SERVER_PREFERENCE).should eq(OpenSSL::SSL::Options::CIPHER_SERVER_PREFERENCE) + {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.1.0") >= 0 %} + (context.options & OpenSSL::SSL::Options::NO_RENEGOTIATION).should eq(OpenSSL::SSL::Options::NO_RENEGOTIATION) + {% end %} context.modes.should eq(OpenSSL::SSL::Modes.flags(AUTO_RETRY, RELEASE_BUFFERS)) context.verify_mode.should eq(OpenSSL::SSL::VerifyMode::NONE) diff --git a/src/openssl/lib_ssl.cr b/src/openssl/lib_ssl.cr index a7ad45ea6da7..688264c488bb 100644 --- a/src/openssl/lib_ssl.cr +++ b/src/openssl/lib_ssl.cr @@ -99,6 +99,9 @@ lib LibSSL NO_TLS_V1_3 = 0x20000000 NO_TLS_V1_2 = 0x08000000 NO_TLS_V1_1 = 0x10000000 + {% if compare_versions(OPENSSL_VERSION, "1.1.0") >= 0 %} + NO_RENEGOTIATION = 0x40000000 + {% end %} NETSCAPE_CA_DN_BUG = 0x20000000 NETSCAPE_DEMO_CIPHER_CHANGE_BUG = 0x40000000 diff --git a/src/openssl/ssl/context.cr b/src/openssl/ssl/context.cr index 031e44d57b27..00f58393d8db 100644 --- a/src/openssl/ssl/context.cr +++ b/src/openssl/ssl/context.cr @@ -191,6 +191,10 @@ abstract class OpenSSL::SSL::Context SINGLE_DH_USE )) + {% if compare_versions(LibSSL::OPENSSL_VERSION, "1.1.0") >= 0 %} + add_options(OpenSSL::SSL::Options::NO_RENEGOTIATION) + {% end %} + add_modes(OpenSSL::SSL::Modes.flags(AUTO_RETRY, RELEASE_BUFFERS)) end From f78efd9a59ccef4816b6180c2e42accc567590f2 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Fri, 11 Sep 2020 19:42:40 +0900 Subject: [PATCH 262/263] Reduce redundant newlines at end of `begin ... end` Follow up #9722 The following example still invokes a formatter issue like #9657. ``` begin 1 # Comment end ``` This commit fixes such issues. --- spec/compiler/formatter/formatter_spec.cr | 26 +++++++++++++++++++++++ src/compiler/crystal/tools/formatter.cr | 18 ++++++++++------ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/spec/compiler/formatter/formatter_spec.cr b/spec/compiler/formatter/formatter_spec.cr index e9b98f9d17c4..6705ae01ca3f 100644 --- a/spec/compiler/formatter/formatter_spec.cr +++ b/spec/compiler/formatter/formatter_spec.cr @@ -1729,6 +1729,32 @@ describe Crystal::Formatter do end CODE + assert_format <<-BEFORE, <<-AFTER + begin + 1 + # Comment + + + end + BEFORE + begin + 1 + # Comment + end + AFTER + + assert_format <<-BEFORE, <<-AFTER + begin + # Comment + + + end + BEFORE + begin + # Comment + end + AFTER + assert_format <<-CODE foo 1, # comment do diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index 1ee111617518..b492778f2385 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -220,9 +220,14 @@ module Crystal write "begin" @indent += 2 write_line - next_token_skip_space_or_newline - if @token.type == :";" - next_token_skip_space_or_newline + next_token + # Cornor case: an empty `begin ... end`. + # In this case, we should not skip space becuase it will do in the below loop. + unless node.expressions.size == 1 && node.expressions[0].is_a?(Nop) + skip_space_or_newline + if @token.type == :";" + next_token_skip_space_or_newline + end end has_begin = true base_indent = @indent @@ -231,6 +236,7 @@ module Crystal end last_aligned_assign = nil + last_found_comment = false max_length = nil skip_space @@ -283,7 +289,7 @@ module Crystal end if last?(i, node.expressions) - skip_space_or_newline last: true + last_found_comment = skip_space_or_newline last: true, next_comes_end: true else if needs_two_lines unless found_comment @@ -307,7 +313,7 @@ module Crystal @indent = old_indent - if has_newline + if has_newline && !last_found_comment write_line write_indent end @@ -4662,7 +4668,7 @@ module Crystal end def write_indent(indent, node) - write_indent(indent) + write_indent(indent) unless node.is_a?(Nop) indent(indent, node) end From 14689871a5c4e67a6c335a87ce001f76f4a07263 Mon Sep 17 00:00:00 2001 From: Ary Borenszweig Date: Fri, 11 Sep 2020 08:42:54 -0300 Subject: [PATCH 263/263] Update src/compiler/crystal/tools/formatter.cr Co-authored-by: Sijawusz Pur Rahnama --- src/compiler/crystal/tools/formatter.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/crystal/tools/formatter.cr b/src/compiler/crystal/tools/formatter.cr index b492778f2385..1e3c9eec07c4 100644 --- a/src/compiler/crystal/tools/formatter.cr +++ b/src/compiler/crystal/tools/formatter.cr @@ -221,8 +221,8 @@ module Crystal @indent += 2 write_line next_token - # Cornor case: an empty `begin ... end`. - # In this case, we should not skip space becuase it will do in the below loop. + # Corner case: an empty `begin ... end`. + # In this case, we should not skip space because it will do in the below loop. unless node.expressions.size == 1 && node.expressions[0].is_a?(Nop) skip_space_or_newline if @token.type == :";"