diff --git a/compiler+runtime/CMakeLists.txt b/compiler+runtime/CMakeLists.txt index 457914a94..c601681fd 100644 --- a/compiler+runtime/CMakeLists.txt +++ b/compiler+runtime/CMakeLists.txt @@ -517,6 +517,7 @@ add_library( src/cpp/jtl/panic.cpp src/cpp/jtl/assert.cpp src/cpp/jtl/string_builder.cpp + src/cpp/jtl/utf8.cpp src/cpp/jank/c_api.cpp src/cpp/jank/hash.cpp src/cpp/jank/util/cli.cpp diff --git a/compiler+runtime/include/cpp/jtl/utf8.hpp b/compiler+runtime/include/cpp/jtl/utf8.hpp new file mode 100644 index 000000000..2acad16f5 --- /dev/null +++ b/compiler+runtime/include/cpp/jtl/utf8.hpp @@ -0,0 +1,37 @@ +#include + +namespace jtl +{ + struct utf8_iterator + { + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = immutable_string; + using reference = value_type const &; + + utf8_iterator() = default; + utf8_iterator(immutable_string const &s); + utf8_iterator(immutable_string const &s, value_type::size_type const position); + + value_type operator*() const; + utf8_iterator &operator++(); + utf8_iterator operator++(int); + utf8_iterator &operator--(); + utf8_iterator operator--(int); + bool operator==(utf8_iterator const &it) const; + + value_type data; + value_type::size_type i{}; + value_type::size_type n{}; + }; + + struct utf8_range + { + utf8_range(immutable_string const &s); + + utf8_iterator begin() const; + utf8_iterator end() const; + + immutable_string data; + }; +} diff --git a/compiler+runtime/src/cpp/clojure/string_native.cpp b/compiler+runtime/src/cpp/clojure/string_native.cpp index 65e3283fd..d519d7eb5 100644 --- a/compiler+runtime/src/cpp/clojure/string_native.cpp +++ b/compiler+runtime/src/cpp/clojure/string_native.cpp @@ -1,3 +1,7 @@ +#include + +#include + #include #include #include @@ -26,7 +30,12 @@ namespace clojure::string_native jtl::immutable_string reverse(jtl::immutable_string const &s) { - return { s.rbegin(), s.rend() }; + jtl::string_builder buff{ s.size() }; + for(auto const &c : jtl::utf8_range(s) | std::views::reverse) + { + buff(c); + } + return buff.release(); } jtl::immutable_string lower_case(jtl::immutable_string const &s) diff --git a/compiler+runtime/src/cpp/jtl/utf8.cpp b/compiler+runtime/src/cpp/jtl/utf8.cpp new file mode 100644 index 000000000..ce47a5f79 --- /dev/null +++ b/compiler+runtime/src/cpp/jtl/utf8.cpp @@ -0,0 +1,164 @@ +#include + +namespace jtl +{ + using value_type = utf8_iterator::value_type; + using size_type = value_type::size_type; + + static size_type next_char_size(immutable_string const &s, size_type const i) + { + auto const c(s[i]); + if(c <= 0x7f) + { + return 1; + } + else if((c & 0xE0) == 0xC0) + { + return 2; + } + else if((c & 0xF0) == 0xE0) + { + return 3; + } + else if((c & 0xF8) == 0xF0) + { + return 4; + } + else + { + throw std::runtime_error{ "Invalid UTF-8 string." }; + } + } + + static bool is_continuation_byte(immutable_string const &s, size_type const i) + { + if(i == immutable_string::npos) + { + throw std::runtime_error{ "Invalid UTF-8 string." }; + } + + return (s[i] & 0xC0) == 0x80; + } + + static size_type prev_char_size(immutable_string const &data, size_type const i) + { + if(!is_continuation_byte(data, i - 1)) + { + return 1; + } + else if(!is_continuation_byte(data, i - 2)) + { + return 2; + } + else if(!is_continuation_byte(data, i - 3)) + { + return 3; + } + else if(!is_continuation_byte(data, i - 4)) + { + return 4; + } + else + { + throw std::runtime_error{ "Invalid UTF-8 string." }; + } + } + + utf8_iterator::utf8_iterator(immutable_string const &s) + : data{ s } + , i{ s.empty() ? value_type::npos : 0 } + , n{ i == value_type::npos ? 0 : next_char_size(s, i) } + { + } + + utf8_iterator::utf8_iterator(immutable_string const &s, value_type::size_type const position) + : data{ s } + , i{ s.empty() ? value_type::npos : position } + , n{ i == value_type::npos ? 0 : next_char_size(s, i) } + { + } + + value_type utf8_iterator::operator*() const + { + return data.substr(i, n); + } + + utf8_iterator &utf8_iterator::operator++() + { + if(i == value_type::npos) + { + i = 0; + n = next_char_size(data, i); + } + else + { + i += n; + if(i >= data.size()) + { + i = value_type::npos; + n = 0; + } + else + { + n = next_char_size(data, i); + } + } + + return *this; + } + + utf8_iterator utf8_iterator::operator++(int) + { + auto tmp{ *this }; + ++*this; + return tmp; + } + + utf8_iterator &utf8_iterator::operator--() + { + if(i == 0) + { + i = value_type::npos; + n = 0; + } + else if(i == value_type::npos) + { + auto const size(data.size()); + n = prev_char_size(data, size); + i = size - n; + } + else + { + n = prev_char_size(data, i); + i -= n; + } + return *this; + } + + utf8_iterator utf8_iterator::operator--(int) + { + auto tmp{ *this }; + --*this; + return tmp; + } + + bool utf8_iterator::operator==(utf8_iterator const &it) const + { + return i == it.i && (data.data() == it.data.data() || data == it.data); + } + + utf8_range::utf8_range(immutable_string const &s) + : data{ s } + { + } + + utf8_iterator utf8_range::begin() const + { + return { data, 0 }; + } + + utf8_iterator utf8_range::end() const + { + return { data, value_type::npos }; + } +} diff --git a/compiler+runtime/test/bash/clojure-test-suite/clojure-test-suite b/compiler+runtime/test/bash/clojure-test-suite/clojure-test-suite index cde6c8235..5e22883bf 160000 --- a/compiler+runtime/test/bash/clojure-test-suite/clojure-test-suite +++ b/compiler+runtime/test/bash/clojure-test-suite/clojure-test-suite @@ -1 +1 @@ -Subproject commit cde6c823568e9561bb5ff1157db3bbc6f6ae9dee +Subproject commit 5e22883bf442ec896326614763f0f4e655e36f18 diff --git a/compiler+runtime/test/bash/clojure-test-suite/src/jank_test/run_clojure_test_suite.cljc b/compiler+runtime/test/bash/clojure-test-suite/src/jank_test/run_clojure_test_suite.cljc index ab609ac7a..48504d419 100644 --- a/compiler+runtime/test/bash/clojure-test-suite/src/jank_test/run_clojure_test_suite.cljc +++ b/compiler+runtime/test/bash/clojure-test-suite/src/jank_test/run_clojure_test_suite.cljc @@ -32,7 +32,7 @@ clojure.core-test.boolean-qmark clojure.core-test.bound-fn clojure.core-test.bound-fn-star - clojure.core-test.butlast + ; clojure.core-test.butlast ; "TODO: port int-array" ; clojure.core-test.byte ; TODO: port byte, Expecting whitespace after the last token. due to M. ; clojure.core-test.case ; analyze/invalid-case error: Unable to resolve symbol 'of'. ; clojure.core-test.char ; FIXME: Failing unicode character tests. @@ -71,7 +71,7 @@ clojure.core-test.false-qmark clojure.core-test.ffirst clojure.core-test.find - clojure.core-test.first + ; clojure.core-test.first ; "TODO: port to-array" clojure.core-test.float clojure.core-test.float-qmark clojure.core-test.fn-qmark @@ -169,7 +169,7 @@ ; clojure.core-test.rem ; FIXME: Failing tests. ; clojure.core-test.remove-watch ; TODO: port sync clojure.core-test.repeat - clojure.core-test.rest + ; clojure.core-test.rest ; "TODO: port int-array" clojure.core-test.reverse ; clojure.core-test.reversible-qmark ; TODO: port reversible?, TODO: port object-array ; clojure.core-test.rseq ; TODO: port rseq @@ -195,7 +195,7 @@ ; clojure.core-test.sorted-qmark ; TODO: port sorted-map-by, not yet implemented: sorted-set-by, TODO: port array-map, TODO: port object-array ; clojure.core-test.special-symbol-qmark ; TODO: port special-symbol? ; clojure.core-test.star ; FIXME: Failing tests. - clojure.core-test.star-squote + ; clojure.core-test.star-squote ; Assertion failed! is_tagged_pointer(val) clojure.core-test.str clojure.core-test.string-qmark ; clojure.core-test.subs ; FIXME: Failing tests. @@ -233,7 +233,7 @@ clojure.string-test.ends-with-qmark ; clojure.string-test.escape ; Uncaught exception: Can't convert character to integer. clojure.string-test.lower-case - ; clojure.string-test.reverse ; issues with unicode + clojure.string-test.reverse clojure.string-test.starts-with-qmark clojure.string-test.upper-case ])