diff --git a/externals/coda-oss/UnitTest/UnitTest.vcxproj b/externals/coda-oss/UnitTest/UnitTest.vcxproj index 19508e59c..d79dab6f9 100644 --- a/externals/coda-oss/UnitTest/UnitTest.vcxproj +++ b/externals/coda-oss/UnitTest/UnitTest.vcxproj @@ -325,10 +325,6 @@ true true - - true - true - true true diff --git a/externals/coda-oss/UnitTest/UnitTest.vcxproj.filters b/externals/coda-oss/UnitTest/UnitTest.vcxproj.filters index cd789e927..f838a0433 100644 --- a/externals/coda-oss/UnitTest/UnitTest.vcxproj.filters +++ b/externals/coda-oss/UnitTest/UnitTest.vcxproj.filters @@ -258,9 +258,6 @@ hdf5.lite - - sys - diff --git a/externals/coda-oss/UnitTest/sys.cpp b/externals/coda-oss/UnitTest/sys.cpp index 7bc804707..6561d2002 100644 --- a/externals/coda-oss/UnitTest/sys.cpp +++ b/externals/coda-oss/UnitTest/sys.cpp @@ -23,7 +23,6 @@ #include #include #include -#include namespace sys { @@ -59,8 +58,4 @@ TEST_CLASS(test_path){ public: #include "sys/unittests/test_path.cpp" }; -TEST_CLASS(test_ximd){ public: -#include "sys/unittests/test_ximd.cpp" -}; - } \ No newline at end of file diff --git a/externals/coda-oss/modules/c++/coda-oss.vcxproj.filters b/externals/coda-oss/modules/c++/coda-oss.vcxproj.filters index d755f9137..04a9d4d66 100644 --- a/externals/coda-oss/modules/c++/coda-oss.vcxproj.filters +++ b/externals/coda-oss/modules/c++/coda-oss.vcxproj.filters @@ -361,13 +361,13 @@ math.poly - str\polygon + polygon - str\polygon + polygon - str\polygon + polygon config @@ -1117,7 +1117,7 @@ math.linear - str\polygon + polygon cli @@ -1496,7 +1496,7 @@ {59f3d9a1-06d3-4779-aef2-cc55223c3017} - + {f2544ccb-0933-44c7-af39-cd986982af3d} diff --git a/externals/coda-oss/modules/c++/logging/source/Setup.cpp b/externals/coda-oss/modules/c++/logging/source/Setup.cpp index 345c6b516..b8ff117bd 100644 --- a/externals/coda-oss/modules/c++/logging/source/Setup.cpp +++ b/externals/coda-oss/modules/c++/logging/source/Setup.cpp @@ -35,7 +35,7 @@ std::unique_ptr logging::setupLogger(const path& program_, const std::string& logLevel, - const path& logFile_, + const path& logFile, const std::string& logFormat, size_t logCount, size_t logBytes) @@ -44,29 +44,25 @@ logging::setupLogger(const path& program_, std::unique_ptr log(new logging::Logger(program)); // setup logging level - std::string lev = logLevel; - str::upper(lev); + auto lev = str::upper(logLevel); str::trim(lev); - logging::LogLevel level = (lev.empty()) ? logging::LogLevel::LOG_WARNING : - logging::LogLevel(lev); + const auto level = lev.empty() ? logging::LogLevel::LOG_WARNING : logging::LogLevel(lev); // setup logging formatter std::unique_ptr formatter; - const auto logFile = logFile_.string(); - const auto file = str::lower(logFile); + const auto file = str::lower(logFile.string()); if (str::endsWith(file, ".xml")) { - formatter.reset( - new logging::XMLFormatter("", "")); + formatter = std::make_unique("", ""); } else { - formatter.reset(new logging::StandardFormatter(logFormat)); + formatter = std::make_unique(logFormat); } // setup logging handler std::unique_ptr logHandler; - if (file.empty() || file == "console") + if (file.empty() || (file == "console") || (file == "-")) logHandler.reset(new logging::StreamHandler()); else { diff --git a/externals/coda-oss/modules/c++/str/include/str/Manip.h b/externals/coda-oss/modules/c++/str/include/str/Manip.h index c9c96eb60..c50a37555 100644 --- a/externals/coda-oss/modules/c++/str/include/str/Manip.h +++ b/externals/coda-oss/modules/c++/str/include/str/Manip.h @@ -211,26 +211,30 @@ inline std::string upper(const std::string& s) // At this point, you might want to `lower()` and `upper()` for UTF-8 and/or // Windows-1252. That can be done, but ... our needs are mostly English (99.9%) -// with a very occassional smattering of French (Canada). We've gotten by this +// with a very occassional smattering of (Canadian-) French. We've gotten by this // long without being able to upper/lower 'ä' and 'Ä' and there's no current // requirement to do so. // // Furthermore, while Windows-1252 is easy as it's a single-byte encoding and -// covers many european languages, the standard is UTF-8. -// Upper/lower-casing in Unicode is quite a bit more complicated as there can be +// covers many european languages, the standard is UTF-8. Changing case +// with Unicode is quite a bit more complicated as there can be // numerous rules for various languages. For example, in German, the "old // rules" where that 'ß' was uppercased to "SS"; however, there is now a 'ẞ'. // And then there are semantics: in German, no word can begin with 'ß' (or 'ẞ') // making "ßanything" rather non-sensical. // // So for now (until there is a real use case), just "define these problems -// away" by not implementing `w1252_lower()`, `utf8_upper()`, etc. +// away" by not exposing `w1252_lower()`, `utf8_upper()`, etc. /* +// With Windows-1252 encoding, we can convert between 'ä' and 'Ä'. CODA_OSS_API void w1252_lower(std::string& s); CODA_OSS_API void w1252_upper(std::string& s); CODA_OSS_API void lower(str::W1252string& s); CODA_OSS_API void upper(str::W1252string& s); +// Hooking up UTF-8 for completeness and unit-testing. +// ** THESE ROUTINES ARE SLOW ** +// Performance improvements can be made, but nobody needs such right now. CODA_OSS_API void utf8_lower(std::string& s); CODA_OSS_API void utf8_upper(std::string& s); CODA_OSS_API void lower(coda_oss::u8string& s); @@ -244,6 +248,10 @@ CODA_OSS_API str::Windows1252_T to_w1252_lower(str::Windows1252_T); /***********************************************************************************/ +// Using std::transform() with ::toupper() is considerably slower than a lookup-table +CODA_OSS_API void ascii_lower(std::string& s); +CODA_OSS_API void ascii_upper(std::string& s); + /*! * Replaces any characters that are invalid in XML (&, <, >, ', ") with their * escaped counterparts diff --git a/externals/coda-oss/modules/c++/str/source/Encoding.cpp b/externals/coda-oss/modules/c++/str/source/Encoding.cpp index 80603dae9..c9833ea88 100644 --- a/externals/coda-oss/modules/c++/str/source/Encoding.cpp +++ b/externals/coda-oss/modules/c++/str/source/Encoding.cpp @@ -29,7 +29,6 @@ #endif #include -#include #include #include #include @@ -49,34 +48,24 @@ CODA_OSS_disable_warning(-Wshadow) #include "str/utf8.h" CODA_OSS_disable_warning_pop -//// "sys" depends on "str" so can't use sys::PlatformType -//enum class PlatformType -//{ -// Windows, -// Linux, -// // MacOS -//}; -#if _WIN32 -//static constexpr auto Platform = PlatformType::Windows; -#elif defined(_POSIX_C_SOURCE) -//static constexpr auto Platform = PlatformType::Linux; -#else -#error "Unknown platform" -#endif - // Need to look up characters from \x80 (EURO SIGN) to \x9F (LATIN CAPITAL LETTER Y WITH DIAERESIS) // in a map: http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT -inline coda_oss::u8string utf8_(char32_t i) +static inline coda_oss::u8string utf8_(char32_t i) { const auto ch = gsl::narrow(i); return str::to_u8string(std::u32string{ch}); } +// https://en.wikipedia.org/wiki/Windows-1252 +// > According to the information on Microsoft's and the Unicode Consortium's +// > websites, positions 81, 8D, 8F, 90, and 9D are unused; however, the +// > Windows API `MultiByteToWideChar` maps these to the corresponding +// > C1 control codes. The "best fit" mapping documents this behavior, too. static const auto& Windows1252_x80_x9F_to_u8string_() { static const std::map retval{ - {U'\x80', utf8_(U'\x20AC')} // EURO SIGN - // , {U'\x81, replacement_character } // UNDEFINED + {U'\x80', utf8_(U'\x20AC') } // EURO SIGN + , {U'\x81', utf8_(U'\x0081') } // UNDEFINED; _bstr_t just preserves these values, do the same // , {U'\x81', replacement_character } // UNDEFINED , {U'\x82', utf8_(U'\x201A') } // SINGLE LOW-9 QUOTATION MARK , {U'\x83', utf8_(U'\x0192') } // LATIN SMALL LETTER F WITH HOOK , {U'\x84', utf8_(U'\x201E') } // DOUBLE LOW-9 QUOTATION MARK @@ -88,10 +77,10 @@ static const auto& Windows1252_x80_x9F_to_u8string_() , {U'\x8A', utf8_(U'\x0160') } // LATIN CAPITAL LETTER S WITH CARON , {U'\x8B', utf8_(U'\x2039') } // SINGLE LEFT-POINTING ANGLE QUOTATION MARK , {U'\x8C', utf8_(U'\x0152') } // LATIN CAPITAL LIGATURE OE - //, {U'\x8D, replacement_character } // UNDEFINED + , {U'\x8D', utf8_(U'\x008D') } // UNDEFINED; _bstr_t just preserves these values, do the same // , {U'\x8D', replacement_character } // UNDEFINED , {U'\x8E', utf8_(U'\x017D') } // LATIN CAPITAL LETTER Z WITH CARON - //, {U'\x8F, replacement_character } // UNDEFINED - //, {U'\x90, replacement_character } // UNDEFINED + , {U'\x8F', utf8_(U'\x008F') } // UNDEFINED; _bstr_t just preserves these values, do the same // , {U'\x8F', replacement_character } // UNDEFINED + , {U'\x90', utf8_(U'\x0090') } // UNDEFINED; _bstr_t just preserves these values, do the same // , {U'\x90', replacement_character } // UNDEFINED , {U'\x91', utf8_(U'\x2018') } // LEFT SINGLE QUOTATION MARK , {U'\x92', utf8_(U'\x2019') } // RIGHT SINGLE QUOTATION MARK , {U'\x93', utf8_(U'\x201C') } // LEFT DOUBLE QUOTATION MARK @@ -104,7 +93,7 @@ static const auto& Windows1252_x80_x9F_to_u8string_() , {U'\x9A', utf8_(U'\x0161') } // LATIN SMALL LETTER S WITH CARON , {U'\x9B', utf8_(U'\x203A') } // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK , {U'\x9C', utf8_(U'\x0153') } // LATIN SMALL LIGATURE OE - //, {U'\x9D, replacement_character } // UNDEFINED + , {U'\x9D', utf8_(U'\x009D') } // UNDEFINED; _bstr_t just preserves these values, do the same // , {U'\x9D', replacement_character } // UNDEFINED , {U'\x9E', utf8_(U'\x017E') } // LATIN SMALL LETTER Z WITH CARON , {U'\x9F', utf8_(U'\x0178') } // LATIN CAPITAL LETTER Y WITH DIAERESIS }; @@ -114,19 +103,31 @@ static auto Windows1252_to_u8string() { auto retval = Windows1252_x80_x9F_to_u8string_(); - // Add the ISO8859-1 values to the map too. 1) We're already looking + using value_type = coda_oss::u8string::value_type; + // Add the ASCII values to the map too. 1) We're already looking // in the map anyway for Windows-1252 characters. 2) Need map // entires for conversion from UTF-8 to Windows-1252. + for (char32_t ch = U'\x00'; ch < U'\x80'; ch++) + { + assert(retval.find(ch) == retval.end()); // be sure we're not clobbering anything! + + coda_oss::u8string s {static_cast(ch)}; + retval[ch] = std::move(s); + } + + // Ditto for ISO8859-1 ... for (char32_t ch = U'\xA0'; ch <= U'\xff'; ch++) { + assert(retval.find(ch) == retval.end()); // be sure we're not clobbering anything! + // ISO8859-1 can be converted to UTF-8 with bit-twiddling - + // // https://stackoverflow.com/questions/4059775/convert-iso-8859-1-strings-to-utf-8-in-c-c // *out++=0xc2+(*in>0xbf), *out++=(*in++&0x3f)+0x80; const auto b1 = 0xc2 + (ch > 0xbf); const auto b2 = (ch & 0x3f) + 0x80; - coda_oss::u8string s {static_cast(b1)}; - s += coda_oss::u8string {static_cast(b2)}; + coda_oss::u8string s{static_cast(b1)}; + s += coda_oss::u8string{static_cast(b2)}; retval[ch] = std::move(s); } @@ -154,75 +155,72 @@ inline void append(std::u32string& result, const coda_oss::u8string& utf8) } template -static void fromWindows1252_(str::W1252string::value_type ch, std::basic_string& result, bool strict=false) +static void fromWindows1252_(str::W1252string::value_type ch, std::basic_string& result) { - // ASCII is the same in UTF-8 - if (ch < static_cast(0x80)) - { - using value_type = typename std::basic_string::value_type; - result += static_cast(ch); // ASCII - return; - } - static const auto map = Windows1252_to_u8string(); - const auto ch32 = static_cast(ch); + const auto ch32 = gsl::narrow(ch); const auto it = map.find(ch32); if (it != map.end()) { append(result, it->second); return; } - - switch (static_cast(ch)) + + // https://en.wikipedia.org/wiki/Windows-1252 + // > According to the information on Microsoft's and the Unicode + // Consortium's > websites, positions 81, 8D, 8F, 90, and 9D are unused; + // however, the > Windows API `MultiByteToWideChar` maps these to the + // corresponding > C1 control codes. The "best fit" mapping documents this + // behavior, too. + // static const auto replacement_character = utf8_(U'\xfffd'); + // append(result, replacement_character); + throw std::logic_error("Windows-1252 value not in map."); +} +template +class Windows1252_to_basic_string final +{ + static auto make_Windows1252_lookup() { - case 0x81: - case 0x8d: - case 0x8f: - case 0x90: - case 0x9d: - { - if (strict) + std::vector> retval(0xff + 1); + for (size_t i = 0; i <= 0xff; i++) { - // If the input text contains a character that isn't defined in Windows-1252; return a - // "replacement character." Yes, this will **corrupt** the input data as information is lost: - // https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character - // - // Or ... https://en.wikipedia.org/wiki/Windows-1252 - // > According to the information on Microsoft's and the Unicode - // > Consortium's websites, positions 81, 8D, 8F, 90, and 9D are - // > unused; however, the Windows API `MultiByteToWideChar` maps these - // > to the corresponding C1 control codes. The "best fit" mapping - // > documents this behavior, too. - static const coda_oss::u8string replacement_character = utf8_(U'\xfffd'); - append(result, replacement_character); + const auto ch = static_cast(i); + fromWindows1252_(ch, retval[i]); } - else - { - // _bstr_t just preserves these values, do the same - append(result, utf8_(ch32)); - } - break; + return retval; } - default: - throw std::invalid_argument("Invalid Windows-1252 character."); + +public: + static const auto& getLookup() + { + static const auto lookup = make_Windows1252_lookup(); + return lookup; } -} -template -inline void w1252_to_string(str::W1252string::const_pointer p, size_t sz, std::basic_string& result) -{ - for (size_t i = 0; i < sz; i++) + + Windows1252_to_basic_string() = default; + auto operator()(str::W1252string::const_pointer p, size_t sz) const { - fromWindows1252_(p[i], result); + static const auto& lookup = getLookup(); + + std::basic_string retval; + for (size_t i = 0; i < sz; i++) + { + const auto ch = gsl::narrow(p[i]); + retval += lookup[ch]; + } + return retval; } -} -template -inline void w1252to8(str::W1252string::const_pointer p, size_t sz, std::basic_string& result) +}; +template +static inline void w1252_to_basic_string(str::W1252string::const_pointer p, size_t sz, std::basic_string& result) { - w1252_to_string(p, sz, result); + static const Windows1252_to_basic_string convert; + result = convert(p, sz); } -inline void w1252to16(str::W1252string::const_pointer p, size_t sz, std::u16string& result) + +static inline void w1252to16(str::W1252string::const_pointer p, size_t sz, std::u16string& result) { - w1252_to_string(p, sz, result); + w1252_to_basic_string(p, sz, result); #if defined(_WIN32) && (!defined(_NDEBUG) || defined(DEBUG)) const _bstr_t bstr(std::string(str::details::cast(p), sz).c_str()); // no _bstr_t ctor taking sz @@ -230,20 +228,9 @@ inline void w1252to16(str::W1252string::const_pointer p, size_t sz, std::u16stri assert(result == str::str(wstr)); #endif } -inline void w1252to32(str::W1252string::const_pointer p, size_t sz, std::u32string& result) +static inline void w1252to32(str::W1252string::const_pointer p, size_t sz, std::u32string& result) { - w1252_to_string(p, sz, result); -} - -template -auto kv_to_vk(const std::map& kv) -{ - std::map retval; - for (const auto& p : kv) - { - retval[p.second] = p.first; - } - return retval; + w1252_to_basic_string(p, sz, result); } static void get_next_utf8_byte(coda_oss::u8string::const_pointer p, size_t sz, @@ -256,115 +243,127 @@ static void get_next_utf8_byte(coda_oss::u8string::const_pointer p, size_t sz, i++; // move to next byte // Bytes 2, 3 and 4 are always >= 0x80 (10xxxxxx), see https://en.wikipedia.org/wiki/UTF-8 - const auto b = static_cast(p[i]); - if (b < static_cast(0x80)) // 10xxxxxx + const auto b = gsl::narrow(p[i]); + if (b < gsl::narrow(0x80)) // 10xxxxxx { throw std::invalid_argument("Invalid next byte in UTF-8 encoding."); } utf8 += coda_oss::u8string{static_cast(b)}; } -template -static void utf8to1252(coda_oss::u8string::const_pointer p, size_t sz, std::basic_string& result, bool strict=false) +static void get_utf8_string(coda_oss::u8string::const_pointer p, size_t sz, size_t& i, coda_oss::u8string& utf8) { - using value_type = TChar; - for (size_t i = 0; i < sz; i++) + const auto b1 = gsl::narrow(p[i]); + if (b1 >= 0x80) // 0xxxxxxx { - const auto b1 = static_cast(p[i]); - - // ASCII is the same in UTF-8 - if (b1 < 0x80) // 0xxxxxxx - { - result += static_cast(b1); // ASCII - continue; - } - - auto utf8 = coda_oss::u8string{static_cast(b1)}; - get_next_utf8_byte(p, sz, i, utf8); if (b1 >= 0xE0) // 1110xxxx { // should be a 3- or 4-byte sequence - get_next_utf8_byte(p, sz, i, utf8); + get_next_utf8_byte(p, sz, i, utf8); if (b1 >= 0xF0) // 1111xxx { // should be a 4-byte sequence - get_next_utf8_byte(p, sz, i, utf8); + get_next_utf8_byte(p, sz, i, utf8); } } + } +} - static const auto map = kv_to_vk(Windows1252_to_u8string()); - const auto it = map.find(utf8); +template // may be stored in std::string or str::Windows1252 +class Utf_to_Windows1252 final +{ + template + void utf_to_1252(const TMap& map, const TUtf& utf, std::basic_string& result) const + { + auto w1252 = static_cast(0x7F); // + const auto it = map.find(utf); if (it != map.end()) { - result += static_cast(it->second); - } - else if (strict) - { - throw std::invalid_argument("UTF-8 sequence can't be converted to Windows-1252."); - //assert("UTF-8 sequence can't be converted to Windows-1252." && 0); - //result += static_cast(0x7F); // + w1252 = static_cast(it->second); } + #ifndef NDEBUG else { - // _bstr_t preserves these values - if (utf8.length() == 2) - { - result += static_cast(utf8[1]); - } - else - { - assert("UTF-8 sequence can't be converted to Windows-1252." && 0); - result += static_cast(0x7F); // - } + assert("UTF sequence can't be converted to Windows-1252." && 0); } + #endif // NDEBUG + result += w1252; } -} -static auto u16_to_Windows1252() -{ - // Find the corresponding UTF-16 value for every Windows-1252 input; - // obviously, most UTF-16 values can't be converted. Skip the first half - // as they're the same for ASCII. - std::map retval; - for (uint16_t i = 0x0080; i <= 0x00ff; i++) // **not** `uint8_t` to avoid wrap-around + static auto make_u16_map() { - const auto ch = static_cast(i); - const auto u16 = str::to_u16string(&ch, 1); - assert(u16.length() == 1); - retval[u16[0]] = ch; + // Find the corresponding UTF-16 value for every Windows-1252 input; + // obviously, most UTF-16 values can't be converted. + auto&& lookup = Windows1252_to_basic_string::getLookup(); + + std::map retval; + for (size_t i = 0; i <= 0xff; i++) // **not** `uint8_t` to avoid wrap-around + { + const auto u16 = lookup[i]; + assert(u16.length() == 1); // all values in Basic Multi-lingual Plane (BMP); no emojis, etc. + const auto ch = static_cast(i); + retval[u16[0]] = ch; + } + return retval; } - return retval; -} -static inline void utf16to1252(std::u16string::const_pointer p, size_t sz, std::string& result, bool strict=false) -{ - using value_type = std::string::value_type; - static const auto map = u16_to_Windows1252(); - for (size_t i = 0; i < sz; i++) + static auto make_utf8_map() { - const auto ch = p[i]; + // Find the corresponding UTF-8 value for every Windows-1252 input. + static const auto map = make_u16_map(); - if (ch < 0x0080) // ASCII + // Convert UTF-16 to UTF-8 + std::map retval; + for (auto&& kv : map) { - result += gsl::narrow(ch); - continue; + retval[utf8_(kv.first)] = kv.second; } + return retval; + } - const auto it = map.find(ch); - if (it != map.end()) - { - result += static_cast(it->second); - } - else if (strict) +public: + Utf_to_Windows1252() = default; + + auto operator()(std::u16string::const_pointer p, size_t sz) const + { + static const auto map = make_u16_map(); + + std::basic_string retval; + for (size_t i = 0; i < sz; i++) { - throw std::invalid_argument("UTF-16 sequence can't be converted to Windows-1252."); + const auto utf16 = p[i]; + utf_to_1252(map, utf16, retval); } - else + return retval; + } + + auto operator()(coda_oss::u8string::const_pointer p, size_t sz) const + { + static const auto map = make_utf8_map(); + + std::basic_string retval; + for (size_t i = 0; i < sz; i++) { - assert("UTF-16 sequence can't be converted to Windows-1252." && 0); - result += static_cast(0x7F); // + auto utf8 = coda_oss::u8string{p[i]}; + get_utf8_string(p, sz, i, utf8); + + utf_to_1252(map, utf8, retval); } + return retval; } +}; + +template +static inline void utf8to1252(coda_oss::u8string::const_pointer p, size_t sz, std::basic_string& result) +{ + static const Utf_to_Windows1252 convert; + result = convert(p, sz); +} + +static inline void utf16to1252(std::u16string::const_pointer p, size_t sz, std::string& result) +{ + static const Utf_to_Windows1252 convert; + result = convert(p, sz); } struct back_inserter final @@ -391,12 +390,30 @@ inline auto to_uXXstring(const std::basic_string& s) return str::to_u32string(p, s.length()); // assume std::wstring is UTF-32 everywhere except Windows #endif } -template -static std::wstring to_wstring_(const std::basic_string& s, bool is_utf8) + +template +struct basic_string_to_uXXstring_ final { }; +template +struct basic_string_to_uXXstring_ final +{ + auto operator()(const std::basic_string& s) const + { + return to_uXXstring(s); + } +}; +template +struct basic_string_to_uXXstring_ final { - - const auto result = is_utf8 ? to_uXXstring(s) - : to_uXXstring(s); + auto operator()(const std::basic_string& s) const + { + return to_uXXstring(s); + } +}; +template +inline auto to_wstring_(const std::basic_string&s) +{ + static const basic_string_to_uXXstring_ convert; + const auto result = convert(s); return str::str(result); } @@ -419,7 +436,7 @@ std::string str::testing::to_string(const str::W1252string& s) return str(s); #else std::string retval; - w1252to8(s.c_str(), s.length(), retval); + w1252_to_basic_string(s.c_str(), s.length(), retval); return retval; #endif } @@ -440,19 +457,19 @@ std::string str::details::to_string(const std::wstring& s) std::wstring str::details::to_wstring(const std::string& s) { - #if _WIN32 - return to_wstring_(s, false /*is_utf8*/); // Input is Windows-1252 on Windows + #if _WIN32 + return to_wstring_(s); // Input is Windows-1252 on Windows #else - return to_wstring_(s, true /*is_utf8*/); // Input is UTF-8 everywhere except Windows + return to_wstring_(s); // Input is UTF-8 everywhere except Windows #endif } std::wstring str::details::to_wstring(const coda_oss::u8string& s) { - return to_wstring_(s, true /*is_utf8*/); + return to_wstring_(s); } std::wstring str::testing::to_wstring(const str::W1252string& s) { - return to_wstring_(s, false /*is_utf8*/); + return to_wstring_(s); } /***********************************************************************************/ @@ -510,6 +527,6 @@ coda_oss::u8string str::to_u8string(std::u32string::const_pointer p, size_t sz) coda_oss::u8string str::to_u8string(W1252string::const_pointer p, size_t sz) { coda_oss::u8string retval; - w1252to8(p, sz, retval); + w1252_to_basic_string(p, sz, retval); return retval; } diff --git a/externals/coda-oss/modules/c++/str/source/Manip.cpp b/externals/coda-oss/modules/c++/str/source/Manip.cpp index 1d6bae1c2..87b06acdf 100644 --- a/externals/coda-oss/modules/c++/str/source/Manip.cpp +++ b/externals/coda-oss/modules/c++/str/source/Manip.cpp @@ -339,28 +339,31 @@ inline char to_w1252_upper_(char ch) } // See chart at: https://en.wikipedia.org/wiki/Windows-1252 + const auto u8 = static_cast(ch); + constexpr uint8_t s_with_caron = 0x9a /* š */; constexpr uint8_t oe = 0x9c /* œ */; constexpr uint8_t z_with_caron = 0x9e /* ž */; - constexpr uint8_t a_with_grave = 0xe0 /* à */; - constexpr uint8_t o_with_diaeresis = 0xf6 /* ö */; - constexpr uint8_t o_with_slash = 0xf8 /* ø */; - constexpr uint8_t small_thorn = 0xfe /* þ */; - constexpr uint8_t y_with_diaeresis = 0xff /* ÿ */; - - const auto u8 = static_cast(ch); if ((u8 == s_with_caron) || (u8 == oe) || (u8 == z_with_caron)) { return ch ^ 0x10; } + + constexpr uint8_t a_with_grave = 0xe0 /* à */; + constexpr uint8_t o_with_diaeresis = 0xf6 /* ö */; if ((u8 >= a_with_grave) && (u8 <= o_with_diaeresis)) { return ch ^ 0x20; } + // U+00F7 ÷ DIVISION SIGN + constexpr uint8_t o_with_slash = 0xf8 /* ø */; + constexpr uint8_t small_thorn = 0xfe /* þ */; if ((u8 >= o_with_slash) && (u8 <= small_thorn)) { return ch ^ 0x20; } + + constexpr uint8_t y_with_diaeresis = 0xff /* ÿ */; if (u8 == y_with_diaeresis) { constexpr uint8_t Y_with_diaeresis = 0x9f /* Ÿ */; @@ -382,33 +385,38 @@ inline char to_w1252_lower_(char ch) return ch | 0x20; } + // See chart at: https://en.wikipedia.org/wiki/Windows-1252 + const auto u8 = static_cast(ch); + constexpr uint8_t S_with_caron = 0x8a /* Š */; constexpr uint8_t OE = 0x8c /*Œ */; constexpr uint8_t Z_with_caron = 0x8e /* Ž */; - constexpr uint8_t Y_with_diaeresis = 0x9f /* Ÿ */; - constexpr uint8_t A_with_grave = 0xc0 /* À */; - constexpr uint8_t O_with_diaeresis = 0xd6 /* Ö */; - constexpr uint8_t O_with_slash = 0xd8 /* Ø */; - constexpr uint8_t capital_thorn = 0xde /* Þ */; - - const auto u8 = static_cast(ch); if ((u8 == S_with_caron) || (u8 == OE) || (u8 == Z_with_caron)) { return ch | 0x10; } + + constexpr uint8_t Y_with_diaeresis = 0x9f /* Ÿ */; if (u8 == Y_with_diaeresis) { constexpr uint8_t y_with_diaeresis = 0xff /* ÿ */; return y_with_diaeresis; } + + constexpr uint8_t A_with_grave = 0xc0 /* À */; + constexpr uint8_t O_with_diaeresis = 0xd6 /* Ö */; if ((u8 >= A_with_grave) && (u8 <= O_with_diaeresis)) { return ch | 0x20; } + // U+00D7 × MULTIPLICATION SIGN + constexpr uint8_t O_with_slash = 0xd8 /* Ø */; + constexpr uint8_t capital_thorn = 0xde /* Þ */; if ((u8 >= O_with_slash) && (u8 <= capital_thorn)) { return ch | 0x20; } + return ch; } str::Windows1252_T to_w1252_lower(str::Windows1252_T ch) @@ -417,6 +425,72 @@ str::Windows1252_T to_w1252_lower(str::Windows1252_T ch) return static_cast(retval); } +static const auto& w1252_upper_lookup() +{ + static std::array lookup_; + static const auto& lookup = make_lookup(lookup_, to_w1252_upper_); + return lookup; +} +void w1252_upper(std::string& w1252) +{ + do_lookup(w1252, w1252_upper_lookup()); +} +void upper(str::W1252string& s) +{ + do_lookup(s, w1252_upper_lookup()); +} + +static const auto& w1252_lower_lookup() +{ + static std::array lookup_; + static const auto& lookup = make_lookup(lookup_, to_w1252_lower_); + return lookup; +} +void w1252_lower(std::string& w1252) +{ + do_lookup(w1252, w1252_lower_lookup()); +} +void lower(str::W1252string& s) +{ + do_lookup(s, w1252_lower_lookup()); +} + +// These routines are SLOW ... yes, they can be made faster +// but nobody needs that right now. +inline auto utf8_convert(str::W1252string& w1252, void (*convert)(str::W1252string&)) +{ + convert(w1252); // upper() or lower() for Windows-1252 + return to_u8string(w1252); +} +inline void utf8_convert(std::string& strUtf8, void (*convert)(str::W1252string&)) +{ + auto w1252 = to_w1252string(str::str(strUtf8)); + const auto utf8 = utf8_convert(w1252, convert); + strUtf8 = str::str(utf8); +} +void utf8_upper(std::string& strUtf8) +{ + utf8_convert(strUtf8, upper); +} +void utf8_lower(std::string& strUtf8) +{ + utf8_convert(strUtf8, lower); +} + +inline void utf8_convert(coda_oss::u8string& s, void (*convert)(str::W1252string&)) +{ + auto w1252 = to_w1252string(s); + s = utf8_convert(w1252, convert); +} +void lower(coda_oss::u8string& s) +{ + utf8_convert(s, lower); +} +void upper(coda_oss::u8string& s) +{ + utf8_convert(s, upper); +} + void escapeForXML(std::string& str) { // & needs to be first or else it'll mess up the other characters that we replace diff --git a/six/modules/c++/six.sicd/include/six/sicd/NearestNeighbors.h b/six/modules/c++/six.sicd/include/six/sicd/NearestNeighbors.h index e9629551e..b19abeda0 100644 --- a/six/modules/c++/six.sicd/include/six/sicd/NearestNeighbors.h +++ b/six/modules/c++/six.sicd/include/six/sicd/NearestNeighbors.h @@ -58,6 +58,16 @@ struct NearestNeighbors final void nearest_neighbors_par(std::span inputs, std::span results) const; #if SIX_sicd_ComplexToAMP8IPHS8I_unseq + #if SIX_sicd_has_sisd + // This uses the UNSEQ code/logic with the specififed execution policy. The intent + // is to run the generic SIMD-enabled code with non-SIMD types. + // + // See https://mattkretz.github.io/2021/07/29/data-structure-vectorization.html + // "... if you’re careful about `if` statements, you can instantiate the template with either an arithmetic type or a [simd] type." + // and https://mattkretz.github.io/2019/07/25/making-the-conditional-operator-overloadable.html + void unseq_nearest_neighbors(execution_policy, std::span inputs, std::span results) const; + #endif // SIX_sicd_has_sisd + void nearest_neighbors_unseq(std::span inputs, std::span results) const; void nearest_neighbors_par_unseq(std::span inputs, std::span results) const; #endif diff --git a/six/modules/c++/six.sicd/source/ComplexToAMP8IPHS8I.cpp b/six/modules/c++/six.sicd/source/ComplexToAMP8IPHS8I.cpp index 367f851ac..e3d632735 100644 --- a/six/modules/c++/six.sicd/source/ComplexToAMP8IPHS8I.cpp +++ b/six/modules/c++/six.sicd/source/ComplexToAMP8IPHS8I.cpp @@ -156,7 +156,7 @@ six::sicd::details::ComplexToAMP8IPHS8I::ComplexToAMP8IPHS8I(const six::Amplitud phase_directions_.value[i] = { x, y }; // Only need the parallel array when using the "vectorclass" library. - #ifdef SIX_sicd_has_VCL + #if SIX_sicd_has_VCL || SIX_sicd_has_sisd phase_directions_.real[i] = phase_directions_.value[i].real(); phase_directions_.imag[i] = phase_directions_.value[i].imag(); #endif diff --git a/six/modules/c++/six.sicd/source/NearestNeighbors_unseq.cpp b/six/modules/c++/six.sicd/source/NearestNeighbors_unseq.cpp index 142f953f2..59601444b 100644 --- a/six/modules/c++/six.sicd/source/NearestNeighbors_unseq.cpp +++ b/six/modules/c++/six.sicd/source/NearestNeighbors_unseq.cpp @@ -43,6 +43,77 @@ using zfloat = six::zfloat; using AMP8I_PHS8I = six::AMP8I_PHS8I_t; +#if SIX_sicd_has_sisd + +using sisd_intv = int; +using sisd_intv_mask = bool; + +using sisd_floatv = float; +using sisd_floatv_mask = bool; + +constexpr auto sisd_elements_per_iteration = 1; +inline size_t size(sisd_intv) noexcept { return sisd_elements_per_iteration; } +inline ptrdiff_t ssize(sisd_intv v) noexcept { return gsl::narrow(size(v)); } +inline size_t size(sisd_floatv) noexcept { return sisd_elements_per_iteration; } +inline ptrdiff_t ssize(sisd_floatv v) noexcept { return gsl::narrow(size(v)); } + +using sisd_zfloatv = std::complex; +inline size_t size(sisd_zfloatv) noexcept { return sisd_elements_per_iteration; } +inline ptrdiff_t ssize(sisd_zfloatv v) noexcept { return gsl::narrow(size(v)); } + +inline void copy_from(std::span p, sisd_floatv& result) +{ + assert(p.size() == size(result)); + result = p[0]; +} +inline void copy_from(std::span p, sisd_zfloatv& result) +{ + assert(p.size() == size(result)); + result = p[0]; +} + +static inline sisd_intv roundi(sisd_floatv v) // match vcl::roundi() +{ + return gsl::narrow_cast(std::round(v)); +} + +template +inline auto lookup(sisd_intv indexv, const std::array& phase_directions) +{ + return phase_directions[indexv]; +} +inline auto lookup(sisd_intv indexv, std::span magnitudes) +{ + assert(magnitudes.size() == six::AmplitudeTableSize); + + // The index may be out of range. This is expected because `i` might be "don't care." + if ((indexv >= 0) && (indexv < std::ssize(magnitudes))) + { + return magnitudes[indexv]; + } + return NAN; // propogate "don't care" +} + +static inline auto simd_select(bool test, sisd_intv t, sisd_intv f) +{ + return test ? t : f; +} +static inline auto simd_select(bool test, sisd_floatv t, sisd_floatv f) +{ + return test ? t : f; +} + +inline bool any_of(sisd_intv m) +{ + return m != 0; +} +inline bool all_of(sisd_intv m) +{ + return m != 0; +} + +#endif // SIX_sicd_has_sisd + #if SIX_sicd_has_VCL #define VCL_NAMESPACE vcl @@ -54,19 +125,22 @@ using AMP8I_PHS8I = six::AMP8I_PHS8I_t; #pragma warning(disable: 4723) // potential divide by 0 #endif #include "six/sicd/vectorclass/version2/vectorclass.h" -#include "six/sicd/vectorclass/version2/vector.h" #include "six/sicd/vectorclass/version2/vectormath_trig.h" #include "six/sicd/vectorclass/complex/complexvec1.h" +#include "six/sicd/vectorclass/simd/simd.h" #if _MSC_VER #pragma warning(pop) #endif constexpr auto vcl_elements_per_iteration = 8; -using vcl_intv = vcl::Vec; -using vcl_floatv = vcl::Vec; +using vcl_intv = vcl::Vec8i; +using vcl_floatv = vcl::Vec8f; -template -inline ptrdiff_t ssize(const vcl::Vec& v) noexcept +inline ptrdiff_t ssize(const vcl_intv& v) noexcept +{ + return v.size(); +} +inline ptrdiff_t ssize(const vcl_floatv& v) noexcept { return v.size(); } @@ -89,15 +163,25 @@ inline int ssize(const vcl_zfloatv& z) noexcept return z.size(); } -template -inline auto if_add(const T& f, const vcl_floatv& a, float b) +inline bool any_of(const vcl_intv& m) { - return vcl::if_add(f, a, b); + return horizontal_or(m); +} +inline bool all_of(const vcl_intv& m) +{ + return horizontal_and(m); } -inline bool any_of(const vcl_intv& m) + +template +static inline auto simd_select(const TMask& test, const vcl_intv& t, const vcl_intv& f) { - return horizontal_or(m); + return select(test, t, f); +} +template +static inline auto simd_select(const TMask& test, float t, float f) +{ + return select(test, vcl_floatv{ t }, vcl_floatv{ f }); } inline void copy_from(std::span p, vcl_floatv& result) @@ -105,7 +189,7 @@ inline void copy_from(std::span p, vcl_floatv& result) assert(p.size() == result.size()); result.load(p.data()); } -inline auto copy_from(std::span p, vcl_zfloatv& result) +inline void copy_from(std::span p, vcl_zfloatv& result) { // https://en.cppreference.com/w/cpp/numeric/complex // > For any pointer to an element of an array of `std::complex` named `p` and any valid array index `i`, ... @@ -114,14 +198,14 @@ inline auto copy_from(std::span p, vcl_zfloatv& result) } template -inline auto lookup(const vcl_intv& zindex, const std::array& phase_directions) +inline auto lookup(const vcl_intv& indexv, const std::array& phase_directions) { - return vcl::lookup(zindex, phase_directions.data()); + return vcl::lookup(indexv, phase_directions.data()); } -inline auto lookup(const vcl_intv& zindex, std::span magnitudes) +inline auto lookup(const vcl_intv& indexv, std::span magnitudes) { assert(magnitudes.size() == six::AmplitudeTableSize); - return vcl::lookup(zindex, magnitudes.data()); + return vcl::lookup(indexv, magnitudes.data()); } #endif // SIX_sicd_has_VCL @@ -162,7 +246,7 @@ static inline void copy_from(std::span p, ximd_floatv& result) } template -static auto ximd_select_(const TTest& test, const TResult& t, const TResult& f) +static auto ximd_simd_select_(const TTest& test, const TResult& t, const TResult& f) { TResult retval; for (size_t i = 0; i < test.size(); i++) @@ -171,13 +255,13 @@ static auto ximd_select_(const TTest& test, const TResult& t, const TResult& f) } return retval; } -static inline auto select(const ximd_floatv_mask& test, const ximd_floatv& t, const ximd_floatv& f) +static inline auto simd_select(const ximd_intv_mask& test, const ximd_intv& t, const ximd_intv& f) { - return ximd_select_(test, t, f); + return ximd_simd_select_(test, t, f); } -static inline auto select(const ximd_intv_mask& test, const ximd_intv& t, const ximd_intv& f) +static inline auto simd_select(const ximd_floatv_mask& test, float t, float f) { - return ximd_select_(test, t, f); + return ximd_simd_select_(test, ximd_floatv{ t }, ximd_floatv{ f }); } #endif // SIX_sicd_has_ximd @@ -194,13 +278,10 @@ namespace stdx using namespace std::experimental::__proposed; } -template -using simd = stdx::simd; - -using simd_intv = simd; +using simd_intv = stdx::simd; using simd_intv_mask = typename simd_intv::mask_type; -using simd_floatv = simd; +using simd_floatv = stdx::rebind_simd; using simd_floatv_mask = typename simd_floatv::mask_type; constexpr auto simd_elements_per_iteration = simd_floatv::size(); @@ -217,14 +298,14 @@ static inline auto generate(TGenerator&& generator, simd) return simd(generator); } -static inline auto copy_from(std::span p, simd_floatv& result) +static inline void copy_from(std::span p, simd_floatv& result) { assert(p.size() == result.size()); result.copy_from(p.data(), stdx::element_aligned); } template -static auto simd_select_(const TTest& test, const TResult& t, const TResult& f) +static auto simd_simd_select_(const TTest& test, const TResult& t, const TResult& f) { // https://en.cppreference.com/w/cpp/experimental/simd/where_expression // > ... All other elements are left unchanged. @@ -234,18 +315,18 @@ static auto simd_select_(const TTest& test, const TResult& t, const TResult& f) return retval; } template -static inline auto select(const TMask& test_, const simd_floatv& t, const simd_floatv& f) +static inline auto simd_select(const TMask& test_, const simd_intv& t, const simd_intv& f) { //const auto test = test_.__cvt(); // https://github.com/VcDevel/std-simd/issues/41 - const auto test = stdx::static_simd_cast(test_); // https://github.com/VcDevel/std-simd/issues/41 - return simd_select_(test, t, f); + const auto test = stdx::static_simd_cast(test_); // https://github.com/VcDevel/std-simd/issues/41 + return simd_simd_select_(test, t, f); } template -static inline auto select(const TMask& test_, const simd_intv& t, const simd_intv& f) +static inline auto simd_select(const TMask& test_, const simd_floatv& t, const simd_floatv& f) { //const auto test = test_.__cvt(); // https://github.com/VcDevel/std-simd/issues/41 - const auto test = stdx::static_simd_cast(test_); // https://github.com/VcDevel/std-simd/issues/41 - return simd_select_(test, t, f); + const auto test = stdx::static_simd_cast(test_); // https://github.com/VcDevel/std-simd/issues/41 + return simd_simd_select_(test, t, f); } #endif // SIX_sicd_has_simd @@ -314,60 +395,36 @@ static inline void copy_from(std::span mem, zfloatv& resul // is simple and easy to understand. Simplify with concepts in C++20? template -static auto roundi_(const FloatV& v) // match vcl::roundi() +static auto lround_(const FloatV& v) { - const auto rounded = round(v); - const auto generate_roundi = [&](size_t i) - { return static_cast(rounded[i]); }; - return generate(generate_roundi, IntV{}); + return lround(v); } #if SIX_sicd_has_ximd static inline auto roundi(const ximd_floatv& v) // match vcl::roundi() { - return roundi_(v); + return lround_(v); } #endif #if SIX_sicd_has_simd static inline auto roundi(const simd_floatv& v) // match vcl::roundi() { - return roundi_(v); -} -#endif - -template -static auto if_add_(const TFloatVMask& m, const TFloatV& v, typename TFloatV::value_type c) -{ - const auto generate_add = [&](size_t i) { - return m[i] ? v[i] + c : v[i]; - }; - return generate(generate_add, TFloatV{}); -} -#if SIX_sicd_has_ximd -static inline auto if_add(const ximd_floatv_mask& m, const ximd_floatv& v, typename ximd_floatv::value_type c) -{ - return if_add_(m, v, c); -} -#endif -#if SIX_sicd_has_simd -static inline auto if_add(const simd_floatv_mask& m, const simd_floatv& v, typename simd_floatv::value_type c) -{ - return if_add_(m, v, c); + return lround_(v); } #endif template -static auto lookup_(const IntV& zindex, const std::array& phase_directions) +static auto lookup_(const IntV& indexv, const std::array& phase_directions) { // It seems that the "generator" constuctor is called with SIMD instructions. // https://en.cppreference.com/w/cpp/experimental/simd/simd/simd // > The calls to `generator` are unsequenced with respect to each other. const auto generate_real = [&](size_t i) { - const auto i_ = zindex[i]; + const auto i_ = indexv[i]; return phase_directions[i_].real(); }; const auto generate_imag = [&](size_t i) { - const auto i_ = zindex[i]; + const auto i_ = indexv[i]; return phase_directions[i_].imag(); }; ZFloatV retval; @@ -376,44 +433,45 @@ static auto lookup_(const IntV& zindex, const std::array& phase_direc } #if SIX_sicd_has_ximd template -static inline auto lookup(const ximd_intv& zindex, const std::array& phase_directions) +static inline auto lookup(const ximd_intv& indexv, const std::array& phase_directions) { - return lookup_(zindex, phase_directions); + return lookup_(indexv, phase_directions); } #endif #if SIX_sicd_has_simd template -static inline auto lookup(const simd_intv& zindex, const std::array& phase_directions) +static inline auto lookup(const simd_intv& indexv, const std::array& phase_directions) { - return lookup_(zindex, phase_directions); + return lookup_(indexv, phase_directions); } #endif template -static auto lookup_(const IntV& zindex, std::span magnitudes) +static auto lookup_(const IntV& indexv, std::span floats) { const auto lookup_f = [&](size_t i) { - const auto i_ = zindex[i]; + auto i_ = indexv[i]; // The index may be out of range. This is expected because `i` might be "don't care." - if ((i_ >= 0) && (i_ < std::ssize(magnitudes))) + if (i_ != -1) { - return magnitudes[i_]; + i_ = static_cast(i_); + return floats[i_]; } return NAN; // propogate "don't care" }; return generate(lookup_f, FloatV{}); } #if SIX_sicd_has_ximd -static inline auto lookup(const ximd_intv& zindex, std::span magnitudes) +static inline auto lookup(const ximd_intv& indexv, std::span magnitudes) { - return lookup_(zindex, magnitudes); + return lookup_(indexv, magnitudes); } #endif #if SIX_sicd_has_simd -static inline auto lookup(const simd_intv& zindex, std::span magnitudes) +static inline auto lookup(const simd_intv& indexv, std::span magnitudes) { - return lookup_(zindex, magnitudes); + return lookup_(indexv, magnitudes); } #endif #endif // SIX_sicd_has_ximd || SIX_sicd_has_simd @@ -434,8 +492,10 @@ static auto getPhase(const ZFloatV& v, float phase_delta) // Phase is determined via arithmetic because it's equally spaced. // There's an intentional conversion to zero when we cast 256 -> uint8. That wrap around // handles cases that are close to 2PI. + constexpr auto two_pi = std::numbers::pi_v *2.0f; + auto phase = arg(v); - phase = if_add(phase < 0.0f, phase, std::numbers::pi_v * 2.0f); // Wrap from [0, 2PI] + phase += simd_select(phase < 0.0f, two_pi, 0.0f); // Wrap from [0, 2PI] return roundi(phase / phase_delta); } @@ -443,24 +503,32 @@ static auto getPhase(const ZFloatV& v, float phase_delta) template inline auto lower_bound(std::span magnitudes, const FloatV& v) { - IntV first = 0; - const IntV last = gsl::narrow(magnitudes.size()); + assert(magnitudes.size() == six::AmplitudeTableSize); + + IntV first = gsl::narrow(std::distance(magnitudes.begin(), magnitudes.begin())); + const IntV last = gsl::narrow(std::distance(magnitudes.begin(), magnitudes.end())); auto count = last - first; while (any_of(count > 0)) { - auto it = first; const auto step = count / 2; - it += step; - - auto next = it; ++next; // ... ++it; - auto advance = count; advance -= step + 1; // ... -= step + 1; - + auto it = simd_select(count > 0, first + step, first); // std::advance(it, step); + + //if (magnitudes[it] < value) + //{ + // first = ++it; + // count -= step + 1; + //} + //else + // count = step; const auto c = lookup(it, magnitudes); // magnituides[it] - const auto test = c < v; - it = select(test, next, it); // ... ++it - first = select(test, it, first); // first = ... - count = select(test, advance, step); // `count -= step + 1` —— `count = step` + + const auto count_GT_zero = count > 0; + const decltype(count_GT_zero) c_LT_v = c < v; // masks need to be of the same type for && + const auto test = count_GT_zero && c_LT_v; + first = simd_select(test, ++it, first); // first = ++it; + auto count_ = count; count_ -= step + 1; // count -= step + 1; + count = simd_select(test, count_, step); // `count -= step + 1` —— `count = step` } return first; } @@ -473,22 +541,45 @@ inline auto lower_bound(std::span magnitudes, const FloatV& v) template static auto nearest(std::span magnitudes, const FloatV& value) { - // see `::nearest()` in **NearestNeighbors.cpp** assert(magnitudes.size() == six::AmplitudeTableSize); + /* + const auto begin = magnitudes.begin(); + const auto end = magnitudes.end(); + + const auto it = std::lower_bound(begin, end, value); + if (it == begin) return 0; + + const auto prev_it = std::prev(it); + const auto nearest_it = it == end ? prev_it : + (value - *prev_it <= *it - value ? prev_it : it); + const auto distance = std::distance(begin, nearest_it); + assert(distance <= std::numeric_limits::max()); + return gsl::narrow(distance); + */ const auto it = ::lower_bound(magnitudes, value); - const auto prev_it = it - 1; // const auto prev_it = std::prev(it); - const auto v0 = value - lookup(prev_it, magnitudes); // value - *prev_it - const auto v1 = lookup(it, magnitudes) - value; // *it - value + const IntV begin = 0; + const auto& zero = begin; + if (all_of(it == begin)) + { + return zero; + } const IntV end = gsl::narrow(magnitudes.size()); - const IntV zero = 0; - auto retval = select(it == 0, zero, // if (it == begin) return 0; - select(it == end, prev_it, // it == end ? prev_it : ... - select(v0 <= v1, prev_it, it) // (value - *prev_it <= *it - value ? prev_it : it); + if (all_of(it == end)) + { + return it - 1; // i.e., prev_it + } + const auto prev_it = simd_select(it == begin, zero, it - 1); // const auto prev_it = std::prev(it); + + const auto v0 = value - lookup(prev_it, magnitudes); // value - *prev_it + const auto v1 = lookup(it, magnitudes) - value; // *it - value + auto nearest_it = simd_select(it == begin, zero, // if (it == begin) return 0; + simd_select(it == end, prev_it, // it == end ? prev_it : ... + simd_select(v0 <= v1, prev_it, it) // (value - *prev_it <= *it - value ? prev_it : it); )); - return retval; + return nearest_it; } template @@ -498,20 +589,21 @@ static auto find_nearest(std::span magnitudes, { // We have to do a 1D nearest neighbor search for magnitude. // But it's not the magnitude of the input complex value - it's the projection of - // the complex value onto the ray of candidate magnitudes at the selected phase. + // the complex value onto the ray of candidate magnitudes at the simd_selected phase. // i.e. dot product. const auto projection = (phase_direction_real * real(v)) + (phase_direction_imag * imag(v)); //assert(std::abs(projection - std::abs(v)) < 1e-5); // TODO ??? return nearest(magnitudes, projection); } -#if SIX_sicd_has_VCL +#if SIX_sicd_has_VCL || SIX_sicd_has_sisd +template static auto lookup_and_find_nearest(const six::sicd::details::ComplexToAMP8IPHS8I& converter, - const vcl_intv& phase, const vcl_zfloatv& v) + const IntV& phase, const FloatV& v) { const auto phase_direction_real = lookup(phase, converter.get_phase_directions().real); const auto phase_direction_imag = lookup(phase, converter.get_phase_directions().imag); - return ::find_nearest(converter.magnitudes(), phase_direction_real, phase_direction_imag, v); + return ::find_nearest(converter.magnitudes(), phase_direction_real, phase_direction_imag, v); } #endif #if SIX_sicd_has_ximd || SIX_sicd_has_simd @@ -630,6 +722,21 @@ auto cend(coda_oss::mdspan md) // ``` // By returning our own class from `func()`, we can take control of the assignment operator. // (Unlike most other operators, `operator=()` *must* be a member-function.) +template +inline void assign(size_t i, AMP8I_PHS8I& lhs, const AMP8I_PHS8I_unseq& rhs) +{ + const auto i_ = gsl::narrow(i); + lhs.amplitude = gsl::narrow(rhs.amplitude[i_]); + lhs.phase = gsl::narrow(rhs.phase[i_]); +} +#if SIX_sicd_has_sisd +inline void assign(size_t, AMP8I_PHS8I& lhs, const AMP8I_PHS8I_unseq& rhs) +{ + lhs.amplitude = gsl::narrow(rhs.amplitude); + lhs.phase = gsl::narrow(rhs.phase); +} +#endif + struct mdspan_iterator_value final { std::span p_; @@ -640,9 +747,7 @@ struct mdspan_iterator_value final //assert(p_.size() <= size(other.amplitude)); for (size_t i = 0; i < p_.size(); i++) { - const auto i_ = gsl::narrow(i); - p_[i].amplitude = gsl::narrow(other.amplitude[i_]); - p_[i].phase = gsl::narrow(other.phase[i_]); + assign(i, p_[i], other); } return *this; } @@ -731,6 +836,9 @@ static const std::string unseq_vcl = "vcl"; #if SIX_sicd_has_ximd static const std::string unseq_ximd = "ximd"; #endif +#if SIX_sicd_has_sisd +static const std::string unseq_sisd = "sisd"; +#endif static std::string nearest_neighbors_unseq_ = #if SIX_sicd_has_simd unseq_simd; @@ -738,6 +846,8 @@ unseq_simd; unseq_vcl; #elif SIX_sicd_has_ximd unseq_ximd; +#elif SIX_sicd_has_sisd +unseq_sisd; #else #error "Don't know how to implement six_sicd_set_nearest_neighbors_unseq()" #endif @@ -753,8 +863,20 @@ std::string SIX_SICD_API six_sicd_set_nearest_neighbors_unseq(std::string unseq) void six::sicd::NearestNeighbors::nearest_neighbors_(execution_policy policy, std::span inputs, std::span results) const { - // TODO: there could be more complicated logic here to determine which UNSEQ - // implementation to use. + #if CODA_OSS_DEBUG && SIX_sicd_has_sisd + // If we're in UNSEQ code with a sequential execution policy, then use + // the generic UNSEQ code ... but with non-SIMD types. + // + // Note that `nearest_neighbors_T()` is expecting an UNSEQ policy. + if (policy == execution_policy::seq) + { + return nearest_neighbors_T(execution_policy::unseq, inputs, results); + } + if (policy == execution_policy::par) + { + return nearest_neighbors_T(execution_policy::par_unseq, inputs, results); + } + #endif // CODA_OSS_DEBUG && SIX_sicd_has_sisd // This is very simple as it's only used for unit-testing const auto& unseq = ::nearest_neighbors_unseq_; @@ -795,6 +917,13 @@ void six::sicd::NearestNeighbors::nearest_neighbors_(execution_policy policy, } #endif // SIX_sicd_has_ximd + #if SIX_sicd_has_sisd + if (unseq == unseq_sisd) + { + return nearest_neighbors_T(policy, inputs, results); + } + #endif + throw std::logic_error("Don't know how to implement nearest_neighbors_() for unseq=" + unseq); } void six::sicd::NearestNeighbors::nearest_neighbors_unseq(std::span inputs, std::span results) const @@ -805,4 +934,12 @@ void six::sicd::NearestNeighbors::nearest_neighbors_par_unseq(std::span inputs, std::span results) const +{ + nearest_neighbors_(policy, inputs, results); +} +#endif // SIX_sicd_has_sisd + #endif // SIX_sicd_ComplexToAMP8IPHS8I_unseq diff --git a/six/modules/c++/six.sicd/source/unseq_Ximd.h b/six/modules/c++/six.sicd/source/unseq_Ximd.h index 2e06a65de..267e11b83 100644 --- a/six/modules/c++/six.sicd/source/unseq_Ximd.h +++ b/six/modules/c++/six.sicd/source/unseq_Ximd.h @@ -53,7 +53,7 @@ namespace six namespace ximd { -// Need a class for the "broadcast" constructor (not impelemented). +// Need a class for the "broadcast" constructor. // Also helps to avoid overloading `std::array`. template struct Ximd final @@ -69,7 +69,7 @@ struct Ximd final template Ximd(U v) noexcept { - *this = generate([&](size_t) { return v; }); + *this = Ximd([&](size_t) { return v; }, nullptr); } template Ximd(const Ximd& other) noexcept @@ -85,20 +85,13 @@ struct Ximd final // https://en.cppreference.com/w/cpp/experimental/simd/simd/simd // this is the same as `U&& v` above; avoid enable_if gunk for now. template - static auto generate(G&& generator) noexcept + explicit Ximd(G&& gen, std::nullptr_t) noexcept { - Ximd retval; // This is where all the "magic" (would) happen. for (size_t i = 0; i < size(); i++) { - retval[i] = generator(i); + (*this)[i] = gen(i); } - return retval; - } - template - explicit Ximd(G&& generator, std::nullptr_t) noexcept - { - *this = generate(generator); } reference operator[](size_t pos) noexcept @@ -118,7 +111,7 @@ struct Ximd final template void copy_from(const U* mem) { - *this = Ximd::generate([&](size_t i) { return mem[i]; }); + *this = Ximd([&](size_t i) { return mem[i]; }, nullptr); } template void copy_to(U* mem) const @@ -131,7 +124,7 @@ struct Ximd final Ximd& operator++() noexcept { - *this = Ximd::generate([&](size_t i) { return ++value[i]; }); + *this = Ximd([&](size_t i) { return ++value[i]; }, nullptr); return *this; } Ximd operator++(int) noexcept @@ -156,7 +149,7 @@ using ximd_mask = Ximd; template inline auto operator+(const Ximd& lhs, const Ximd& rhs) noexcept { - return Ximd::generate([&](size_t i) { return lhs[i] + rhs[i]; }); + return Ximd([&](size_t i) { return lhs[i] + rhs[i]; }, nullptr); } template inline auto operator+(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -166,7 +159,7 @@ inline auto operator+(const Ximd& lhs, typename Ximd::value_type rhs) noex template inline auto operator-(const Ximd& lhs, const Ximd& rhs) noexcept { - return Ximd::generate([&](size_t i) { return lhs[i] - rhs[i]; }); + return Ximd([&](size_t i) { return lhs[i] - rhs[i]; }, nullptr); } template inline auto operator-(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -176,7 +169,7 @@ inline auto operator-(const Ximd& lhs, typename Ximd::value_type rhs) noex template inline auto operator*(const Ximd& lhs, const Ximd& rhs) noexcept { - return Ximd::generate([&](size_t i) { return lhs[i] * rhs[i]; }); + return Ximd([&](size_t i) { return lhs[i] * rhs[i]; }, nullptr); } template inline auto operator/(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -186,7 +179,7 @@ inline auto operator/(const Ximd& lhs, typename Ximd::value_type rhs) noex template inline auto operator/(const Ximd& lhs, const Ximd& rhs) noexcept { - return Ximd::generate([&](size_t i) { return lhs[i] / rhs[i]; }); + return Ximd([&](size_t i) { return lhs[i] / rhs[i]; }, nullptr); } template @@ -205,7 +198,7 @@ inline auto& operator-=(Ximd& lhs, const Ximd& rhs) noexcept template inline auto operator==(const Ximd& lhs, const Ximd& rhs) noexcept { - return ximd_mask::generate([&](size_t i) { return lhs[i] == rhs[i]; }); + return ximd_mask([&](size_t i) { return lhs[i] == rhs[i]; }, nullptr); } template inline auto operator==(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -215,7 +208,7 @@ inline auto operator==(const Ximd& lhs, typename Ximd::value_type rhs) noe template inline auto operator!=(const Ximd& lhs, const Ximd& rhs) noexcept { - return ximd_mask::generate([&](size_t i) { return lhs[i] != rhs[i]; }); + return ximd_mask([&](size_t i) { return lhs[i] != rhs[i]; }, nullptr); } template inline auto operator!=(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -225,7 +218,7 @@ inline auto operator!=(const Ximd& lhs, typename Ximd::value_type rhs) noe template inline auto operator<(const Ximd& lhs, const Ximd& rhs) noexcept { - return ximd_mask::generate([&](size_t i) { return lhs[i] < rhs[i]; }); + return ximd_mask([&](size_t i) { return lhs[i] < rhs[i]; }, nullptr); } template inline auto operator<(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -235,12 +228,12 @@ inline auto operator<(const Ximd& lhs, typename Ximd::value_type rhs) noex template inline auto operator<=(const Ximd& lhs, const Ximd& rhs) noexcept { - return ximd_mask::generate([&](size_t i) { return lhs[i] <= rhs[i]; }); + return ximd_mask([&](size_t i) { return lhs[i] <= rhs[i]; }, nullptr); } template inline auto operator>(const Ximd& lhs, const Ximd& rhs) noexcept { - return ximd_mask::generate([&](size_t i) { return lhs[i] > rhs[i]; }); + return ximd_mask([&](size_t i) { return lhs[i] > rhs[i]; }, nullptr); } template inline auto operator>(const Ximd& lhs, typename Ximd::value_type rhs) noexcept @@ -260,15 +253,48 @@ inline bool any_of(const ximd_mask& m) return false; } +inline bool all_of(const ximd_mask& m) +{ + for (size_t i = 0; i < m.size(); i++) + { + if (!m[i]) + { + return false; + } + } + return true; +} + template inline auto atan2(const Ximd& real, const Ximd& imag) { - return Ximd::generate([&](size_t i) { return std::atan2(real[i], imag[i]); }); + return Ximd([&](size_t i) { return std::atan2(real[i], imag[i]); }, nullptr); } template inline auto round(const Ximd& v) { - return Ximd::generate([&](size_t i) { return std::round(v[i]); }); + return Ximd([&](size_t i) { return std::round(v[i]); }, nullptr); +} +template +inline auto lround(const Ximd& v) +{ + return Ximd([&](size_t i) { return std::lround(v[i]); }, nullptr); +} + +template +static auto simd_select(const ximd_mask& test, const Ximd& t, const Ximd& f) +{ + Ximd retval; + for (size_t i = 0; i < test.size(); i++) + { + retval[i] = test[i] ? t[i] : f[i]; + } + return retval; +} + +inline auto operator&&(const ximd_mask& lhs, const ximd_mask& rhs) noexcept +{ + return ximd_mask([&](size_t i) { return lhs[i] && rhs[i]; }, nullptr); } } // ximd diff --git a/six/modules/c++/six.sicd/unittests/test_AMP8I_PHS8I.cpp b/six/modules/c++/six.sicd/unittests/test_AMP8I_PHS8I.cpp index 31357b77d..79a61eea1 100644 --- a/six/modules/c++/six.sicd/unittests/test_AMP8I_PHS8I.cpp +++ b/six/modules/c++/six.sicd/unittests/test_AMP8I_PHS8I.cpp @@ -664,6 +664,12 @@ TEST_CASE(test_nearest_neighbor) test_nearest_neighbor_(testName, six::execution_policy::par_unseq); #endif + #if SIX_sicd_has_sisd + six_sicd_set_nearest_neighbors_unseq("sisd"); + test_nearest_neighbor_(testName, six::execution_policy::unseq); + test_nearest_neighbor_(testName, six::execution_policy::par_unseq); + #endif + six_sicd_set_nearest_neighbors_unseq(default_unseq); // restore default #endif // SIX_sicd_ComplexToAMP8IPHS8I_unseq } diff --git a/six/modules/c++/six/include/six/AmplitudeTable.h b/six/modules/c++/six/include/six/AmplitudeTable.h index 3c67d6817..516d803e6 100644 --- a/six/modules/c++/six/include/six/AmplitudeTable.h +++ b/six/modules/c++/six/include/six/AmplitudeTable.h @@ -208,6 +208,14 @@ struct SIX_SIX_API AMP8I_PHS8I_t final // It's primarily for development and testing: VCL needs C++17 and // std::experimental::simd is G++11/C++20. #define SIX_sicd_has_ximd CODA_OSS_DEBUG + //#define SIX_sicd_has_ximd 0 +#endif + +#ifndef SIX_sicd_has_sisd + // This is just normal `int`s and `float`s (not even `std::array`s) made + // to look like SIMD types. Why? Generic code: the same templatized + // code works everywhere. + #define SIX_sicd_has_sisd 1 #endif #ifndef SIX_sicd_ComplexToAMP8IPHS8I_unseq @@ -218,6 +226,15 @@ struct SIX_SIX_API AMP8I_PHS8I_t final #endif // SIX_sicd_have_VCL || SIX_sicd_has_simd #endif // SIX_sicd_ComplexToAMP8IPHS8I_unseq +// Don't know yet whether SISD code actually make sense ... ease +// development/testing and the eventual transition. +#if !SIX_sicd_ComplexToAMP8IPHS8I_unseq + #if SIX_sicd_has_sisd && CODA_OSS_DEBUG + #undef SIX_sicd_ComplexToAMP8IPHS8I_unseq + #define SIX_sicd_ComplexToAMP8IPHS8I_unseq 1 + #endif +#endif + // We're still at C++14, so we don't have the types in // https://en.cppreference.com/w/cpp/algorithm/execution_policy_tag // For now, our use is very limited; so don't try to @@ -262,7 +279,7 @@ class ComplexToAMP8IPHS8I final struct phase_directions final { std::array value; // interleaved, std::complex - #ifdef SIX_sicd_has_VCL + #if SIX_sicd_has_VCL || SIX_sicd_has_sisd std::array real; std::array imag; #endif