From 18a764e5ebc65f098eb9fbb11df113c339655d7d Mon Sep 17 00:00:00 2001 From: "J. Daniel Smith" Date: Wed, 27 Sep 2023 16:48:32 -0400 Subject: [PATCH] Provide io::OutputStream overloads to make it easier to write data in pieces (#700) * use "range for" rather than iterators * utility routines to get the various low-level "write" functions * writeSupportData() utility routine * writePVPData() static utility * making a new DataWriter instance is cheap * use a DataWriter directly * simplify writePVPData() utility * pass a DataWriter to writeSupportData() * writeMetadata() takes a io::OutputStream * make a new DataWriter for writeCPHDData() * pass a io::OutputStreeam to writeSupportDataImpl() * pass a io::OutputStream to more places * separate routine to write the channel data * Squashed commit of the following: commit 15e3d7437348268e0b39dffae35b87d292b82f0d Merge: a018099ff 1ee90a9a7 Author: Dan Smith Date: Wed Sep 27 12:04:49 2023 -0400 Merge commit '1ee90a9a723449a28da2ccb6ff615163c6283a13' into develop/sync_externals commit 1ee90a9a723449a28da2ccb6ff615163c6283a13 Author: Dan Smith Date: Wed Sep 27 12:04:49 2023 -0400 Squashed 'externals/coda-oss/' changes from c92a55d7c..369737085 369737085 use std::ssize() to reduce casts (#738) 8ae7dabac std::size() and std::ssize() (#737) 8db480be5 enable ASAN for GitHub builds (#736) git-subtree-dir: externals/coda-oss git-subtree-split: 369737085573989ba74a68a4dc2da9d4be8be3a6 commit a018099fff07d81c8b0f50a24ec970d03c1711fc Author: Dan Smith Date: Wed Sep 27 12:04:45 2023 -0400 latest from CODA-OSS commit 669feb79e9ad901ec24c16a24864030197079bb4 Merge: 3738c9123 d95832afc Author: Dan Smith Date: Wed Sep 27 12:02:14 2023 -0400 Merge branch 'main' into develop/sync_externals commit 3738c91239438e9847fcdaa16b301c02111e1d0a Merge: a537665a3 804168483 Author: Dan Smith Date: Tue Sep 12 14:47:51 2023 -0400 Merge commit '804168483daa854eafff59910e56704e1282ed40' into develop/sync_externals commit 804168483daa854eafff59910e56704e1282ed40 Author: Dan Smith Date: Tue Sep 12 14:47:47 2023 -0400 Squashed 'externals/nitro/' changes from 49f6338d2..e418beceb e418beceb Merge commit '6144b2cfb436a5696bab62c81651b47edf07aa8c' 6144b2cfb Squashed 'externals/coda-oss/' changes from 8dee6f7a68..c92a55d7c7 32d9c4fe4 latest from CODA-OSS git-subtree-dir: externals/nitro git-subtree-split: e418beceb503c1186119ea058211f1e5b493afc6 commit f12914abbb5d7014dd904fd6319a3edd5ba788cd Author: Dan Smith Date: Tue Sep 12 14:47:44 2023 -0400 Squashed 'externals/coda-oss/' changes from 333b91024..c92a55d7c c92a55d7c build in SIX git-subtree-dir: externals/coda-oss git-subtree-split: c92a55d7c788ec177b094284572b227a40a8fcd7 commit a537665a353bdff78f00a83b4e6801b93bf7ab6a Merge: 3c1377805 f12914abb Author: Dan Smith Date: Tue Sep 12 14:47:44 2023 -0400 Merge commit 'f12914abbb5d7014dd904fd6319a3edd5ba788cd' into develop/sync_externals commit 3c137780502c3539ee778485df3be4f1cea83912 Author: Dan Smith Date: Tue Sep 12 14:28:54 2023 -0400 build after CODA-OSS updates commit 8abc353e0d58bce5621f5a854f5167679f1239fb Merge: a9e87fdfa a5674e744 Author: Dan Smith Date: Tue Sep 12 14:18:14 2023 -0400 Merge commit 'a5674e74433a563a11e4bbdaae3803673dff99f1' into develop/sync_externals commit a5674e74433a563a11e4bbdaae3803673dff99f1 Author: Dan Smith Date: Tue Sep 12 14:18:10 2023 -0400 Squashed 'externals/nitro/' changes from 72cd86cbf..49f6338d2 49f6338d2 latest from CODA-OSS (#581) git-subtree-dir: externals/nitro git-subtree-split: 49f6338d2998fc6b4d033bf1456f1fb1bcb53567 commit 5ab880228dfd4d6fd1b2b11630fd414b394d292b Author: Dan Smith Date: Tue Sep 12 14:18:06 2023 -0400 Squashed 'externals/coda-oss/' changes from 00e2dd928..333b91024 333b91024 Squashed commit of the following: ac8c312db Only allow va_args formatting under controlled circumstances (#735) 589aacfd7 str::to_native() for when conversion to std::string really is needed 743feb41e try hard not to lose string encoding (#734) 1854a9b46 hide use of str::cast() inside `details` namespace (#733) fbe0c85f9 compile-time getSIMDInstructionSet() (#732) ee2c46358 more consistent naming for routines to convert between encodings (#731) git-subtree-dir: externals/coda-oss git-subtree-split: 333b9102479d3d2c7d41425dbcfa41a7eabf7b1d commit a9e87fdfaabf5951ba3efd78a2d5b468df7f71cc Merge: 3cbdfb940 5ab880228 Author: Dan Smith Date: Tue Sep 12 14:18:06 2023 -0400 Merge commit '5ab880228dfd4d6fd1b2b11630fd414b394d292b' into develop/sync_externals commit 3cbdfb940a9494ad52a0611db60d30eb78932703 Author: Dan Smith Date: Tue Sep 12 14:18:02 2023 -0400 latest from NITRO commit a681ebd1daf4d7268d181e0295b95558e7d4ff95 Author: Dan Smith Date: Tue Sep 12 14:08:14 2023 -0400 latest from CODA-OSS commit 7a363b43f351cd7780ae628a1d6ddbd84300129c Merge: 8cd5e77e4 80aec3ad3 Author: Dan Smith Date: Tue Sep 12 14:07:29 2023 -0400 Merge branch 'main' into develop/sync_externals commit 8cd5e77e408f83589d60c31968f812ba7b8c9ec6 Merge: 4ee925cae 369de34a1 Author: Dan Smith Date: Tue Aug 29 13:25:44 2023 -0400 Merge commit '369de34a1c027a68c3f7d35be60c091e828d6417' into develop/sync_externals commit 369de34a1c027a68c3f7d35be60c091e828d6417 Author: Dan Smith Date: Tue Aug 29 13:25:41 2023 -0400 Squashed 'externals/nitro/' changes from e488297c3..72cd86cbf 72cd86cbf latest from CODA-OSS (#580) e4920a743 FmtX() -> str::Format() git-subtree-dir: externals/nitro git-subtree-split: 72cd86cbf6abb1122350388c98ea448484d76d91 commit bb4f3dc0f7aeb187acf39d6cae5e96b711bb4dc8 Author: Dan Smith Date: Tue Aug 29 13:25:37 2023 -0400 Squashed 'externals/coda-oss/' changes from 8dee6f7a6..00e2dd928 00e2dd928 another FmtX() overload 165a799c5 more FmtX() overloads (#729) git-subtree-dir: externals/coda-oss git-subtree-split: 00e2dd9286b44e32f8393165f20485b543d52170 commit 4ee925caecf49d4aa1d5881fdb4ef2ea95fa13c3 Merge: baeaa33d1 bb4f3dc0f Author: Dan Smith Date: Tue Aug 29 13:25:37 2023 -0400 Merge commit 'bb4f3dc0f7aeb187acf39d6cae5e96b711bb4dc8' into develop/sync_externals commit baeaa33d1156f31f5c333941d4533e893d885573 Author: Dan Smith Date: Tue Aug 29 13:25:33 2023 -0400 latest from CODA-OSS and NITRO commit a4a755c8c6644480c1765080e67cd30d554a9a3e Merge: 6fbe8b8dd 8a9b1e9e5 Author: Dan Smith Date: Tue Aug 29 13:24:48 2023 -0400 Merge branch 'main' into develop/sync_externals commit a5d077e25b3d48901ed9615ccec88e4aaebbfce3 Author: Dan Smith Date: Tue Aug 29 09:20:49 2023 -0400 Squashed 'externals/nitro/' changes from 72209535d..e488297c3 e488297c3 Merge commit '81ea6f1789863f26f06dc447f7784383dc7cb595' 81ea6f178 Squashed 'externals/coda-oss/' changes from e87c32b4de..8dee6f7a68 git-subtree-dir: externals/nitro git-subtree-split: e488297c31fbf52dd2c1ae6ba94f95a6cdf9eb30 commit 6fbe8b8dd9ca1378a87de28f8161826d1ce0bf18 Merge: 3441e8047 a5d077e25 Author: Dan Smith Date: Tue Aug 29 09:20:49 2023 -0400 Merge commit 'a5d077e25b3d48901ed9615ccec88e4aaebbfce3' into develop/sync_externals commit 3441e8047b47ac79118e841600cc2aac7c00b1db Merge: 69b0fb4e3 8538e1d22 Author: Dan Smith Date: Tue Aug 29 09:15:44 2023 -0400 Merge commit '8538e1d222d56604050582bd2b0f128dda51f925' into develop/sync_externals commit 8538e1d222d56604050582bd2b0f128dda51f925 Author: Dan Smith Date: Tue Aug 29 09:15:34 2023 -0400 Squashed 'externals/nitro/' changes from fc1f59799..72209535d 72209535d toString() changes from CODA-OSS (#579) git-subtree-dir: externals/nitro git-subtree-split: 72209535d071d6a6f18c475413f948d086cd8787 commit d982f60807e49f64d019eb973639ba555e9d7942 Author: Dan Smith Date: Tue Aug 29 09:15:26 2023 -0400 Squashed 'externals/coda-oss/' changes from bf7ae4a71..8dee6f7a6 8dee6f7a6 revert toString() changes (#728) git-subtree-dir: externals/coda-oss git-subtree-split: 8dee6f7a68d560f8bb3469f2d0375bb65e2f532f commit 69b0fb4e37fc959ef878394712dbfe391a69f6bb Merge: c4dcdbac4 d982f6080 Author: Dan Smith Date: Tue Aug 29 09:15:26 2023 -0400 Merge commit 'd982f60807e49f64d019eb973639ba555e9d7942' into develop/sync_externals commit c4dcdbac49f64f0c7135ef1d5f59cbe771474d2f Author: Dan Smith Date: Tue Aug 29 08:43:57 2023 -0400 toString() changes from CODA-OSS commit 7de4dcfa0704b847fa46dca9457a0bd7eab756e9 Merge: 48895c9ec 69731d3c8 Author: Dan Smith Date: Tue Aug 29 08:37:24 2023 -0400 Merge branch 'main' into develop/sync_externals commit 48895c9ec4e9918e131c7f12e087440d675a6e0b Merge: cb1a2c945 a2de0b527 Author: Dan Smith Date: Mon Aug 28 16:20:55 2023 -0400 Merge commit 'a2de0b5279a6763f39f6f34536eb6f680e98d146' into develop/sync_externals commit a2de0b5279a6763f39f6f34536eb6f680e98d146 Author: Dan Smith Date: Mon Aug 28 16:20:50 2023 -0400 Squashed 'externals/nitro/' changes from e7ea728ec..fc1f59799 fc1f59799 latest from CODA-OSS (#578) 02e8f80c6 latest from CODA-OSS (#577) git-subtree-dir: externals/nitro git-subtree-split: fc1f59799a3082a3c1605dd9a6c3dd12ce2412d3 commit 6c23d5de45eadac47028f968db108e4161666aa1 Author: Dan Smith Date: Mon Aug 28 16:20:46 2023 -0400 Squashed 'externals/coda-oss/' changes from 22e050207..bf7ae4a71 bf7ae4a71 use overloads rather than varargs for std::format() (#727) git-subtree-dir: externals/coda-oss git-subtree-split: bf7ae4a718bd97a55ed37d2dd0a514deebc97776 commit cb1a2c945855d9b07e9692d5410883af654f8861 Merge: bbc8367d0 6c23d5de4 Author: Dan Smith Date: Mon Aug 28 16:20:46 2023 -0400 Merge commit '6c23d5de45eadac47028f968db108e4161666aa1' into develop/sync_externals commit bbc8367d058b11a37666ad581582dae2169c438d Author: Dan Smith Date: Mon Aug 28 15:35:19 2023 -0400 latest from CODA-OSS commit 02e1917799c7201c477e57a1852d68cb49696f97 Author: Dan Smith Date: Mon Aug 28 14:09:36 2023 -0400 str::format changes from CODA-OSS commit e4b5c45c5e38de90b6b4b234225cbb92a152195a Merge: 0584296f1 80128f8d6 Author: Dan Smith Date: Mon Aug 28 14:08:34 2023 -0400 Merge branch 'main' into develop/sync_externals commit 2928617bd4525f20316de415f07b6cf3d88e4878 Author: Dan Smith Date: Fri Aug 25 13:31:39 2023 -0400 Squashed 'externals/nitro/' changes from 547d0aa9f..e7ea728ec e7ea728ec reduce differences between cpp17 and main branches (#576) git-subtree-dir: externals/nitro git-subtree-split: e7ea728ec6c80eb3a7564f9c721563c49d744c51 commit 0584296f1e8e9126bd99f1eeba8b3e2cf6280998 Merge: 8a996f260 2928617bd Author: Dan Smith Date: Fri Aug 25 13:31:39 2023 -0400 Merge commit '2928617bd4525f20316de415f07b6cf3d88e4878' into develop/sync_externals commit 9c6572a9e7ccc4d54e79a6f7ebc373b4414e7df9 Author: Dan Smith Date: Fri Aug 25 13:31:36 2023 -0400 Squashed 'externals/coda-oss/' changes from 168cbae01..22e050207 22e050207 utility routines to reduce duplicated code (#726) a5ea60647 remove str::EncodedStringView (#725) git-subtree-dir: externals/coda-oss git-subtree-split: 22e05020768d791498db105c261439dc78981445 commit 8a996f260f3c3878e5df2efdf56670571cde1032 Merge: e509306b1 9c6572a9e Author: Dan Smith Date: Fri Aug 25 13:31:36 2023 -0400 Merge commit '9c6572a9e7ccc4d54e79a6f7ebc373b4414e7df9' into develop/sync_externals commit e509306b1df529a760e0f0603582eb0d1549d41f Author: Dan Smith Date: Fri Aug 25 13:31:21 2023 -0400 latest from CODA-OSS and NITRO commit 62fc405c4b95f2fcdace714e127c9dc27b16d7f7 Merge: 40827a753 49e28e2ae Author: Dan Smith Date: Fri Aug 25 13:30:22 2023 -0400 Merge branch 'main' into develop/sync_externals commit 40827a753f1d663b39ad3380f0de498bd45e582f Author: Dan Smith Date: Mon Aug 21 09:23:42 2023 -0400 CRLF commit 209b4e374c74229516ca3071258f971c78b70410 Merge: 238a6fc8a 993d755ef Author: Dan Smith Date: Mon Aug 21 09:22:36 2023 -0400 Merge branch 'main' into develop/sync_externals commit 238a6fc8a2e5a5ae4826d1f8edce80e2fa046b0b Merge: 1c1e44094 83ec173c2 Author: Dan Smith Date: Mon Aug 21 09:15:00 2023 -0400 Merge commit '83ec173c23943efe4e24689aad8d744aad69749c' into develop/sync_externals commit 83ec173c23943efe4e24689aad8d744aad69749c Author: Dan Smith Date: Mon Aug 21 09:14:53 2023 -0400 Squashed 'externals/nitro/' changes from c8ecbe9ae..547d0aa9f 547d0aa9f NITRO-2.11.4 (#575) fe309c4ba Merge branch 'main' of github.com:mdaus/nitro 10efa9990 latest from CODA-OSS (#574) d08f1c0a1 CRLF db5d3d484 latest from CODA-OSS 98c755048 CRLF aa1482543 CRLF bfdbe69a4 Latest from CODA-OSS (#573) 9e4ce0b58 latest from CODA-OSS (#572) 227a8a8f4 trust coda-oss for right -std flags 13869687e latest from CODA-OSS (#571) 5724d8c18 latest from CODA-OSS (#570) fb794f0fe latest from CODA-OSS (#569) 7a6132ba0 update files changes in cpp17 branch (#568) de91d4977 Fix bug creating NITFException (#567) 225273436 fix YAML for 'main' 5d1c83d11 single project for unittests (#566) 102a019db latest from CODA-OSS (known broken build) (#565) b4ae2d429 match YAML from coda-oss 176bcaf6d build NITRO.SLN using msbuild (#562) 85e9043b8 latest from CODA-OSS (#561) 49ec50325 use new "PlatformToolkit" special environment variable (#560) 6c06e3711 latest from CODA-OSS (#559) cee9feb42 latest from CODA-OSS (#558) 3f01809fa latest from coda-oss (#557) 471fb3fc1 Update test_j2k_loading++.cpp b12caf2fc latest from CODA-OSS (#556) 0cd432624 use sys::make_span (#555) 05dae18a3 patch to build other projects 8f974e995 NITRO-2.11.3 (#554) 870aa6afd update to coda-oss 2023-06-05 (#553) 2fd7a0bfa latest from coda-oss (#552) 0eecce004 invoke() utility to reduce code duplication (#550) 59fb02fe9 latest from coda-oss (#551) 9fbf2b7b8 Fill out adapter free block which is used for nitf decompression (#549) 089ba0b5b latest from coda-oss 3b52f0025 latest from coda-oss (#547) 90c6263e2 latest from coda-oss (#544) 90d513ac5 latest from coda-oss (#543) git-subtree-dir: externals/nitro git-subtree-split: 547d0aa9ffcf34ad94f9ea71aa415f39837b2e68 commit c57aa2691a9635f55f5c604723d61ee495c59ed2 Author: Dan Smith Date: Mon Aug 21 09:14:48 2023 -0400 Squashed 'externals/coda-oss/' changes from e87c32b4d..168cbae01 168cbae01 reduce differences between 'main' and 'cpp17' (#724) 5b892bf5f release 2023-08-18 (#723) 0b1327d1e restore mem::AutoPtr; too much of a hassle moving it to numpyutils :-( (#722) 7691adb56 normalize line endings (#721) b7d50efdd update to HDF5 1.14.2 (#720) fcc96ec69 Update .gitattributes (#719) bb82a94c9 xerces-c 3.2.4 (#718) ef4ad7cf3 Update to e2fsprogs 1.47.0 (#717) b4ca18a11 fix compiler warning about order of operations a5df5b823 update to HighFive 2.7.1 (#716) 4109ee5d2 `mem::AutoPtr` is only for Python bindings (#715) 00d843f87 remove hdf5.lite (#714) 443dd3825 Reduce compiler warnings, speed up builds (#713) d60861821 get optional working with partial C++17 (#712) d5bd0d804 "warning STL4036: is removed in C++20." c19ece7f9 Don't interfere with a partial C++20 implementation (#711) 065e86ddc operator==() for QName (#710) 1c16380ce remove zint* typedefs; don't want to encourage use of types::Complex (#709) 6dd247991 our optional<> is now closer to C++17 (#708) 87ac61739 strict checking on `std::complex` is too much of a hassle for now 034d52c86 overloads to byte-swap type::Complex are too much trouble (#707) 42d449c33 sys::byteSwap overloads for types::Complex (#705) 4092fd8e7 remove work-around for NITRO bug (#704) e2472acfc build in NITRO and SIX (#703) 32ccf9105 Use same build paths as Visual Studio (#701) 366ac9f43 Another round of removing compiler warnings (#702) a7f8ef260 Fix compile warnings from building CODA (#700) f70b4202f account for coda-oss.vcxproj being in other SLNs af3faebfc Remove more compiler/code-analysis warnings (#699) 82be2a6db unittests should work w/o install (#698) dbb90a06d add msbuild for coda-oss.sln (#697) 5a417140a reduce compiler warnings (#696) 288619dfa all modules now part of coda-oss.vcxproj (#695) c2fc5fc66 add more projects to coda-oss-lite.vcxproj (#694) ceb86c186 support $(PlatformToolset) as a "special" environment variable (#693) d78a8595a OS::getSIMDInstructionSet() utility (#692) 2d2df467d fix `python3 waf dumplib` 54033e70e Merge branch 'main' into feature/complex_short 3c63f9f65 std::numbers from C++20 (#691) 892dd0e00 ComplexInteger and ComplexReal to better match existing naming conventions (#690) b3872181e match coda-oss naming conventions (#688) 704d6867f beef-up our complex type (#687) 47c1c1cd6 check is_absolute() for URLs (#686) c042373e1 be sure our Path::isAbolute() matches std::filesystem::path::absolute() (#684) ad10286bc volatile is about "special" memory, not threading (#685) f4d42005f fix build error in NITRO ff11a5557 keep using std::complex for now (#682) c88b9c053 types::complex (#681) d1244a080 don't need our own make_unique in C++14 (#680) aeec0131c assert()s for mem::CopyablePtr (#679) 72b0ebd60 add types::complex_short (#678) 932130a58 patch to build other projects c00c1f203 coda-oss release 2023-06-06 (#677) ef54bbcd5 remove more compiler warnings (#676) dadfc5ce6 distinguish between byte-swapping a buffer and single value (#674) 90187f6cd more xml.lite tweaks for SIDD-3.0/ISM (#675) eb9960772 stronger type-checking for byteSwap() (#673) ff4f820ed xml.lite tweaks to support SIDD 3.0 ISM (#672) b1de8c0e5 std::byte should be a unique type (#671) c05bf9a02 allow enums to be byte-swapped too 1f9fd88d6 remove spurious 47684c45b byteSwap now uses byte buffers (#670) cbc659db2 add swapBytes() utility from SIX (#669) 891481b64 simplify byte-swapping (#668) 540ae763e more byteSwap() tweaks 0774c03c4 threaded byteSwap() (#667) d156370d3 swapping a single-byte value makes no sense c120e3255 be sure parameter is used to avoid compiler warning e85ec9331 --output-on-failure for CTest (#666) e80376197 turn off "there is no warning number" warning c5f0a5d15 A C-string may not be NULL-terminated (#665) 0c5eb29ae use platform-specific routines for byteSwap() (#664) 0b7d581fa remove transform_async() (#663) f6489b6be Merge branch 'main' into feature/xml.lite_tweaks 836c426a2 use function-pointers so that isConsoleOutput is only checked once 56e3c45b1 move depthPrint() functionality into non-member function in preparation for future changes ddcd26d97 Merge branch 'main' into feature/xml.lite_tweaks 69cc0e506 use the more rigorous create_and_check_datatype() 9efb87558 Merge branch 'main' into feature/hdf5 4d2f2f417 more HighFive unittests (#662) 14191a844 HighFive::create_datatype() goes from C++ to HighFive 98583473f utility routines to read string attributes 1fa75ce81 use the C API to read a string attribute 71e7b69f5 still can't figure out how to read a string attribute :-( e96f37a69 test reading the file attributes a25244519 getAttribute() unittest 8f12a3000 getDataType() unittest 857ff0af3 HighFive utility routines (#661) 1d687db57 writeDataSet() utility overload 106aa6894 sigh ... H5Easy::dump() fill fails on Windows/WAF :-( 2641b60b2 Merge branch 'main' into feature/hdf5 35c19e7e8 change actions to @v3 (#660) 212bbd3a3 works on local machine, but not build server ... ? 7125118b0 dump of 1D vector doesn't work :-( c704db435 sigh ... WAF build still failing :-( 01aae4616 does dump() of a 1D vector work? 76a53c813 comment-out H5 writing :-( 0f0e19aff test_highfive_dump() cb8f73795 trying to get highfive_dump() unittest working w/Windows-WAF 6584a264a does test_highfive_create() work? cde6147ce tweak HighFive wrappers (#659) 316566854 Revert "trying HighFive "write" unittests again" a9ec24ca4 trying HighFive "write" unittests again 585ad49a5 tweak names of utility routines 4c91a4d97 make it easier to read a std::vector and std::vecotr 0217ffa26 readDataSet() now works for 1D data 95e8973f2 trying to get hdf5::lite::load() working 4d294611d "const" correctness 5e6305c3f fix load_complex() 4a134dc5f start work on utility routines to read complex data from HDF5 9d76a7f41 Merge branch 'main' into feature/hdf5 8f9667a24 whitepsace ebd3fc99e Merge branch 'main' into feature/hdf5 40091b069 comment-out writing tests for now ... need to figure our WAF bulid failure 935aa3459 be sure the dataset has real data 114b9bf33 update release notes bd9c0b26c tweak HighFive utility routines acda1ef57 turn off diagnostics around expected failures 22a748840 readDataSet() utility routine for HighFive bd88a8c25 HighFive writeDataSet() utility to work with our SpanRC 6142f5b33 use HighFive routines to write a HDF5 file 5bbf1abaf Use HiveFive routines to get info about the file 84fbc8378 duplicate unittests with H5Easy f1f054c03 Merge branch 'main' into feature/hdf5 9b63ca470 fix directory names f6f826689 fix directory names 7aeb82c33 Merge branch 'main' into feature/hdf5 d028baaeb hook up HighFive header-only library (#653) 3083b0a31 Revert "HighFive 2.6.2" 246985a7f Revert ""build" HighFive HDF5 library" a8b75a586 Revert "turn off HighFive Boost support" ec68d5f83 Revert "Add HighFive unittests" f1f85b9e7 Revert "get test_high_five_base more-or-less compiling" 5ea634ee1 Revert "more work on getting HighFive unittests to build" ecc45433c more work on getting HighFive unittests to build bb194788a get test_high_five_base more-or-less compiling d42bde000 Add HighFive unittests ddc86bb32 turn off HighFive Boost support b255122d4 "build" HighFive HDF5 library 396cc3ef2 HighFive 2.6.2 5e5f9d9c0 Merge branch 'main' into feature/hdf5 ee938b4a5 changes from SIX bb764df90 Merge branch 'main' into feature/xml.lite_tweaks de2a24380 make derived classes 'final' if possible 14e19bcd2 Change xml lite function to virtual (#645) 8f42ac8e9 Merge branch 'main' into feature/xml.lite_tweaks 18ad90645 hdf5Write unittest 3462e1179 createFile() and writeFile() overloads ecee81d53 fix typos 197eecfa6 sketch-out hdf5::lite::writeFile() bd2311795 use SpanRC for writeFile(), not yet implemented ea9af7510 simple SpanRC to hold a 2D-size and pointer 027c19ee8 createFile() unittest 1f9d07ecb hook up createFile() 8c7e4473f start hooking up HDF5 writing 146e0bea3 Merge branch 'main' into feature/hdf5 88ca9fcb7 Merge branch 'main' into feature/hdf5 42b604b46 Squashed commit of the following: 10ee602c2 Merge branch 'main' into feature/hdf5 67aa42b69 restore changes from "main" 8bbfcbfbf unittests can be simplified to match fewer "view" classes 126bb802e Merge branch 'main' into feature/hdf5 3f8ba7a42 again, don't need a class just to convert from std::vector<> to std::span<> 24c2b489c Squashed commit of the following: 2703c119d Squashed commit of the following: 9d5228a2b don't need an entire class just to convert a std::vector<> into std::span<> 51bc931dc Merge branch 'main' into feature/hdf5 a84f25816 Squashed commit of the following: c4d2ed696 add missing #include guards, fix type in existing #include guard d541525a0 use a single ComplexViewConstIterator for all views 86e6a459f CODA_OSS_disable_warning causes GCC errors :-( 5d4b9c2cb only need an custom iterator for ComplexSpansView d9f0fb128 hook up iterators b9329e4db initial pass at a ComplexViewConstIterator 635238873 remove compiler warning about unused "constexpr" variables b39f6096f use the casing from H5 to make copy/pasting code slightly easier 0887b13eb Merge branch 'main' into feature/hdf5 bd07df1ca Consistent casing for Dataset, Datatype, Dataspace 7acd30ee2 tweak hdf5.lite dependencies 38ab914df Jupyter notebook for creating H5 files 95a040e0b _small.h5 is now (correctly) FLOAT32 107e7c487 make a simple values() member function to avoid template magic e1feca919 use TEST_SPECIFIC_EXCEPTION macro instead of try/catch 738333688 readDatasetT() now throws for the wrong buffer type 6b2cc2529 Merge branch 'feature/hdf5' of github.com:mdaus/coda-oss into feature/hdf5 310f8fd3d can't get template magic right for copy_axis() 86b306d59 stepping through copy ctors in the debugger is annoying f243e92d6 trying to make wrong type of buffer fail 2b10d9652 read in new sample file a28e59d8c help the compiler with type deduction 49bf5e9bc nested_complex_float_data_small.h5 e029325fc utility routines to "deconstruct" and array of std::complex dede3bd39 Merge branch 'main' into feature/hdf5 904b1ef5e tweak class names, make_() and copy() utility routines 8237b9efb make it harder to pass the wrong types to ComplexViews 4d9aeda2c ComplexArrayView and ComplexParallelView utility classes f5e367dfa test std::span> a4a2844f2 read in the nested "i" and "r" data 115615265 sample file has subgroups 8e1b7869a Merge branch 'feature/hdf5' of github.com:mdaus/coda-oss into feature/hdf5 9f4232a1d update sample H5 file 8c55db73a walk through HDF5 sub-groups 7775ed9c4 Update 123_barfoo_catdog_cx.h5 677975d7c Matlab code to create sample H5 file a0e7dfe07 Update test_hdf5info.cpp 0b67e1602 pass __FILE__ and __LINE__ from calling site for a more accurate exception message 86a677321 skeleton for more sample data 85f79b099 Merge branch 'main' into feature/hdf5 18088e942 Merge branch 'main' into feature/hdf5 3a1d17692 Merge branch 'main' into feature/hdf5 1755c69d7 Merge branch 'main' into feature/hdf5 9ad015432 No more "11" suffix on exception names c20d96251 Squashed commit of the following: c88cee999 other values to be filled-in bca4a4ecd incorporation NamedObject from HDF5 docs 61fa68f72 groupInfo() 460e7d766 datasetInfo() 14eb9b764 start filling in DatasetInfo afe5f1c3a start to fill in DatasetInfo 77a968c72 start filling in GroupInfo d81bcdfd9 openGroup() to open groups (loc) a0cd29469 comment-out "dataset" unittest for now 86e006024 begin filling in FileInfo 366dda6ab a return_type_of utility is needed to deduce the return type e21928263 explicitly pass return type to template 4937ccd11 template to reduce boilerplate when calling try_catch_H5Exceptions b3b5ebde7 use new exception utility routines ea1c03ef0 put exception handling/conversion in a utility routine 819a99d39 utility routine for exception handling 6f34eea97 put utilities in a separate file for easier reuse fcbde4f24 break utility routines into smaller pieces for easier reuse 52358ea8a WIN32 no longer automatically defined? 5a4286472 Revert "build HDF5 with C89" 680e599e9 build HDF5 with C89 a87a07121 Merge branch 'main' into feature/hdf5 8447c1a90 Revert "sym-links instead of copying files" db3b5e12b Merge branch 'main' into feature/xml.lite_tweaks fb60b5696 Merge branch 'main' into feature/hdf5 5110a5cc8 Comments about _u and _q 1a937d32c Merge branch 'main' into feature/xml.lite_tweaks fa06f04d7 get ready for hdf5.lite enhancdements b040c7c43 sym-links instead of copying files aa431bb47 use _u for xml::lite::Uri 3d0c6d58c fix case-sensitive #include filename 93dcd0e52 operator() for getElementByName() 75a93af85 more operator[] overloads to make attribute management easier 4ab8216f8 user-defined string literals to remove some noise around xml::lite::QName f82f0b0fc Merge branch 'main' into feature/xml.lite_tweaks ae30e3644 Merge branch 'feature/xml.lite_tweaks' of github.com:mdaus/coda-oss into feature/xml.lite_tweaks ffdd9beb0 simplify attribute creation 9bf5414f5 simplify attribute creation 82d7a4e95 SWIG gets confused about namespaces 7a61d0741 fix bug on Element ctor uncovered by unittest fdd7e58c1 QName is also in the xerces namespace which confuses SWIG bindings a325b7053 operator+=() overload for addChild daf30e6c0 Merge branch 'feature/xml.lite_tweaks' of github.com:mdaus/coda-oss into feature/xml.lite_tweaks b887d2b47 provide overloads for Element& rather than creating new "reference" types 1fa6bba38 rename test_xmleasy.cpp 7c8c9e0f1 += overload 850da6f63 overload for std::string 4547fc5a7 use UIT-8 strings for characterData 4723462a3 convenient addChild() overloads e48720753 copy over ElementReference from xml.easy a4ca30a0d Merge branch 'main' into feature/xml.lite_tweaks 6ae9f0b71 Revert "check-in of new xml.easy (to move code between computers)" f7466a6d7 Revert "simple routines for single element" a5490230d Revert "make some operators simplier ways of calling functions" c9a25630a Revert "get document creation working" 8af8710b0 Revert "free functions instead of member functions" 16c3847cb Revert "ElementReference distinct from Element" 7d68e156f Revert "ElementMutableReference" 00eb2a282 Merge branch 'main' into feature/xml.lite_tweaks a42969c1f ElementMutableReference a20ae9355 ElementReference distinct from Element 14eeeea0b free functions instead of member functions 4aae014b3 get document creation working 883569269 make some operators simplier ways of calling functions 053bd1212 simple routines for single element 8bf701a2e check-in of new xml.easy (to move code between computers) 41f959051 unittests for creating XML documents from scratch 9752d50ae Merge branch 'main' into feature/xml.lite_tweaks 1531d5709 by default, don't validate strings passed to Uri() 46d13d4bf Merge branch 'master' into feature/xml.lite_tweaks 39b547d32 remove more vestiges of Expat and LibXML ec8274d52 remove LibXML and Expat as they're no longer used/supported. 20eeefeef Merge branch 'master' into feature/xml.lite_tweaks 95074b9b1 update for newer Intel compiler 7024f71e1 Merge branch 'master' into feature/xml.lite_tweaks 57b1cbc83 Merge branch 'master' into feature/xml.lite_tweaks 4b67561c3 remove validate() overload that nobody is using fa15f1e5d Squashed commit of the following: 1484a9090 test the new validate() API 470da70fb hookup StringStreamT routines 2cddf2504 begin hooking up validate() overloads 1b5d910f3 overload validate() for UTF-8 and Windows-1252 03309b8c9 Squashed commit of the following: b72c6c5bf older compiler doesn't like our make_unique af8f00307 validate UTF-8 XML on Linux 211188613 unit-test for LEGACY XML validation 3c1169d2b Squashed commit of the following: 3afff19ca std::filesystem::path for FileInputStreamOS 908d452f8 WIP: validate all of our sample XML files 00f9bb16b validate against a XML schema 243d8c356 Merge branch 'master' into feature/xml.lite_tweaks 2815d707d fix to work with SWIG bindings. :-( 460862132 trying (again) to remove vestiages of old code e3c83a858 Revert "new code should use UTF-8" 811207c92 new code should use UTF-8 0ffd835f9 Squashed commit of the following: 1e7e03ded Merge branch 'master' into feature/xml.lite_tweaks c1d806aff Merge branch 'master' into feature/xml.lite_tweaks 850d3c811 str::strip() that can be easier to use than str::trim() 580ba9c8c explicitly =delete move 2b39831a8 Squashed commit of the following: 39eebdc23 Merge branch 'master' into feature/xml.lite_tweaks 9adf86cba force calling new UTF-8 write() routines ea61b6204 Merge branch 'master' into feature/xml.lite_tweaks 8a34583fa overload to take schemaPaths as filesystem::path 8671b442f parse XML embedded in a binary file ec4a902f1 updates from xerces.lite 80dc4d963 updates from xerces.lite 549766d6c Attributes::contains() no longer catches an exception 8a645ceac need "sys/" when building in other environments 36af08269 super-simple URI validation 78ef28a3e SWIG bindings are a PITA! :-( e9cba8491 SWIG needs help with Uri 8a8d8dc07 another routines used by pre-build SWIG bindings 818e1ec5d pre-build SWIG bindings use getElementByTagName() member function 067cac5d8 old compiler gets confused on unadorned QName ba92c0ae7 more use of Uri and QName 446c7d17a use QName in new code d6f8b0c83 more direct use of QName 90fff1c73 use xml::lite::QName instead of tuple 646cbb5ed more direct use of QName and Uri ba589ea3b make QName more robust bab0ee8b5 createElement() -> addNewElement() e3a145747 grab changes from six-library 32285e95c Merge branch 'master' into feature/xml.lite_tweaks 9f79f0bf6 Merge branch 'master' into feature/xml.lite_tweaks a12bbc32c make it easier to create new Elements with a value fc9967f98 make it easy for callers to addChild() keep a reference to the Element 4627766b7 be sure test_xmlparser works in "externals" of other projects bf2276396 "private" is part of the name-mangling fad92bcc8 making sure copy-ctor is implemented f90fdcead consolidate common XML test code 9fc53f2d5 use str:: utility for casting 6da6f794b still trying to find the right macro for SWIG 0c1b86c56 still trying to fix SWIG fdc6fc9bd trying to fix SWIG build error 7835e8c27 SWIG needs copy-ctor 585695942 disable copy/assignment for Element, it's probably almost always wrong 391fed613 fix double-delete caused by copying 61790fe69 retry parsing XML with Windows-1252 if first parse() fails 63cffac59 change string_encoding to match coda-oss style of PascalCase 010479bbe read an XML file we know is wrongly encoded as Windows-1252 9a0505062 more references instead of pointers 2d44b6951 Reading Windows-1252 w/o "encoding" fails 63dc7b076 read Windows-1252 too c9434c9cb test as UIT-8 too f310ccf0c get reading from UTF-8 XML working on Windows 1fa39c2be get testReadUtf8XmlFile working on Linux 1a83cd815 sys::Path is too much trouble right now ed60aa22c unit-test to read XML from a file a9336db7c Squashed commit of the following: 0825beb0d Merge branch 'master' into feature/xml.lite_tweaks c618489be Merge branch 'master' into feature/xml.lite_tweaks e8e4b8fe1 determine string_encoding based on platform 1f43bcfc2 create a new Element by using the platform to determine "characterData" encoding 961bef66b Merge branch 'master' into feature/xml.lite_tweaks e9798a5cb fix static_assert() 6f7772874 Merge branch 'master' into feature/xml.lite_tweaks b98d4f5a9 Merge branch 'master' into feature/xml.lite_tweaks 1b5abba2a The (old) version of SWIG we're using doesn't like certain C++11 features. 53bdeabaf Merge branch 'master' into feature/xml.lite_tweaks 60cf8ae80 "" doesn't work with decltype() in older C++ 97e72477a reduce getValue() overloads by making "key" a template argument 5e6373e55 reduce code duplication f9e7cfeee provide castValue instead of getValue(T&) cbd0bd8f2 castValue throws instead of returning a bool like getValue(T&) 87c7514fc Merge branch 'master' into feature/xml.lite_tweaks 10cc61223 make getElement*() consistent for zero or >1 results f5b137e3c Merge branch 'master' into feature/xml.lite_tweaks 1765efc62 allow clients to specify toType() and toString() for getValue() and setValue() df8b746e1 allow clients to specify their own toType/toString routines 66702726a Merge branch 'master' into feature/xml.lite_tweaks 6956311f1 Merge branch 'master' into feature/xml.lite_tweaks d505f3593 Merge branch 'master' into feature/xml.lite_tweaks fbd106115 catch a BadCastException and return false from getValue() 3a78377b5 use a template to reduce duplicated code 0ad4b8606 Merge branch 'master' into feature/xml.lite_tweaks a848aa3a2 get & set the characer data as a type f3ee1ee12 utility routines to set an attribute value 595227683 templates to get an attribute value convert to a specific type 06639227b miised a change in last commit 1aa458ef8 add getValue() overloads that return true/false rather than throwing faa6d3075 added getElementByTagName() overloads as that's a very common use-case git-subtree-dir: externals/coda-oss git-subtree-split: 168cbae01bb1e5fc0a02c2d96e8b02619bf2831d commit 1c1e44094b5b567ae0a5368fc5060716fc83de1c Merge: 10eded084 c57aa2691 Author: Dan Smith Date: Mon Aug 21 09:14:48 2023 -0400 Merge commit 'c57aa2691a9635f55f5c604723d61ee495c59ed2' into develop/sync_externals commit 10eded0843f0d2e281becaf6d74f90902b99d1de Author: Dan Smith Date: Mon Aug 21 09:14:19 2023 -0400 latest from CODA-OSS and NITRO * most code should work with just io::SeekableStream&, not a shared_ptr * use std::vector* --- .../c++/cphd/include/cphd/CPHDWriter.h | 136 +++++--- six/modules/c++/cphd/source/CPHDWriter.cpp | 328 +++++++++++------- .../test_compressed_signal_block_round.cpp | 2 +- .../cphd/unittests/test_pvp_block_round.cpp | 2 +- .../unittests/test_support_block_round.cpp | 2 +- 5 files changed, 306 insertions(+), 164 deletions(-) diff --git a/six/modules/c++/cphd/include/cphd/CPHDWriter.h b/six/modules/c++/cphd/include/cphd/CPHDWriter.h index 49988b826..49aef9d77 100644 --- a/six/modules/c++/cphd/include/cphd/CPHDWriter.h +++ b/six/modules/c++/cphd/include/cphd/CPHDWriter.h @@ -28,6 +28,9 @@ #include #include #include +#include +#include +#include #include #include @@ -74,6 +77,12 @@ struct CPHDWriter final * that may be used if byte swapping is necessary. * Default is 4 MB */ + CPHDWriter( + const Metadata& metadata, + io::SeekableOutputStream& stream, + const std::vector* pSchemaPaths = nullptr, + size_t numThreads = 0, + size_t scratchSpaceSize = 4 * 1024 * 1024); CPHDWriter( const Metadata& metadata, std::shared_ptr stream, @@ -106,6 +115,12 @@ struct CPHDWriter final const std::vector& schemaPaths = std::vector(), size_t numThreads = 0, size_t scratchSpaceSize = 4 * 1024 * 1024); + CPHDWriter( + const Metadata& metadata, + const std::filesystem::path& pathname, + const std::vector* pSchemaPaths = nullptr, + size_t numThreads = 0, + size_t scratchSpaceSize = 4 * 1024 * 1024); CPHDWriter() = delete; CPHDWriter(const CPHDWriter&) = delete; @@ -114,6 +129,9 @@ struct CPHDWriter final CPHDWriter& operator=(CPHDWriter&&) = delete; ~CPHDWriter() = default; + std::unique_ptr make_DataWriter(io::OutputStream&) const; + std::unique_ptr make_DataWriter() const; + /* * \func write * \brief Writes the complete CPHD into the file. @@ -129,14 +147,27 @@ struct CPHDWriter final * \param supportData (Optional) The support array data to write to disk. */ template - void write(const PVPBlock& pvpBlock, const T* widebandData, + void write(io::SeekableOutputStream&, const PVPBlock& pvpBlock, const T* widebandData, std::span supportData); template + void write(const PVPBlock& pvpBlock, const T* widebandData, + std::span supportData) + { + write(mStream, pvpBlock, widebandData, supportData); + } + template void write(const PVPBlock& pvpBlock, const T* widebandData) { write(pvpBlock, widebandData, std::span()); } + template + size_t writeChannel(DataWriter&, const T* widebandData, size_t); + template + void writeChannels(DataWriter&, const T* widebandData); + + + /* * \func writeMetadata * \brief Writes the header, and metadata into the file. @@ -146,7 +177,13 @@ struct CPHDWriter final * * \param pvpBlock The vector based metadata to write. */ - void writeMetadata(const PVPBlock& pvpBlock); + void writeMetadata(io::OutputStream&, const PVPBlock& pvpBlock); + void writeMetadata(const PVPBlock& pvpBlock) + { + writeMetadata(mStream, pvpBlock); + } + void getMetadata(const PVPBlock&, size_t& supportSize, /*Optional*/ size_t& pvpSize, size_t& cphdSize) const; + /* * \func writeSupportData @@ -159,12 +196,17 @@ struct CPHDWriter final * \param id The unique identifier of the support array */ template - void writeSupportData(const T* data_, - const std::string& id) + void writeSupportData(io::OutputStream& stream, + const T* data_, const std::string& id) { const auto size = mMetadata.data.getSupportArrayById(id).size_bytes(); - const auto data = sys::make_span(data_, size); - writeSupportDataImpl(data, mMetadata.data.getSupportArrayById(id).bytesPerElement); + const auto data = sys::make_span(data_, size); + writeSupportDataImpl(stream, data, mMetadata.data.getSupportArrayById(id).bytesPerElement); + } + template + void writeSupportData(const T* data, const std::string& id) + { + writeSupportData(mStream, data, id); } /* @@ -175,16 +217,28 @@ struct CPHDWriter final * * \param data A pointer to the start of the support array data block */ - void writeSupportData(std::span); + void writeSupportData(io::SeekableOutputStream&, std::span); + + template + void writeSupportData(io::SeekableOutputStream& stream, std::span data) + { + writeSupportData(stream, std::as_bytes(data)); + } template void writeSupportData(std::span data) { - writeSupportData(std::as_bytes(data)); + writeSupportData(mStream, data); + } + + template + void writeSupportData(io::SeekableOutputStream& stream, const std::vector& data) + { + writeSupportData(stream, sys::make_span(data)); } template void writeSupportData(const std::vector& data) { - writeSupportData(sys::make_span(data)); + writeSupportData(mStream, data); } /* @@ -194,7 +248,13 @@ struct CPHDWriter final * \param PVPBlock A populated PVPBlock object that will be written * to the file as a block of data */ - void writePVPData(const PVPBlock& PVPBlock); + void writePVPData(io::SeekableOutputStream&, const PVPBlock&); + void writePVPData(const PVPBlock& pvpBlock) + { + writePVPData(mStream, pvpBlock); + } + static void writePvpPadBytes(io::OutputStream& stream, int64_t pvpPadBytes); + static void writePVPData(DataWriter& dataWriter, const cphd::Data&, const PVPBlock&); /* * \func writeCPHDData @@ -214,53 +274,49 @@ struct CPHDWriter final * \param channel For selecting channel of compressed signal block */ template - void writeCPHDData(const T* data, + void writeCPHDData(DataWriter&, + const T* data, + size_t numElements, + size_t channel = 1); + template + void writeCPHDData(io::SeekableOutputStream&, + const T* data, size_t numElements, size_t channel = 1); - - void close() + template + void writeCPHDData(const T* data, + size_t numElements, + size_t channel = 1) { - mStream->close(); + writeCPHDData(mStream, data, numElements, channel); } + std::shared_ptr getStream() const + { + return mSharedStream; + } + io::SeekableOutputStream& stream() const { return mStream; } + void close() + { + stream().close(); + } private: /* * Write metadata helper */ - void writeMetadata( + void writeMetadata(io::OutputStream&, size_t supportSize, // Optional size_t pvpSize, size_t cphdSize); - /* - * Write pvp helper - */ - void writePVPData(const std::byte* pvpBlock, - size_t index); - - /* - * Implementation of write wideband - */ - void writeCPHDDataImpl(const std::byte* data, - size_t size); - - /* - * Implementation of write compressed wideband - */ - void writeCompressedCPHDDataImpl(const std::byte* data, - size_t channel); - /* * Implementation of write support data */ - void writeSupportDataImpl(std::span, size_t elementSize); - - //! DataWriter object - std::unique_ptr mDataWriter; + void writeSupportDataImpl(io::OutputStream&, std::span, size_t elementSize); // Book-keeping element //! metadata information @@ -272,9 +328,11 @@ struct CPHDWriter final //! number of threads for parallelism const size_t mNumThreads; //! schemas for XML validation - const std::vector mSchemaPaths; + const std::vector mSchemaPaths; + const std::vector* mpSchemaPaths = nullptr; //! Output stream contains CPHD file - std::shared_ptr mStream; + std::shared_ptr mSharedStream; + io::SeekableOutputStream& mStream; //! header information FileHeader mHeader; }; diff --git a/six/modules/c++/cphd/source/CPHDWriter.cpp b/six/modules/c++/cphd/source/CPHDWriter.cpp index 9d649c18f..403e74750 100644 --- a/six/modules/c++/cphd/source/CPHDWriter.cpp +++ b/six/modules/c++/cphd/source/CPHDWriter.cpp @@ -23,7 +23,7 @@ #include #include -#include +#include #include @@ -46,6 +46,28 @@ static Version getVersion(const Metadata& metadata) return retval; } +CPHDWriter::CPHDWriter(const Metadata& metadata, + io::SeekableOutputStream& outStream, + const std::vector* pSchemaPaths, + size_t numThreads, + size_t scratchSpaceSize) : + mMetadata(metadata), + mElementSize(metadata.data.getNumBytesPerSample()), + mScratchSpaceSize(scratchSpaceSize), + mNumThreads(numThreads), + mpSchemaPaths(pSchemaPaths), + mStream(outStream), + mHeader(getVersion(metadata)) +{ +} + +static auto transform(const std::vector& schemaPaths) +{ + std::vector retval; + std::transform(schemaPaths.begin(), schemaPaths.end(), std::back_inserter(retval), + [](const std::string& s) { return s; }); + return retval; +} CPHDWriter::CPHDWriter(const Metadata& metadata, std::shared_ptr outStream, const std::vector& schemaPaths, @@ -55,13 +77,12 @@ CPHDWriter::CPHDWriter(const Metadata& metadata, mElementSize(metadata.data.getNumBytesPerSample()), mScratchSpaceSize(scratchSpaceSize), mNumThreads(numThreads), - mSchemaPaths(schemaPaths), - mStream(outStream), + mSchemaPaths(transform(schemaPaths)), + mpSchemaPaths(&mSchemaPaths), + mSharedStream(outStream), + mStream(*mSharedStream), mHeader(getVersion(metadata)) { - // Get the correct dataWriter. - // The CPHD file needs to be big endian. - mDataWriter = make_DataWriter(*mStream, mNumThreads, mScratchSpaceSize); } CPHDWriter::CPHDWriter(const Metadata& metadata, @@ -76,60 +97,64 @@ CPHDWriter::CPHDWriter(const Metadata& metadata, scratchSpaceSize) { } - -void CPHDWriter::writeMetadata(size_t supportSize, - size_t pvpSize, - size_t cphdSize) +CPHDWriter::CPHDWriter(const Metadata& metadata, + const std::filesystem::path& pathname, + const std::vector* pSchemaPaths, + size_t numThreads, + size_t scratchSpaceSize) : + mMetadata(metadata), + mElementSize(metadata.data.getNumBytesPerSample()), + mScratchSpaceSize(scratchSpaceSize), + mNumThreads(numThreads), + mpSchemaPaths(pSchemaPaths), + mSharedStream(std::make_shared(pathname)), // Initialize output stream + mStream(*mSharedStream), + mHeader(getVersion(metadata)) { - const auto xmlMetadata(CPHDXMLControl().toXMLString(mMetadata, mSchemaPaths)); +} +void CPHDWriter::writeMetadata(io::OutputStream& stream, + size_t supportSize, size_t pvpSize, size_t cphdSize) +{ mHeader.setVersion(getVersion(mMetadata)); // update classification and release info - if (!six::Init::isUndefined( - mMetadata.collectionID.getClassificationLevel()) && - !six::Init::isUndefined(mMetadata.collectionID.releaseInfo)) + auto&& collectionID = mMetadata.collectionID; + if (!six::Init::isUndefined(collectionID.getClassificationLevel()) && !six::Init::isUndefined(collectionID.releaseInfo)) { - mHeader.setClassification( - mMetadata.collectionID.getClassificationLevel()); - mHeader.setReleaseInfo(mMetadata.collectionID.releaseInfo); + mHeader.setClassification(collectionID.getClassificationLevel()); + mHeader.setReleaseInfo(collectionID.releaseInfo); } else { - throw except::Exception(Ctxt("Classification level and Release " - "informaion must be specified")); + throw except::Exception(Ctxt("Classification level and Release informaion must be specified")); } + + const auto xmlMetadata(CPHDXMLControl().toXMLString(mMetadata, mpSchemaPaths)); + // set header size, final step before write mHeader.set(xmlMetadata.size(), supportSize, pvpSize, cphdSize); - mStream->write(mHeader.toString()); - mStream->write("\f\n"); - mStream->write(xmlMetadata); - mStream->write("\f\n"); + stream.write(mHeader.toString()); + stream.write("\f\n"); + stream.write(xmlMetadata); + stream.write("\f\n"); } -void CPHDWriter::writePVPData(const std::byte* pvpBlock, size_t channel) +std::unique_ptr CPHDWriter::make_DataWriter(io::OutputStream& stream) const { - const size_t size = (mMetadata.data.getNumVectors(channel) * - mMetadata.data.getNumBytesPVPSet()) / - 8; - - //! The vector based parameters are always 64 bit - (*mDataWriter)(pvpBlock, size, 8); + // Get the correct dataWriter. + // The CPHD file needs to be big endian. + return cphd::make_DataWriter(stream, mNumThreads, mScratchSpaceSize); } - -void CPHDWriter::writeCPHDDataImpl(const std::byte* data, size_t size) +std::unique_ptr CPHDWriter::make_DataWriter() const { - //! We have to pass in the data as though it was not complex - // thus we pass in twice the number of elements at half the size. - (*mDataWriter)(data, size * 2, mElementSize / 2); + return make_DataWriter(mStream); } -void CPHDWriter::writeCompressedCPHDDataImpl(const std::byte* data, - size_t channel) +void CPHDWriter::writeSupportDataImpl(io::OutputStream& stream, std::span data, size_t elementSize) { - //! We have to pass in the data as though it was 1 signal array sized - // element of ubytes - (*mDataWriter)(data, mMetadata.data.getCompressedSignalSize(channel), 1); + auto dataWriter = make_DataWriter(stream); + (*dataWriter)(data, elementSize); } static auto make_span(std::span data, const cphd::Data::SupportArray& dataArray) @@ -137,116 +162,148 @@ static auto make_span(std::span data, const cphd::Data::Support const auto pData = data.data() + dataArray.arrayByteOffset; return sys::make_span(pData, dataArray.size_bytes()); } - -void CPHDWriter::writeSupportDataImpl(std::span data, size_t elementSize) -{ - (*mDataWriter)(data, elementSize); -} -void CPHDWriter::writeSupportData(std::span data) +void CPHDWriter::writeSupportData(io::SeekableOutputStream& stream, std::span data) { + const auto supportBlockByteOffset = mHeader.getSupportBlockByteOffset(); + + auto dataWriter = make_DataWriter(stream); for (auto&& mapEntry : mMetadata.data.supportArrayMap) { auto&& dataArray = mapEntry.second; // Move inputstream head to offset of particular support array - mStream->seek(mHeader.getSupportBlockByteOffset() + dataArray.arrayByteOffset, io::SeekableOutputStream::START); - writeSupportDataImpl(make_span(data, dataArray), dataArray.bytesPerElement); + stream.seek(supportBlockByteOffset + dataArray.arrayByteOffset, io::SeekableOutputStream::START); + + (*dataWriter)(make_span(data, dataArray), dataArray.bytesPerElement); } // Move inputstream head to the end of the support block after all supports have been written - mStream->seek(mHeader.getSupportBlockByteOffset() + mHeader.getSupportBlockSize(), io::SeekableOutputStream::START); + stream.seek(supportBlockByteOffset + mHeader.getSupportBlockSize(), io::SeekableOutputStream::START); } +template +size_t CPHDWriter::writeChannel(DataWriter& dataWriter, const T* widebandData, size_t channel) +{ + auto&& data = mMetadata.data; + + const size_t numElements = data.getNumVectors(channel) * data.getNumSamples(channel); + // writeCPHDData handles compressed data as well + writeCPHDData(dataWriter, widebandData, numElements, channel); + return numElements; +} template -void CPHDWriter::write(const PVPBlock& pvpBlock, +void CPHDWriter::writeChannels(DataWriter& dataWriter, const T* widebandData) +{ + // Doesn't require pading because pvp block is always 8 bytes words + // Write wideband (or signal) block + size_t elementsWritten = 0; // Used to increment widebandData pointer + for (size_t ii = 0; ii < mMetadata.data.getNumChannels(); ++ii) + { + elementsWritten += writeChannel(dataWriter, widebandData + elementsWritten, ii); + } +} + +template +void CPHDWriter::write(io::SeekableOutputStream& stream, + const PVPBlock& pvpBlock, const T* widebandData, std::span supportData) { + // Throw exception right away ... before writing anything. + if (mMetadata.data.getNumSupportArrays() != 0) + { + if (supportData.empty()) + { + throw except::Exception(Ctxt("SupportData is not provided")); + } + } + // Write File header and metadata to file // Padding is added in writeMetadata - writeMetadata(pvpBlock); + writeMetadata(stream, pvpBlock); // Write optional support array block // Padding is added in writeSupportData if (mMetadata.data.getNumSupportArrays() != 0) { - if (supportData.empty()) - { - throw except::Exception(Ctxt("SupportData is not provided")); - } writeSupportData(supportData); } // Write pvp data block // Padding is added in writePVPData - writePVPData(pvpBlock); + writePVPData(stream, pvpBlock); - // Doesn't require pading because pvp block is always 8 bytes words - // Write wideband (or signal) block - size_t elementsWritten = 0; // Used to increment widebandData pointer - for (size_t ii = 0; ii < mMetadata.data.getNumChannels(); ++ii) + if (!mMetadata.data.isCompressed()) { - size_t numElements = mMetadata.data.getNumVectors(ii) * - mMetadata.data.getNumSamples(ii); - // writeCPHDData handles compressed data as well - writeCPHDData(widebandData + elementsWritten, numElements, ii); - elementsWritten += numElements; + if (mElementSize != sizeof(T)) + { + throw except::Exception(Ctxt("Incorrect buffer data type used for metadata!")); + } } + auto dataWriter = make_DataWriter(stream); + writeChannels(*dataWriter, widebandData); } -template void CPHDWriter::write(const PVPBlock&, const std::byte* widebandData, std::span); // For compressed data -template void CPHDWriter::write(const PVPBlock&, const cphd::zint8_t* widebandData, std::span); -template void CPHDWriter::write(const PVPBlock&, const cphd::zint16_t* widebandData, std::span); -template void CPHDWriter::write(const PVPBlock&, const cphd::zfloat* widebandData, std::span); -//template void CPHDWriter::write(const PVPBlock&, const cphd::zint8_t* widebandData, std::span); -//template void CPHDWriter::write(const PVPBlock&, const cphd::zint16_t* widebandData, std::span); -//template void CPHDWriter::write(const PVPBlock&, const cphd::zfloat* widebandData, std::span); - -void CPHDWriter::writeMetadata(const PVPBlock& pvpBlock) +template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const std::byte* widebandData, std::span); // For compressed data +template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zint8_t* widebandData, std::span); +template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zint16_t* widebandData, std::span); +template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zfloat* widebandData, std::span); +//template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zint8_t* widebandData, std::span); +//template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zint16_t* widebandData, std::span); +//template void CPHDWriter::write(io::SeekableOutputStream&, const PVPBlock&, const cphd::zfloat* widebandData, std::span); + +void CPHDWriter::getMetadata(const PVPBlock& pvpBlock, + size_t& totalSupportSize, size_t& totalPVPSize, size_t& totalCPHDSize) const { + auto&& data = mMetadata.data; + // Update the number of bytes per PVP - if (mMetadata.data.numBytesPVP != pvpBlock.getNumBytesPVPSet()) + if (data.numBytesPVP != pvpBlock.getNumBytesPVPSet()) { std::ostringstream ostr; - ostr << "Number of pvp block bytes in metadata: " - << mMetadata.data.numBytesPVP - << " does not match calculated size of pvp block: " - << pvpBlock.getNumBytesPVPSet(); + ostr << "Number of pvp block bytes in metadata: " << data.numBytesPVP + << " does not match calculated size of pvp block: " << pvpBlock.getNumBytesPVPSet(); throw except::Exception(ostr.str()); } - const size_t numChannels = mMetadata.data.getNumChannels(); - size_t totalSupportSize = 0; - size_t totalPVPSize = 0; - size_t totalCPHDSize = 0; + const size_t numChannels = data.getNumChannels(); - for (auto it = mMetadata.data.supportArrayMap.begin(); - it != mMetadata.data.supportArrayMap.end(); - ++it) + totalSupportSize = 0; + for (auto&& kv : data.supportArrayMap) { - totalSupportSize += it->second.getSize(); + totalSupportSize += kv.second.getSize(); } + totalPVPSize = 0; + totalCPHDSize = 0; for (size_t ii = 0; ii < numChannels; ++ii) { totalPVPSize += pvpBlock.getPVPsize(ii); - totalCPHDSize += mMetadata.data.getNumVectors(ii) * - mMetadata.data.getNumSamples(ii) * mElementSize; + totalCPHDSize += data.getNumVectors(ii) * data.getNumSamples(ii) * mElementSize; } - - writeMetadata(totalSupportSize, totalPVPSize, totalCPHDSize); +} +void CPHDWriter::writeMetadata(io::OutputStream& stream, const PVPBlock& pvpBlock) +{ + size_t totalSupportSize; + size_t totalPVPSize; + size_t totalCPHDSize; + getMetadata(pvpBlock, totalSupportSize, totalPVPSize, totalCPHDSize); + writeMetadata(stream, totalSupportSize, totalPVPSize, totalCPHDSize); } -void CPHDWriter::writePVPData(const PVPBlock& pvpBlock) +void CPHDWriter::writePvpPadBytes(io::OutputStream& stream, int64_t pvpPadBytes) { // Add padding char zero = 0; - for (int64_t ii = 0; ii < mHeader.getPvpPadBytes(); ++ii) + for (int64_t ii = 0; ii < pvpPadBytes; ++ii) { - mStream->write(&zero, 1); + stream.write(&zero, 1); } +} +void CPHDWriter::writePVPData(DataWriter& dataWriter, const cphd::Data& data, const PVPBlock& pvpBlock) +{ // Write each PVP array - const size_t numChannels = mMetadata.data.getNumChannels(); + const size_t numChannels = data.getNumChannels(); std::vector pvpData; for (size_t ii = 0; ii < numChannels; ++ii) { @@ -257,48 +314,75 @@ void CPHDWriter::writePVPData(const PVPBlock& pvpBlock) ostr << "PVPBlock of channel " << ii << " is empty"; throw except::Exception(Ctxt(ostr.str())); } - writePVPData(pvpData.data(), ii); + + const size_t size = (data.getNumVectors(ii) * data.getNumBytesPVPSet()) / 8; + + //! The vector based parameters are always 64 bit + dataWriter(pvpData.data(), size, 8); } } +void CPHDWriter::writePVPData(io::SeekableOutputStream& stream, const PVPBlock& pvpBlock) +{ + // Add padding + writePvpPadBytes(stream, mHeader.getPvpPadBytes()); + + auto dataWriter = make_DataWriter(stream); + writePVPData(*dataWriter, mMetadata.data, pvpBlock); +} template -void CPHDWriter::writeCPHDData(const T* data, - size_t numElements, - size_t channel) +void CPHDWriter::writeCPHDData(DataWriter& dataWriter, + const T* data_, + size_t numElements, + size_t channel) { - const void* const pData = data; + const void* const pData = data_; + const std::byte* const data = static_cast(pData); + if (mMetadata.data.isCompressed()) { - writeCompressedCPHDDataImpl(static_cast(pData), channel); + //! We have to pass in the data as though it was 1 signal array sized + // element of ubytes + dataWriter(data, mMetadata.data.getCompressedSignalSize(channel), 1); } else + { + //! We have to pass in the data as though it was not complex + // thus we pass in twice the number of elements at half the size. + dataWriter(data, numElements * 2, mElementSize / 2); + } +} + +template +void CPHDWriter::writeCPHDData(io::SeekableOutputStream& stream, + const T* data, + size_t numElements, + size_t channel) +{ + if (!mMetadata.data.isCompressed()) { if (mElementSize != sizeof(T)) { throw except::Exception(Ctxt("Incorrect buffer data type used for metadata!")); } - writeCPHDDataImpl(static_cast(pData), numElements); } + auto dataWriter = make_DataWriter(stream); + writeCPHDData(*dataWriter, data, numElements, channel); } -// For compressed data -template void CPHDWriter::writeCPHDData(const sys::ubyte* data, - size_t numElements, - size_t channel); -template void CPHDWriter::writeCPHDData(const std::byte* data, - size_t numElements, - size_t channel); - -template void CPHDWriter::writeCPHDData( - const cphd::zint8_t* data, - size_t numElements, - size_t channel); - -template void CPHDWriter::writeCPHDData( - const cphd::zint16_t* data, - size_t numElements, - size_t channel); - -template void CPHDWriter::writeCPHDData( - const cphd::zfloat* data, size_t numElements, size_t channel); +template void CPHDWriter::writeCPHDData(io::SeekableOutputStream&, + const sys::ubyte* data, // For compressed data + size_t numElements, size_t channel); +template void CPHDWriter::writeCPHDData(io::SeekableOutputStream&, + const std::byte* data, + size_t numElements, size_t channel); +template void CPHDWriter::writeCPHDData(io::SeekableOutputStream&, + const cphd::zint8_t* data, + size_t numElements, size_t channel); +template void CPHDWriter::writeCPHDData(io::SeekableOutputStream&, + const cphd::zint16_t* data, + size_t numElements, size_t channel); +template void CPHDWriter::writeCPHDData(io::SeekableOutputStream&, + const cphd::zfloat* data, + size_t numElements, size_t channel); } diff --git a/six/modules/c++/cphd/unittests/test_compressed_signal_block_round.cpp b/six/modules/c++/cphd/unittests/test_compressed_signal_block_round.cpp index b897bd57d..b55216334 100644 --- a/six/modules/c++/cphd/unittests/test_compressed_signal_block_round.cpp +++ b/six/modules/c++/cphd/unittests/test_compressed_signal_block_round.cpp @@ -76,7 +76,7 @@ void writeCompressedCPHD(const std::string& outPathname, size_t numThreads, } } - cphd::CPHDWriter writer(metadata, outPathname, std::vector(), numThreads); + cphd::CPHDWriter writer(metadata, outPathname, nullptr /*pSchemaPaths*/, numThreads); writer.writeMetadata(pvpBlock); diff --git a/six/modules/c++/cphd/unittests/test_pvp_block_round.cpp b/six/modules/c++/cphd/unittests/test_pvp_block_round.cpp index 2a7673b76..51d5bd975 100644 --- a/six/modules/c++/cphd/unittests/test_pvp_block_round.cpp +++ b/six/modules/c++/cphd/unittests/test_pvp_block_round.cpp @@ -128,7 +128,7 @@ void writeCPHD(const std::string& outPathname, cphd::CPHDWriter writer(metadata, outPathname, - std::vector(), + nullptr /*pSchemaPaths*/, numThreads); writer.writeMetadata(pvpBlock); writer.writePVPData(pvpBlock); diff --git a/six/modules/c++/cphd/unittests/test_support_block_round.cpp b/six/modules/c++/cphd/unittests/test_support_block_round.cpp index 9dbb37eec..36899ec5e 100644 --- a/six/modules/c++/cphd/unittests/test_support_block_round.cpp +++ b/six/modules/c++/cphd/unittests/test_support_block_round.cpp @@ -84,7 +84,7 @@ void writeSupportData(const std::string& outPathname, size_t numThreads, cphd::setVectorParameters(ii, jj, pvpBlock); } } - cphd::CPHDWriter writer(metadata, outPathname, std::vector(), numThreads); + cphd::CPHDWriter writer(metadata, outPathname, nullptr /*pSchemaPaths*/, numThreads); writer.writeMetadata(pvpBlock); writer.writeSupportData(writeData); writer.writePVPData(pvpBlock);