diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..4e9379f --- /dev/null +++ b/.clang-format @@ -0,0 +1,88 @@ +--- +Language: Cpp +# AccessModifierOffset: -2 +# AlignAfterOpenBracket: Align +# AlignConsecutiveAssignments: false +# AlignConsecutiveDeclarations: false +# AlignEscapedNewlinesLeft: false +# AlignOperands: true +# AlignTrailingComments: true +# AllowAllParametersOfDeclarationOnNextLine: true +# AllowShortBlocksOnASingleLine: false +# AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: false +# AllowShortIfStatementsOnASingleLine: false +# AllowShortLoopsOnASingleLine: false +# AlwaysBreakAfterDefinitionReturnType: None +# AlwaysBreakAfterReturnType: None +# AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +# BinPackArguments: true +# BinPackParameters: true +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: false + AfterStruct: true + AfterUnion: false + BeforeCatch: false + BeforeElse: true + IndentBraces: false +#BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: true +ColumnLimit: 200 +# CommentPragmas: '^ IWYU pragma:' +# ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 0 +# ContinuationIndentWidth: 4 +# Cpp11BracedListStyle: true +# DerivePointerAlignment: false +# DisableFormat: false +# ExperimentalAutoDetectBinPacking: false +# ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +# IncludeCategories: +# - Regex: '^"(llvm|llvm-c|clang|clang-c)/' +# Priority: 2 +# - Regex: '^(<|"(gtest|isl|json)/)' +# Priority: 3 +# - Regex: '.*' +# Priority: 1 +# IndentCaseLabels: false +# IndentWidth: 2 +# IndentWrappedFunctionNames: false +# KeepEmptyLinesAtTheStartOfBlocks: true +# MacroBlockBegin: '' +# MacroBlockEnd: '' +# MaxEmptyLinesToKeep: 1 +# NamespaceIndentation: None +# ObjCBlockIndentWidth: 2 +# ObjCSpaceAfterProperty: false +# ObjCSpaceBeforeProtocolList: true +# PenaltyBreakBeforeFirstCallParameter: 19 +# PenaltyBreakComment: 300 +# PenaltyBreakFirstLessLess: 120 +# PenaltyBreakString: 1000 +# PenaltyExcessCharacter: 1000000 +# PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Left +# ReflowComments: true +SortIncludes: true +# SpaceAfterCStyleCast: false +# SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: Never +# SpaceInEmptyParentheses: false +# SpacesBeforeTrailingComments: 1 +# SpacesInAngles: false +# SpacesInContainerLiterals: true +# SpacesInCStyleCastParentheses: false +# SpacesInParentheses: false +# SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 2 +UseTab: Never +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..ca2482a --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,40 @@ +--- +Checks: ' + -abseil-*, + bugprone-*, + -boost-*, + -cert-*, + clang-diagnostic-*, + clang-analyzer-*, + cppcoreguidelines-*, + -cppcoreguidelines-avoid-magic-numbers, + -darwin-*, + -fuchsia-*, + -google-*, + -hicpp-*, + -linuxkernel-*, + -llvm-*, + misc-*, + moderize-*, + -modernize-use-trailing-return-type, + -modernize-use-nodiscard, + -moderize-use-auto, + -mpi-*, + -objc-*, + -openmp-*, + performance-*, + -portability-*, + readability-*, + -readability-function-cognitive-complexity, + -readability-function-size, + -readability-identifier-naming, + -readability-uppercase-literal-suffix, + -readability-magic-numbers +' +WarningsAsErrors: '*' +HeaderFilterRegex: 'complex/.*\.hpp' +FormatStyle: file +CheckOptions: + - key: cppcoreguidelines-macro-usage.AllowedRegexp + value: 'COMPLEX_EXPORT|COMPLEX_NO_EXPORT|COMPLEX_DEPRECATED' +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..547ec8f --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ + +.vscode/ +.idea/ +vcpkg_installed/ +CMakeLists.txt.* +/[Bb]uild/ +/[Bb]uilds/ +/Build-ninja/ +/Build-Xcode/ +.cproject +.project +/clang/ +/xcode/ +/x64/ +/i386/ +/qtcreator/ +/Build2/ +/QT_BUILD/.settings +.floo +/ninja/ +/linux/ +.idea +/[Dd]ebug/ +/[Rr]elease/ +.ctags +.vscode +*.user* +test_data/ +CMakeUserPresets.json +Workspace diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..fa56f57 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.19) + + +project(BCFTools + VERSION 0.1.0 + DESCRIPTION "SFS Reader" + HOMEPAGE_URL "" + LANGUAGES CXX + ) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + + +#------------------------------------------------------------------------------- +# unbcf executable +#------------------------------------------------------------------------------- +set(unbcf_sources + ${BCFTools_SOURCE_DIR}/src/SFSReader.h + ${BCFTools_SOURCE_DIR}/src/SFSReader.cpp + + ${BCFTools_SOURCE_DIR}/src/SFSNodeItem.h + ${BCFTools_SOURCE_DIR}/src/SFSNodeItem.cpp + + ${BCFTools_SOURCE_DIR}/src/SFSUtils.hpp + ${BCFTools_SOURCE_DIR}/src/unbcf.cpp +) + +add_executable(unbcf ${unbcf_sources}) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..819216e --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,9 @@ +# BCF Tools # + +This repository contains a few programs to allow extraction of EBSD Data from the Bruker Esprit .bcf files. + +## unbcf ## + +The `unbcf` program is a general tool to unpack a non-compressed and non-encrypted .bcf file into a folder. The program only requires 2 arguments, the input .bcf file (or SFS file for that matter) and a directory to place the contents. A subfolder will be created for you inside of the given output folder that has the name of the input file (without the extension) + +The SFS Reader code were heavily influenced from the [HyperSpy](https://hyperspy.org/) project. diff --git a/src/BcfHdf5Convertor.cpp b/src/BcfHdf5Convertor.cpp new file mode 100644 index 0000000..105343e --- /dev/null +++ b/src/BcfHdf5Convertor.cpp @@ -0,0 +1,1237 @@ +#include "BcfHdf5Convertor.h" + +#include "BrukerIntegration/BrukerIntegrationConstants.h" +#include "BrukerIntegration/BrukerIntegrationStructs.h" +#include "BrukerIntegrationFilters/BrukerDataLoader.h" + +#include "SIMPLib/DataArrays/DataArray.hpp" +#include "SIMPLib/Math/SIMPLibMath.h" + +#include "H5Support/H5Lite.h" +#include "H5Support/H5ScopedErrorHandler.h" +#include "H5Support/H5ScopedSentinel.h" +#include "H5Support/H5Utilities.h" +using namespace H5Support; + +#include "SFSNodeItem.h" +#include "SFSReader.h" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) && !defined(__APPLE__) +#define BDL_POS(var) var.__pos +#else +#define BDL_POS(var) var +#endif + +namespace +{ +const std::string k_EBSD("EBSD"); +const std::string k_SEM("SEM"); +const std::string k_Manufacturer("Manufacturer"); +const std::string k_Version("Version"); +const std::string k_Data("Data"); +const std::string k_Header("Header"); + +const int32_t k_FileVersion = 3; + +/****************************************************************************** + * START TIFF WRITING SECTION + *****************************************************************************/ +const uint16_t PHOTOMETRIC_MINISBLACK = 0x0001; +const uint16_t PHOTOMETRIC_RGB = 0x0002; + +struct TIFTAG +{ + int16_t tagId = 0; // The tag identifier + int16_t dataType = 0; // The scalar type of the data items + int32_t dataCount = 0; // The number of items in the tag data + int32_t dataOffset = 0; // The byte offset to the data items + + friend std::ostream& operator<<(std::ostream& out, const TIFTAG& tag) + { + out.write(reinterpret_cast(&tag.tagId), sizeof(tag.tagId)); + out.write(reinterpret_cast(&tag.dataType), sizeof(tag.dataType)); + out.write(reinterpret_cast(&tag.dataCount), sizeof(tag.dataCount)); + out.write(reinterpret_cast(&tag.dataOffset), sizeof(tag.dataOffset)); + return out; + } +}; + +enum class Endianess +{ + Little = 0, + Big +}; + +Endianess checkEndianess() +{ + constexpr uint32_t i = 0x01020304; + const std::byte* u8 = reinterpret_cast(&i); + + return u8[0] == std::byte{0x01} ? Endianess::Big : Endianess::Little; +} + +// This is just here to silence the compiler. We are not fully supporting tiff writing as an option. Just for debugging +std::pair WriteGrayScaleImage(const std::string& filepath, int32_t width, int32_t height, const uint16_t* data) +{ + return {-1, "16 Bit tiff not supported"}; +} + +// ----------------------------------------------------------------------------- +std::pair WriteGrayScaleImage(const std::string& filepath, int32_t width, int32_t height, const uint8_t* data) +{ + int32_t samplesPerPixel = 1; + // Check for Endianess of the system and write the appropriate magic number according to the tiff spec + std::array magicNumber = {0x49, 0x49, 0x2A, 0x00}; + + if(checkEndianess() == Endianess::Big) + { + magicNumber = {0x4D, 0x4D, 0x00, 0x2A}; + } + + // open file and write header + std::ofstream outputFile(filepath, std::ios::binary); + if(!outputFile.is_open()) + { + return {-1, "Could not open output file for writing"}; + } + + outputFile.write(magicNumber.data(), magicNumber.size()); + // Generate the offset into the Image File Directory (ifd) which we are going to write first + constexpr uint32_t ifd_Offset = 8; + outputFile.write(reinterpret_cast(&ifd_Offset), sizeof(ifd_Offset)); + + std::vector tags; + tags.push_back(TIFTAG{0x00FE, 0x0004, 1, 0x00000000}); // NewSubfileType + tags.push_back(TIFTAG{0x0100, 0x0004, 1, width}); // ImageWidth + tags.push_back(TIFTAG{0x0101, 0x0004, 1, height}); // ImageLength + tags.push_back(TIFTAG{0x0102, 0x0003, 1, 8 * sizeof(char)}); // BitsPerSample + tags.push_back(TIFTAG{0x0103, 0x0003, 1, 0x0001}); // Compression + tags.push_back(TIFTAG{0x0106, 0x0003, 1, PHOTOMETRIC_MINISBLACK}); // PhotometricInterpretation // For SamplesPerPixel = 3 or 4 (RGB or RGBA) + tags.push_back(TIFTAG{0x0112, 0x0003, 1, 1}); // Orientation + tags.push_back(TIFTAG{0x0115, 0x0003, 1, 1}); // SamplesPerPixel + tags.push_back(TIFTAG{0x0116, 0x0004, 1, height}); // RowsPerStrip + tags.push_back(TIFTAG{0x0117, 0x0004, 1, width * height * samplesPerPixel}); // StripByteCounts + // TIFTAG XResolution; + // TIFTAG YResolution; + // TIFTAG ResolutionUnit; + tags.push_back(TIFTAG{0x011c, 0x0003, 1, 0x0001}); // PlanarConfiguration + + // Now compute the offset to the image data so that we can put that into the tag. + // THESE NEXT 2 LINES MUST BE THE LAST TAG TO BE PUSHED BACK INTO THE VECTOR OR THE MATH WILL BE WRONG + int32_t imageDataOffset = static_cast(8 + ((tags.size() + 1) * 12) + 6); // Header + tags + IDF Tag entry count and Next IFD Offset + tags.push_back(TIFTAG{0x0111, 0x0004, 1, imageDataOffset}); // StripOffsets + + // Write the number of tags to the IFD section + uint16_t numEntries = static_cast(tags.size()); + outputFile.write(reinterpret_cast(&numEntries), sizeof(numEntries)); + // write the tags to the file. + for(const auto& tag : tags) + { + outputFile << tag; + } + // Write the "Next Tag Offset" + constexpr uint32_t nextOffset = 0; + outputFile.write(reinterpret_cast(&nextOffset), sizeof(nextOffset)); + + // Now write the actual image data + int32_t imageByteCount = width * height * samplesPerPixel; + outputFile.write(reinterpret_cast(data), imageByteCount); + + // and we are done. + return {0, "No Error"}; +} + +} // namespace + +// ----------------------------------------------------------------------------- +BcfHdf5Convertor::BcfHdf5Convertor(QString inputFile, QString outputFile) +: m_InputFile(std::move(inputFile)) +, m_OutputFile(std::move(outputFile)) +{ +} + +// ----------------------------------------------------------------------------- +BcfHdf5Convertor::~BcfHdf5Convertor() = default; + +void BcfHdf5Convertor::setReorder(bool reorder) +{ + m_Reorder = reorder; +} + +void BcfHdf5Convertor::setFlipPatterns(bool flipPatterns) +{ + m_FlipPatterns = flipPatterns; +} + +// ----------------------------------------------------------------------------- +int32_t writeCameraConfiguration(hid_t semGrpId, hid_t ebsdGrpId, const QString& cameraConfiguration) +{ + QString errorStr; + int errorLine = -1; + int errorColumn = -1; + + QFile device(cameraConfiguration); + + QDomDocument domDocument; + QDomElement root; + + if(!domDocument.setContent(&device, true, &errorStr, &errorLine, &errorColumn)) + { + QString ss = QObject::tr("Parse error at line %1, column %2:\n%3").arg(errorLine).arg(errorColumn).arg(errorStr); + qDebug() << ss; + return -7080; + } + + root = domDocument.documentElement(); + QDomElement classInstance = root.firstChildElement("ClassInstance"); + if(classInstance.isNull()) + { + std::cout << "XML DOM entry ClassInstance was not found." << std::endl; + return -7002; + } + int32_t err = 0; + + // Start gathering the required information from the XML file + QString pixelFormat = classInstance.firstChildElement("PixelFormat").text(); + int32_t pixelByteCount = 0; + if(pixelFormat == "Gray8") + { + pixelByteCount = 1; + } + else if(pixelFormat == "Gray16") + { + pixelByteCount = 2; + } + err = H5Lite::writeScalarDataset(ebsdGrpId, "PixelByteCount", pixelByteCount); + + return err; +} + +// ----------------------------------------------------------------------------- +int32_t writeAuxIndexingOptions(hid_t semGrpId, hid_t ebsdGrpId, const QString& calibrationFile) +{ + QString errorStr; + int errorLine = -1; + int errorColumn = -1; + + QFile device(calibrationFile); + + QDomDocument domDocument; + QDomElement root; + + if(!domDocument.setContent(&device, true, &errorStr, &errorLine, &errorColumn)) + { + QString ss = QObject::tr("Parse error at line %1, column %2:\n%3").arg(errorLine).arg(errorColumn).arg(errorStr); + qDebug() << ss; + return -7080; + } + + root = domDocument.documentElement(); + QDomElement classInstance = root.firstChildElement("ClassInstance"); + if(classInstance.isNull()) + { + std::cout << "XML DOM entry ClassInstance was not found." << std::endl; + return -7002; + } + bool ok = false; + int32_t err = 0; + + // Start gathering the required information from the XML file + int32_t minIndexedBands = classInstance.firstChildElement("MinIndexedBandCount").text().toInt(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "MinIndexedBands", minIndexedBands); + + // Start gathering the required information from the XML file + double madMax = classInstance.firstChildElement("MaxMAD").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "MADMax", madMax); + + return err; +} + +// ----------------------------------------------------------------------------- +int32_t writeCalibrationData(hid_t semGrpId, hid_t ebsdGrpId, const QString& calibrationFile, float& pcx, float& pcy) +{ + QString errorStr; + int errorLine = -1; + int errorColumn = -1; + + QFile device(calibrationFile); + + QDomDocument domDocument; + QDomElement root; + + if(!domDocument.setContent(&device, true, &errorStr, &errorLine, &errorColumn)) + { + QString ss = QObject::tr("Parse error at line %1, column %2:\n%3").arg(errorLine).arg(errorColumn).arg(errorStr); + qDebug() << ss; + return -7080; + } + + root = domDocument.documentElement(); + QDomElement classInstance = root.firstChildElement("ClassInstance"); + if(classInstance.isNull()) + { + std::cout << "XML DOM entry ClassInstance was not found." << std::endl; + return -7002; + } + bool ok = false; + int32_t err = 0; + + // Start gathering the required information from the XML file + double workingDistance = classInstance.firstChildElement("WorkingDistance").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(semGrpId, "SEM WD", workingDistance); + err = H5Lite::writeScalarDataset(ebsdGrpId, "WD", workingDistance); + + double topClip = classInstance.firstChildElement("TopClip").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "TopClip", topClip); + + pcx = classInstance.firstChildElement("PCX").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "PCX", pcx); + + pcy = classInstance.firstChildElement("PCY").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "PCX", pcy); + + float sampleTilt = classInstance.firstChildElement("ProbeTilt").text().toDouble(&ok); + err = H5Lite::writeScalarDataset(ebsdGrpId, "SampleTilt", sampleTilt); + + + return err; +} + +// ----------------------------------------------------------------------------- +int32_t writeSEMData(hid_t semGrpId, hid_t ebsdGrpId, const QString& semFile) +{ + QString errorStr; + int errorLine; + int errorColumn; + + QFile device(semFile); + + QDomDocument domDocument; + QDomElement root; + + if(!domDocument.setContent(&device, true, &errorStr, &errorLine, &errorColumn)) + { + QString ss = QObject::tr("Parse error at line %1, column %2:\n%3").arg(errorLine).arg(errorColumn).arg(errorStr); + qDebug() << ss; + return -7080; + } + + root = domDocument.documentElement(); + QDomElement classInstance = root.firstChildElement("ClassInstance"); + if(classInstance.isNull()) + { + std::cout << "XML DOM entry ClassInstance was not found." << std::endl; + return -7002; + } + bool ok = false; + int32_t err = 0; + + QString date = classInstance.firstChildElement("Date").text(); + err = H5Lite::writeStringDataset(ebsdGrpId, "Date", date.toStdString()); + err = H5Lite::writeStringAttribute(ebsdGrpId, "Date", "Format (ISO 8601)", "dd.mm.yyyy"); + + QString time = classInstance.firstChildElement("Time").text(); + err = H5Lite::writeStringDataset(ebsdGrpId, "Time", time.toStdString()); + err = H5Lite::writeStringAttribute(ebsdGrpId, "Time", "Format (ISO 8601)", "hh:mm:ss"); + + int32_t width = classInstance.firstChildElement("Width").text().toInt(&ok); + err = H5Lite::writeScalarDataset(semGrpId, "SEM ImageWidth", width); + + int32_t height = classInstance.firstChildElement("Height").text().toInt(&ok); + err = H5Lite::writeScalarDataset(semGrpId, "SEM ImageHeight", height); + + std::vector tDims = {static_cast(height), static_cast(width)}; + + float xRes = classInstance.firstChildElement("XCalibration").text().toFloat(&ok); + if(xRes == 0.0) { xRes = 1.0f;} + err = H5Lite::writeScalarDataset(semGrpId, "SEM XResolution", xRes); + err = H5Lite::writeScalarDataset(ebsdGrpId, "SEPixelSizeX", xRes); + err = H5Lite::writeScalarDataset(ebsdGrpId, "XSTEP", xRes); + + float yRes = classInstance.firstChildElement("YCalibration").text().toFloat(&ok); + if(yRes == 0.0) { yRes = 1.0f; } + err = H5Lite::writeScalarDataset(semGrpId, "SEM YResolution", yRes); + err = H5Lite::writeScalarDataset(ebsdGrpId, "SEPixelSizeY", yRes); + err = H5Lite::writeScalarDataset(ebsdGrpId, "YSTEP", yRes); + + int itemSize = classInstance.firstChildElement("ItemSize").text().toInt(&ok); + + int32_t planeCount = classInstance.firstChildElement("PlaneCount").text().toInt(&ok); + for(int32_t p = 0; p < planeCount; p++) + { + QString tagName = "Plane" + QString::number(p); + QDomElement planeDomEle = classInstance.firstChildElement(tagName); + QByteArray b64str = planeDomEle.firstChildElement("Data").text().toLatin1(); + QByteArray decodedImage = QByteArray::fromBase64(b64str); + QString nameDomEle = planeDomEle.firstChildElement("Name").text(); + QString descDomEle = planeDomEle.firstChildElement("Description").text(); + + if(!nameDomEle.isEmpty() && !descDomEle.isEmpty()) + { + if(itemSize == 1) + { + err = H5Lite::writePointerDataset(semGrpId, "SEM Image", 2, tDims.data(), reinterpret_cast(decodedImage.data())); + } + else if(itemSize == 2) + { + err = H5Lite::writePointerDataset(semGrpId, "SEM Image", 2, tDims.data(), reinterpret_cast(decodedImage.data())); + } + err = H5Lite::writeStringAttribute(semGrpId, "SEM Image", "CLASS", "IMAGE"); + err = H5Lite::writeStringAttribute(semGrpId, "SEM Image", "IMAGE_SUBCLASS", "IMAGE_INDEXED"); + err = H5Lite::writeStringAttribute(semGrpId, "SEM Image", "IMAGE_VERSION", "1.2"); + + // And now write the same image in the EBSD/Header group? + if(itemSize == 1) + { + err = H5Lite::writePointerDataset(ebsdGrpId, "SEM Image", 2, tDims.data(), reinterpret_cast(decodedImage.data())); + } + else if(itemSize == 2) + { + err = H5Lite::writePointerDataset(ebsdGrpId, "SEM Image", 2, tDims.data(), reinterpret_cast(decodedImage.data())); + } + err = H5Lite::writeStringAttribute(ebsdGrpId, "SEM Image", "CLASS", "IMAGE"); + err = H5Lite::writeStringAttribute(ebsdGrpId, "SEM Image", "IMAGE_SUBCLASS", "IMAGE_INDEXED"); + err = H5Lite::writeStringAttribute(ebsdGrpId, "SEM Image", "IMAGE_VERSION", "1.2"); + + if(!nameDomEle.isEmpty()) + { + err = H5Lite::writeStringAttribute(semGrpId, "SEM Image", "Name", nameDomEle.toStdString()); + err = H5Lite::writeStringAttribute(ebsdGrpId, "SEM Image", "Name", nameDomEle.toStdString()); + } + if(!descDomEle.isEmpty()) + { + err = H5Lite::writeStringAttribute(semGrpId, "SEM Image", "Description", descDomEle.toStdString()); + err = H5Lite::writeStringAttribute(ebsdGrpId, "SEM Image", "Description", descDomEle.toStdString()); + } + } + } + + float semKV = 0.0f; + float semMag = -1.0f; + + QDomElement trtHeaderedClass = classInstance.firstChildElement("TRTHeaderedClass"); + QDomElement tRTREMHeader = trtHeaderedClass.firstChildElement("ClassInstance"); + if(!tRTREMHeader.isNull()) + { + semKV = tRTREMHeader.firstChildElement("Energy").text().toFloat(&ok); + semMag = tRTREMHeader.firstChildElement("Magnification").text().toFloat(&ok); + // float wd = tRTREMHeader.firstChildElement("WorkingDistance").text().toFloat(&ok); + // err = H5Lite::writeScalarDataset(semGrpId, "SEM WD", wd); + } + err = H5Lite::writeScalarDataset(semGrpId, "SEM KV", semKV); + err = H5Lite::writeScalarDataset(ebsdGrpId, "KV", semKV); + err = H5Lite::writeScalarDataset(semGrpId, "SEM Magnification", semMag); + err = H5Lite::writeScalarDataset(ebsdGrpId, "Magnification", semMag); + + return 0; +} + +// ----------------------------------------------------------------------------- +int32_t writePhaseInformation(hid_t headerGrpId, const QString& phaseListFile) +{ + + hid_t phaseGrpId = H5Utilities::createGroup(headerGrpId, Bruker::Header::Phases.toStdString()); + H5GroupAutoCloser phaseGrpAutoClose(phaseGrpId); + QString errorStr; + int errorLine; + int errorColumn; + + QFile device(phaseListFile); + + QDomDocument domDocument; + QDomElement root; + + if(!domDocument.setContent(&device, true, &errorStr, &errorLine, &errorColumn)) + { + QString ss = QObject::tr("Parse error at line %1, column %2:\n%3").arg(errorLine).arg(errorColumn).arg(errorStr); + std::cout << ss.toStdString() << std::endl; + return -70000; + } + + root = domDocument.documentElement(); + + QDomElement classInstance = root.firstChildElement("ClassInstance"); + if(classInstance.isNull()) + { + std::cout << "XML DOM entry ClassInstance was not found." << std::endl; + return -7002; + } + + QDomElement childClassInstances = classInstance.firstChildElement("ChildClassInstances"); + if(childClassInstances.isNull()) + { + std::cout << "XML DOM entry ChildClassInstances was not found." << std::endl; + return -7003; + } + + QDomNodeList phaseInstaces = childClassInstances.childNodes(); + int32_t count = phaseInstaces.count(); + for(int32_t i = 0; i < count; i++) + { + int32_t err = 0; + bool ok = true; + hid_t grpId = H5Utilities::createGroup(phaseGrpId, QString::number(i + 1).toStdString()); + H5GroupAutoCloser groupAutoCloser(grpId); + + QDomNode phaseInstance = phaseInstaces.at(i); + QString name = phaseInstance.toElement().attribute("Name"); + + err = H5Lite::writeStringDataset(grpId, "Name", name.toStdString()); + + QDomNode tEBSDPhaseEntry = phaseInstance.firstChildElement("TEBSDPhaseEntry"); + + // phaseInstance.firstChildElement("POS0").text() ); + + // phaseInstance.firstChildElement("REF").text() ); + + // phaseInstance.firstChildElement("Complete").text() ); + + // phaseInstance.firstChildElement("BPS1").text() ); + + // phaseInstance.firstChildElement("BPS2").text() ); + QString strValue = tEBSDPhaseEntry.firstChildElement("Chem").text(); + err = H5Lite::writeStringDataset(grpId, "Formula", strValue.toStdString()); + + // phaseInstance.firstChildElement("DENSITY").text() ); + + // phaseInstance.firstChildElement("Elem").text() ); + + /* Parse a DefaultGlideSystem subclass */ + // m_DefaultGlideSystem = DefaultGlideSystem::Pointer(new DefaultGlideSystem(parent())); + // QDomElement ele_DefaultGlideSystem = phaseInstance.firstChildElement("DefaultGlideSystem"); + // err = m_DefaultGlideSystem->parse(ele_DefaultGlideSystem); + + // phaseInstance.firstChildElement("ISUSERDB").text() ); + + // phaseInstance.firstChildElement("PKEY").text() ); + + // phaseInstance.firstChildElement("DBNAME").text() ); + + /* Parse a Cell subclass */ + // m_Cell = Cell::Pointer(new Cell(parent())); + QDomElement ele_Cell = tEBSDPhaseEntry.firstChildElement("Cell"); + { + std::vector latticeConstants(6); + QStringList values = ele_Cell.firstChildElement("Dim").text().split(","); + latticeConstants[0] = values[0].toFloat(&ok); + latticeConstants[1] = values[1].toFloat(&ok); + latticeConstants[2] = values[2].toFloat(&ok); + values = ele_Cell.firstChildElement("Angles").text().split(","); + latticeConstants[3] = values[0].toFloat(&ok); + latticeConstants[4] = values[1].toFloat(&ok); + latticeConstants[5] = values[2].toFloat(&ok); + std::vector dims = {6}; + err = H5Lite::writeVectorDataset(grpId, "LatticeConstants", dims, latticeConstants); + } + + // err = m_Cell->parse(ele_Cell); + + // phaseInstance.firstChildElement("MU").text() ); + + // phaseInstance.firstChildElement("USED").text() ); + + // phaseInstance.firstChildElement("ENTRYID").text() ); + + // phaseInstance.firstChildElement("Encrypted").text() ); + + int32_t se = tEBSDPhaseEntry.firstChildElement("SE").text().toInt(&ok); + err = H5Lite::writeScalarDataset(grpId, "Setting", se); + + // phaseInstance.firstChildElement("CC").text() ); + + // phaseInstance.firstChildElement("REFLVERSION").text() ); + + strValue = tEBSDPhaseEntry.firstChildElement("SG").text(); + err = H5Lite::writeStringDataset(grpId, "SpaceGroup", strValue.toStdString()); + + int32_t it = tEBSDPhaseEntry.firstChildElement("IT").text().toInt(&ok); + err = H5Lite::writeScalarDataset(grpId, "IT", it); + + hid_t atGrpId = H5Utilities::createGroup(grpId, "AtomPositions"); + H5GroupAutoCloser atGrpIdAutoCloser(atGrpId); + // This tells us the number of atoms that we need to parse + it = tEBSDPhaseEntry.firstChildElement("AT").text().toInt(&ok); + for(int32_t atom = 1; atom <= it; atom++) + { + QString tagName = "POS" + QString::number(atom - 1); + strValue = tEBSDPhaseEntry.firstChildElement(tagName).text(); + err = H5Lite::writeStringDataset(atGrpId, QString::number(atom).toStdString(), strValue.toStdString()); + } + } + + return 0; +} + +// ----------------------------------------------------------------------------- +template +int32_t writeDataArrayToHdf5(DataArray* data, hid_t grpId, size_t numElements) +{ + std::vector cDims = data->getComponentDimensions(); + if(cDims.size() == 1 && cDims[0] == 1) + { + cDims.clear(); + } +// int32_t rank = 1 + cDims.size(); + std::vector dims; + dims.push_back(numElements); + for(const auto& dim : cDims) + { + dims.push_back(dim); + } + + int32_t err = H5Lite::writePointerDataset(grpId, data->getName().toStdString(), static_cast(dims.size()), dims.data(), data->getPointer(0)); + return err; +} + +// ----------------------------------------------------------------------------- +int32_t analyzeFrameDescriptionFile(const QString& descFile) +{ + std::cout << "************** Frame Description File START ****************************" << std::endl; + QString filePath = descFile; + QFileInfo descFileInfo(filePath); + if(!descFileInfo.exists()) + { + std::cout << "The FrameDescription File does not exist: '" << filePath.toStdString() << "'" << std::endl; + return -10; + } + + // Open the FrameData File + FILE* f = fopen(descFileInfo.absoluteFilePath().toStdString().c_str(), "rb"); + if(nullptr == f) + { + std::cout << "Could not open the FrameData File" << std::endl; + return -14; + } + + FrameDescriptionHeader_t descHeader; + // uint64_t filePos = 0; + size_t nRead = fread(&descHeader, 12, 1, f); + std::cout << "Frame Description File:" << std::endl; + std::cout << " Width:" << descHeader.width << std::endl; + std::cout << " Height:" << descHeader.height << std::endl; + std::cout << " Total Possible Scanned Points:" << descHeader.patternCount << std::endl; + int32_t totalPatternsAvailable = 0; + uint64_t maxIndex = 0; + for(int32_t i = 0; i < descHeader.patternCount; i++) + { + uint64_t offset = 0; + fpos_t pos; + fgetpos(f, &pos); + nRead = fread(&offset, sizeof(uint64_t), 1, f); + if(offset != 0xFFFFFFFFFFFFFFFF) + { + totalPatternsAvailable++; + if(offset > maxIndex) + { + maxIndex = offset; + } + } + } + std::cout << "Total Pixels Measured: " << totalPatternsAvailable << std::endl; + // std::cout << "Max File Index: " << maxIndex << std::endl; + fclose(f); + std::cout << "************** Frame Description File END ****************************" << std::endl; + + return 0; +} + + +// ----------------------------------------------------------------------------- +template +int32_t writePatternData(const SFSReader& sfsFile, hid_t native_type, int32_t mapWidth, int32_t mapHeight, int32_t ebspWidth, + int32_t ebspHeight, bool flipPatterns, const QString& tempDir, const QString& dataFile, + const QString& descFile, hid_t dataGrpId) +{ + int32_t err = 0; + // =================================================== + // Get the XBEAM and YBEAM data from the HDF5 file + std::vector xbeam; + err = H5Lite::readVectorDataset(dataGrpId, Bruker::IndexingResults::XBEAM.toStdString(), xbeam); + + std::vector ybeam; + err = H5Lite::readVectorDataset(dataGrpId, Bruker::IndexingResults::YBEAM.toStdString(), ybeam); + + // =================================================== + // Check the FrameDescription File exists + QString filePath = descFile; + QFileInfo descFileInfo(filePath); + if(!descFileInfo.exists()) + { + std::cout << "The FrameDescription File does not exist: '" << filePath.toStdString() << "'" << std::endl; + return -10; + } + err = analyzeFrameDescriptionFile(descFile); + if(err < 0) + { + return err; + } + + // Open the FrameDescription File. This tells for every scan point, where the pattern + // data starts in the FrameData file. + std::vector frameDescription; + { + FILE* f = fopen(descFileInfo.absoluteFilePath().toStdString().c_str(), "rb"); + FrameDescriptionHeader_t descHeader; + size_t nRead = fread(&descHeader, 12, 1, f); + frameDescription.resize(descHeader.patternCount); + nRead = fread(frameDescription.data(), sizeof(size_t), descHeader.patternCount, f); + fclose(f); + } + + // =================================================== + // Extract the FrameData file from the .bcf file. this can take a bit.... + err = sfsFile.extractFile(tempDir.toStdString(), dataFile.toStdString()); + if(err != 0) + { + std::cout << "Error extracting the " << dataFile.toStdString() << ". This data set will not be included in the resulting HDF5 file." << std::endl; + return err; + } + + filePath = tempDir + "/" + dataFile; + QFileInfo dataFileInfo(filePath); + if(!dataFileInfo.exists()) + { + std::cout << "The FrameData File does not exist: '" << filePath.toStdString() << "'" << std::endl; + return -11; + } + qint64 filesize = dataFileInfo.size(); + + // Open the FrameData File + FILE* f = fopen(dataFileInfo.absoluteFilePath().toStdString().c_str(), "rb"); + if(nullptr == f) + { + std::cout << "Could not open the FrameData File" << std::endl; + return -14; + } + + // Put the file pointer back to the start of the file + rewind(f); + FrameDataHeader_t patternHeader; + std::cout << "Parsing the Pattern Size from the first data Record...." << std::endl; + // Read the first pattern header which will give us the height & width of the actual pattern data. + size_t nRead = fread(&patternHeader, 1, 25, f); + if(nRead != 25) + { + std::cout << "Could not read the Frame Data Header values. Only " << nRead << " values were parsed" << std::endl; + fclose(f); + return -15; + } + + std::cout << "Pattern size is W=" << patternHeader.width << "\tH=" << patternHeader.height << "\tBytes_Per_Pixel=" << patternHeader.bytesPerPixel << std::endl; + // Put the file pointer back to the start of the file + rewind(f); + + int32_t patternDataTupleCount = patternHeader.width * patternHeader.height; + // Allocate a row's worth of memory for the pattern data to be read into + std::vector patternData(mapWidth * patternHeader.width * patternHeader.height); + + // =================================================== + int32_t patternRank = 3; + std::array dims = {static_cast(mapWidth), static_cast(ebspHeight), static_cast(ebspWidth)}; + std::array maxdims = {static_cast(mapWidth * mapHeight), static_cast(ebspHeight), static_cast(ebspWidth)}; + hid_t dataspace = H5Screate_simple(patternRank, dims.data(), maxdims.data()); + + // Modify dataset creation properties, i.e. enable chunking. + std::array chunk_dims = {static_cast(mapWidth), static_cast(ebspHeight), static_cast(ebspWidth)}; + hid_t cparms = H5Pcreate(H5P_DATASET_CREATE); + herr_t status = H5Pset_chunk(cparms, patternRank, chunk_dims.data()); + T fillvalue = 0; + status = H5Pset_fill_value(cparms, native_type, &fillvalue); + + // Create a new dataset within the file using cparms creation properties. + hid_t dataset = H5Dcreate2(dataGrpId, Bruker::IndexingResults::EBSP.toStdString().c_str(), native_type, dataspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + hid_t filespace = 0; + + size_t beamIdx = 0; + for(int32_t y = 0; y < mapHeight; y++) + { + std::cout << dataFileInfo.fileName().toStdString() << " Writing Row " << y << "/" << mapHeight << "\r"; + std::cout.flush(); + + for(int32_t x = 0; x < mapWidth; x++) + { + fpos_t pos; + size_t patternDataPtrOffset = x * ebspWidth * ebspHeight; + uint64_t filePos = frameDescription[beamIdx++]; // Get the file position of the pattern + if(filePos != 0xFFFFFFFFFFFFFFFF) + { + fseek(f, filePos + 25, SEEK_SET); // Set the file position to the pattern data + // Compute the index into the current pattern vector to store the pattern + // Read the Data associated with the header that was read. + fgetpos(f, &pos); + + if(flipPatterns) + { + std::vector tiffPatternData(patternDataTupleCount, 0); + nRead = fread(tiffPatternData.data(), sizeof(T), patternDataTupleCount, f); + size_t targetIndex = 0; + T* targetPattern = patternData.data() + patternDataPtrOffset; + for(int h = patternHeader.height -1; h >= 0; h--) + { + size_t pIndex = h * patternHeader.width; + T* sourcePatternRowPtr = &tiffPatternData[pIndex]; + T* targetPatternRowPtr = &targetPattern[targetIndex]; + ::memcpy(targetPatternRowPtr, sourcePatternRowPtr, sizeof(T) * patternHeader.width); + targetIndex += patternHeader.width; + } +#if 0 +SERIOUSLY: DO NOT UNCOMMENT THIS UNLESS YOU WANT TO KILL YOUR COMPUTER + std::stringstream ss; + ss << "/tmp/pattern_" << x << "_" << y << ".tiff"; + std::pair result = ::WriteGrayScaleImage(ss.str(), patternHeader.width, patternHeader.height, patternData.data() + patternDataPtrOffset); + if(result.first < 0) + { + std::cout << result.second << std::endl; + } +#endif + } + else + { + nRead = fread(patternData.data() + patternDataPtrOffset, sizeof(T), patternDataTupleCount, f); + } +#if 0 +// This section is for writing patterns to a tiff file. ONLY DO THIS IF YOU ARE IN +// A DEBUGGER STEPPING THROUGH THE CODE. Dumping a few hundred thousand files onto +// your desktop is not going to end well for ANY operating system, yes, Linux included. + { + fseek(f, filePos + 25, SEEK_SET); // Set the file position to the pattern data + // Compute the index into the current pattern vector to store the pattern + // Read the Data associated with the header that was read. + fgetpos(f, &pos); + std::stringstream ss; + ss << "/tmp/pattern_" << x << "_" << y << ".tiff"; + std::vector tiffPatternData(patternDataTupleCount, 0); + nRead = fread(tiffPatternData.data(), sizeof(T), patternDataTupleCount, f); + + std::pair result = ::WriteGrayScaleImage(ss.str(), patternHeader.width, patternHeader.height, tiffPatternData.data()); + if(result.first < 0) + { + std::cout << result.second << std::endl; + } + } +#endif + } + else + { + // Write ZEROS to the pattern data + std::memset(patternData.data() + patternDataPtrOffset, 0x00, patternDataTupleCount); + } + + + if(feof(f) != 0) + { + std::cout << "Unexpected End of File (EOF) was encountered. Details follow" << std::endl; + std::cout << "File Size: " << filesize << std::endl; + std::cout << "ferror(f) = " << ferror(f) << " feof(f) = " << feof(f) << std::endl; + // std::cout << "File Pos When Reading: " << static_cast(pos) << std::endl; + printf("File Pos When Reading: %llu\n", static_cast(BDL_POS(pos))); + fgetpos(f, &pos); + std::cout << "error reading data file: nRead=" << nRead << " but needed: " << patternDataTupleCount << std::endl; + printf(" Current File Position: %llu\n", static_cast(BDL_POS(pos))); + std::cout << "X,Y Position from Pattern Header: " << patternHeader.xIndex << " , " << patternHeader.yIndex << std::endl; + y = mapHeight; + x = mapWidth; + break; + } + } + + // Extend the dataset. + std::array size = {static_cast(mapWidth * (y + 1)), static_cast(ebspHeight), static_cast(ebspWidth)}; + status = H5Dset_extent(dataset, size.data()); + + // Select a hyperslab. + std::array offset = {static_cast(mapWidth * y), 0, 0}; + filespace = H5Dget_space(dataset); + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset.data(), nullptr, dims.data(), nullptr); + + // Define memory space + dataspace = H5Screate_simple(patternRank, dims.data(), nullptr); + + // Write the data to the hyperslab. + status = H5Dwrite(dataset, native_type, dataspace, filespace, H5P_DEFAULT, patternData.data()); + } + // Close/release resources. + H5Dclose(dataset); + H5Sclose(dataspace); + H5Sclose(filespace); + H5Pclose(cparms); + // Close our FrameData File + fclose(f); + + std::cout << std::endl; + std::cout.flush(); + return 0; +} + +// ----------------------------------------------------------------------------- +void BcfHdf5Convertor::execute() +{ + const bool k_ShowHdf5Errors = true; + + // We are going to construct a QTemporaryDir _templatepath_ variable so we use a temp + // location next to the input file. This *should* be ok for most situations. + // Qt will clean up the temp dir when it goes out of scope. + QFileInfo ifInfo(m_InputFile); + QString tmpPath = ifInfo.absolutePath() + "/" + ifInfo.baseName() + "_XXXXXX"; + QTemporaryDir tempDir(tmpPath); + if(!tempDir.isValid()) + { + m_ErrorCode = -7000; + m_ErrorMessage = QString("Temp Directory could not be created."); + return; + } + + int32_t err = 0; + + hid_t fid = H5Utilities::createFile(m_OutputFile.toStdString()); + H5ScopedFileSentinel fileSentinel(fid, k_ShowHdf5Errors); + + // *************************************************************************** + // IF ANYTHING IN HERE CHANGES YOU NEED TO INCREMENT THE FILEVERSION NUMBER + // WHICH INDICATES THAT THE ORGANIZATION HAS BEED APPENDED/EDITED/REVISED. + // *************************************************************************** + err = H5Lite::writeScalarAttribute(fid, "/", "FileVersion", ::k_FileVersion); + + QFileInfo fi(m_InputFile); + QString baseInputFileName = fi.completeBaseName(); + + hid_t topGrpId = H5Utilities::createGroup(fid, baseInputFileName.toStdString()); + fileSentinel.addGroupId(topGrpId); + + hid_t ebsdGrpId = H5Utilities::createGroup(topGrpId, k_EBSD); + fileSentinel.addGroupId(ebsdGrpId); + + hid_t dataGrpId = H5Utilities::createGroup(ebsdGrpId, k_Data); + fileSentinel.addGroupId(dataGrpId); + + hid_t semGrpId = H5Utilities::createGroup(topGrpId, k_SEM); + fileSentinel.addGroupId(semGrpId); + + hid_t headerGrpId = H5Utilities::createGroup(ebsdGrpId, k_Header); + fileSentinel.addGroupId(headerGrpId); + + std::string manufacturer("DREAM.3D"); + err = H5Lite::writeStringDataset(fid, ::k_Manufacturer, manufacturer); + if(err < 0) + { + m_ErrorCode = err; + m_ErrorMessage = QString("Manufacturer Dataset was not written"); + return; + } + + std::string version = "0.2.0"; + err = H5Lite::writeStringDataset(fid, ::k_Version, version); + if(err < 0) + { + m_ErrorCode = err; + m_ErrorMessage = QString("Version Dataset was not written"); + return; + } + + SFSReader sfsFile; + sfsFile.parseFile(m_InputFile.toStdString()); + QString outputFile; + QTextStream outFileStrm(&outputFile); + + std::cout << "Using Temp Dir: " << tempDir.path().toStdString() << std::endl; + + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::FrameDescription; + QString descFile = outputFile; + // std::cout << "Extracting FrameDescription"; + err = sfsFile.extractFile(tempDir.path().toStdString(), descFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7020; + m_ErrorMessage = QString("Could not extract EBSDData/FrameDescription File."); + return; + } + + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::IndexingResults; + QString indexingResultsFile = outputFile; + // std::cout << "Extracting IndexingResults"; + err = sfsFile.extractFile(tempDir.path().toStdString(), indexingResultsFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7030; + m_ErrorMessage = QString("Could not extract EBSDData/IndexingResults File."); + return; + } + + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::Auxiliarien; + QString auxiliarienFile = outputFile; + // std::cout << "Extracting Auxiliarien"; + err = sfsFile.extractFile(tempDir.path().toStdString(), auxiliarienFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7040; + m_ErrorMessage = QString("Could not extract EBSDData/Auxiliarien File."); + return; + } + + outputFile.clear(); + outFileStrm << tempDir.path() << "/" << Bruker::Files::EBSDData; + QString ebsdDataDir = outputFile; + + // Read the Dimensions of the Map and EBSP + int32_t mapHeight; + int32_t mapWidth; + int32_t ebspHeight; + int32_t ebspWidth; + err = BrukerDataLoader::ReadScanSizes(ebsdDataDir, mapWidth, mapHeight, ebspWidth, ebspHeight); + if(err < 0) + { + std::cout << "Error reading Scan Sizes from Description file: " << err << std::endl; + m_ErrorCode = -7050; + return; + } + auto numElements = static_cast(mapWidth * mapHeight); + std::vector cDims = {2}; + std::vector tDims = {numElements}; + + std::vector roi; + + // Scope this entire next part so that the arrays get cleaned up when we are done.... + { + UInt16ArrayType::Pointer indices = UInt16ArrayType::CreateArray(numElements, cDims, "Positions", true); + cDims[0] = 3; + FloatArrayType::Pointer eulers = FloatArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::Eulers, true); + + cDims[0] = 1; + FloatArrayType::Pointer patQual = FloatArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::RadonQuality, true); + UInt16ArrayType::Pointer detectedBands = UInt16ArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::RadonBandCount, true); + Int16ArrayType::Pointer phases = Int16ArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::Phase, true); + UInt16ArrayType::Pointer indexedBands = UInt16ArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::NIndexedBands, true); + FloatArrayType::Pointer bmm = FloatArrayType::CreateArray(numElements, cDims, Bruker::IndexingResults::MAD, true); + + descFile = tempDir.path() + "/" + Bruker::Files::EBSDData + "/" + Bruker::Files::FrameDescription; + indexingResultsFile = tempDir.path() + "/" + Bruker::Files::EBSDData + "/" + Bruker::Files::IndexingResults; + + err = BrukerDataLoader::LoadIndexingResults(descFile, indexingResultsFile, indices, eulers, patQual, detectedBands, phases, indexedBands, bmm, mapWidth, mapHeight, roi, m_Reorder); + if(err < 0) + { + m_ErrorCode = -7050; + m_ErrorMessage = QString("Error Reading IndexingResults from extracted file: "); + return; + } + + // Write all the data to the HDF5 file with conversions + Int32ArrayType::Pointer i32Array = Int32ArrayType::CreateArray(numElements, Bruker::IndexingResults::XBEAM, true); + for(size_t i = 0; i < numElements; i++) + { + auto value = static_cast(indices->getComponent(i, 0)); + i32Array->setValue(i, value); + } + err = writeDataArrayToHdf5(i32Array.get(), dataGrpId, numElements); + i32Array->setName(Bruker::SEM::SEMIX); + err = writeDataArrayToHdf5(i32Array.get(), semGrpId, numElements); + + i32Array->setName(Bruker::IndexingResults::YBEAM); + for(size_t i = 0; i < numElements; i++) + { + auto value = static_cast(indices->getComponent(i, 1)); + i32Array->setValue(i, value); + } + err = writeDataArrayToHdf5(i32Array.get(), dataGrpId, numElements); + i32Array->setName(Bruker::SEM::SEMIY); + err = writeDataArrayToHdf5(i32Array.get(), semGrpId, numElements); + + // Convert from a single array of Eulers to 3 separate arrays and convert to degrees .. why? + std::vector names = {{Bruker::IndexingResults::phi1, Bruker::IndexingResults::PHI, Bruker::IndexingResults::phi2}}; + std::vector comp = {{0, 1, 2}}; + FloatArrayType::Pointer euler = FloatArrayType::CreateArray(numElements, Bruker::IndexingResults::phi1, true); + for(size_t c = 0; c < 3; c++) + { + euler->setName(names.at(c)); + for(size_t i = 0; i < numElements; i++) + { + float value = eulers->getComponent(i, c) * SIMPLib::Constants::k_180OverPiD; + euler->setValue(i, value); + } + err = writeDataArrayToHdf5(euler.get(), dataGrpId, numElements); + } + + // This one doesn't need a conversion. + err = writeDataArrayToHdf5(patQual.get(), dataGrpId, numElements); + + // Convert RadonBandCount + i32Array->setName(Bruker::IndexingResults::RadonBandCount); + for(size_t i = 0; i < numElements; i++) + { + auto value = static_cast(detectedBands->getValue(i)); + i32Array->setValue(i, value); + } + err = writeDataArrayToHdf5(i32Array.get(), dataGrpId, numElements); + + // Convert Phase + i32Array->setName(Bruker::IndexingResults::Phase); + for(size_t i = 0; i < numElements; i++) + { + auto value = static_cast(phases->getValue(i)); + i32Array->setValue(i, value); + } + err = writeDataArrayToHdf5(i32Array.get(), dataGrpId, numElements); + + // Convert NIndexedBands + i32Array->setName(Bruker::IndexingResults::NIndexedBands); + for(size_t i = 0; i < numElements; i++) + { + auto value = static_cast(indexedBands->getValue(i)); + i32Array->setValue(i, value); + } + err = writeDataArrayToHdf5(i32Array.get(), dataGrpId, numElements); + + // MAD doesn't need a conversion + err = writeDataArrayToHdf5(bmm.get(), dataGrpId, numElements); + } + + // Write all the Header information + { + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::PhaseList; + QString phaseListFile = outputFile; + // std::cout << "Extracting PhaseList"; + err = sfsFile.extractFile(tempDir.path().toStdString(), phaseListFile.toStdString()); + phaseListFile = tempDir.path() + "/" + phaseListFile; + if(err == 0) + { + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::NCOLS.toStdString(), mapWidth); + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::NROWS.toStdString(), mapHeight); + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::NPoints.toStdString(), numElements); + err = H5Lite::writeStringDataset(headerGrpId, Bruker::Header::OriginalFile.toStdString(), m_InputFile.toStdString()); + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::PatternWidth.toStdString(), ebspWidth); + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::PatternHeight.toStdString(), ebspHeight); + err = H5Lite::writeStringDataset(headerGrpId, Bruker::Header::GridType.toStdString(), Bruker::Header::isometric.toStdString()); + double zOffset = 0.0; + err = H5Lite::writeScalarDataset(headerGrpId, Bruker::Header::ZOffset.toStdString(), zOffset); + writePhaseInformation(headerGrpId, phaseListFile); + } + } + + // Write the SEM Data + { + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::SEMImage; + QString semFile = outputFile; + // std::cout << "Extracting SEIMImage"; + err = sfsFile.extractFile(tempDir.path().toStdString(), semFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7060; + m_ErrorMessage = QString("Could not extract EBSDData/SEMImage File."); + return; + } + semFile = tempDir.path() + "/" + semFile; + + writeSEMData(semGrpId, headerGrpId, semFile); + } + + // Write the Calibration Data + { + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::Calibration; + QString semFile = outputFile; + // std::cout << "Extracting Calibration"; + err = sfsFile.extractFile(tempDir.path().toStdString(), semFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7060; + m_ErrorMessage = QString("Could not extract EBSDData/Calibration File."); + return; + } + semFile = tempDir.path() + "/" + semFile; + + float pcx = 0.0f; + float pcy = 0.0f; + writeCalibrationData(semGrpId, headerGrpId, semFile, pcx, pcy); + + std::vector dims = { static_cast (mapHeight * mapWidth)}; + { + std::vector patternCenter(mapWidth * mapHeight, pcx); + err = H5Lite::writeVectorDataset(dataGrpId, "PCX", dims, patternCenter); + } + { + std::vector patternCenter(mapWidth * mapHeight, pcy); + err = H5Lite::writeVectorDataset(dataGrpId, "PCY", dims, patternCenter); + } + + } + + // Write the AuxIndexingOptions Data + { + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::AuxIndexingOptions; + QString semFile = outputFile; + // std::cout << "Extracting AuxIndexingOptions"; + err = sfsFile.extractFile(tempDir.path().toStdString(), semFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7060; + m_ErrorMessage = QString("Could not extract EBSDData/AuxIndexingOptions File."); + return; + } + semFile = tempDir.path() + "/" + semFile; + + writeAuxIndexingOptions(semGrpId, headerGrpId, semFile); + } + + int32_t pixelByteCount = 0; + // Write the CameraConfiguration Data + { + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::CameraConfiguration; + QString semFile = outputFile; + // std::cout << "Extracting CameraConfiguration"; + err = sfsFile.extractFile(tempDir.path().toStdString(), semFile.toStdString()); + if(err < 0) + { + m_ErrorCode = -7060; + m_ErrorMessage = QString("Could not extract EBSDData/CameraConfiguration File."); + return; + } + semFile = tempDir.path() + "/" + semFile; + + writeCameraConfiguration(semGrpId, headerGrpId, semFile); + // Get the Pattern Pixel Byte Count + err = H5Lite::readScalarDataset(headerGrpId, "PixelByteCount", pixelByteCount); + } + + outputFile.clear(); + outFileStrm << Bruker::Files::EBSDData << "/" << Bruker::Files::FrameData; + QString dataFile = outputFile; + if(pixelByteCount == 1) + { + writePatternData(sfsFile, H5T_NATIVE_UINT8, mapWidth, mapHeight, ebspWidth, ebspHeight, m_FlipPatterns, tempDir.path(), dataFile, descFile, dataGrpId); + } + else if(pixelByteCount == 2) + { + writePatternData(sfsFile, H5T_NATIVE_UINT16, mapWidth, mapHeight, ebspWidth, ebspHeight, m_FlipPatterns, tempDir.path(), dataFile, descFile, dataGrpId); + } +} + +// ----------------------------------------------------------------------------- +int32_t BcfHdf5Convertor::getErrorCode() const +{ + return m_ErrorCode; +} + +// ----------------------------------------------------------------------------- +QString BcfHdf5Convertor::getErrorMessage() const +{ + return m_ErrorMessage; +} diff --git a/src/BcfHdf5Convertor.h b/src/BcfHdf5Convertor.h new file mode 100644 index 0000000..a341960 --- /dev/null +++ b/src/BcfHdf5Convertor.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +class BcfHdf5Convertor +{ +public: + BcfHdf5Convertor(QString inputFile, QString outputFile); + ~BcfHdf5Convertor(); + + BcfHdf5Convertor(const BcfHdf5Convertor&) = delete; // Copy Constructor Not Implemented + BcfHdf5Convertor(BcfHdf5Convertor&&) = delete; // Move Constructor Not Implemented + BcfHdf5Convertor& operator=(const BcfHdf5Convertor&) = delete; // Copy Assignment Not Implemented + BcfHdf5Convertor& operator=(BcfHdf5Convertor&&) = delete; // Move Assignment Not Implemented + + void setReorder(bool reorder); + void setFlipPatterns(bool flipPatterns); + void execute(); + + int32_t getErrorCode() const; + QString getErrorMessage() const; + +private: + QString m_InputFile; + QString m_OutputFile; + + QString m_ErrorMessage = QString("No Error"); + int32_t m_ErrorCode = 0; + bool m_Reorder = false; + bool m_FlipPatterns = false; +}; diff --git a/src/SFSNodeItem.cpp b/src/SFSNodeItem.cpp new file mode 100644 index 0000000..bf2dbfd --- /dev/null +++ b/src/SFSNodeItem.cpp @@ -0,0 +1,498 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#include "SFSNodeItem.h" + +#if(_MSC_VER >= 1) +#pragma warning(error : 4715) /* Not all control points return a value */ +#pragma warning(error : 4258) /* Nested Variable Definitions are ignored */ + //#ifdef SIMPLib_DISABLE_MSVC_WARNINGS +#pragma warning(disable : 4244) +#pragma warning(disable : 4267) +#pragma warning(disable : 4305) +//#endif +#endif + +#include +#include +#include +#include + +#include "SFSReader.h" +#include "SFSUtils.hpp" + +// ----------------------------------------------------------------------------- +SFSNodeItem::SFSNodeItem(const uint8_t* ptr, FILE* fin, SFSReader* reader) +: m_Reader(reader) +{ + ::memcpy(&m_PointerTableInit, ptr, 4); + ::memcpy(&m_FileSize, ptr + 4, 8); + ::memcpy(&m_FileCreationTime, ptr + 12, 8); + ::memcpy(&m_FileModificationTime, ptr + 20, 8); + ::memcpy(&m_LastAccessTime, ptr + 28, 8); + ::memcpy(&m_Permissions, ptr + 36, 4); + ::memcpy(&m_ParentItemIndex, ptr + 40, 4); + + m_String176 = std::string(reinterpret_cast(ptr + 44), 176); + ::memcpy(&m_Directory, ptr + 220, 1); + m_String3 = std::string(reinterpret_cast(ptr + 221)); + m_FileName = std::string(reinterpret_cast(ptr + 224)); + m_String32 = std::string(reinterpret_cast(ptr + 480)); + + m_ChunkCount = getPointerTableEntryCount(); + + if(!m_Directory) + { + generateFilePointerTable(fin); + } + m_IsValid = true; +} + +// ----------------------------------------------------------------------------- +SFSNodeItem::~SFSNodeItem() = default; + +// ----------------------------------------------------------------------------- +int32_t SFSNodeItem::getPointerTableEntryCount() const +{ + int32_t chunkCount = static_cast(std::ceil(static_cast(m_FileSize) / static_cast(m_Reader->getUsableChunkSize()))); + return chunkCount; +} + +#if 0 +namespace neolib +{ +template +inline void hex_dump(const void* aData, std::size_t aLength, std::basic_ostream& aStream, std::size_t aWidth = 16) +{ + const char* const start = static_cast(aData); + const char* const end = start + aLength; + const char* line = start; + while(line != end) + { + aStream.width(4); + aStream.fill('0'); + aStream << std::hex << line - start << " : "; + std::size_t lineLength = std::min(aWidth, static_cast(end - line)); + for(std::size_t pass = 1; pass <= 2; ++pass) + { + for(const char* next = line; next != end && next != line + aWidth; ++next) + { + char ch = *next; + switch(pass) + { + case 1: + aStream << (ch < 32 ? '.' : ch); + break; + case 2: + if(next != line) + aStream << " "; + aStream.width(2); + aStream.fill('0'); + aStream << std::hex << std::uppercase << static_cast(static_cast(ch)); + break; + } + } + if(pass == 1 && lineLength != aWidth) + aStream << std::string(aWidth - lineLength, ' '); + aStream << " "; + } + aStream << std::endl; + line = line + lineLength; + aStream << std::dec; + } +} +} // namespace neolib +#endif + +// ----------------------------------------------------------------------------- +void SFSNodeItem::generateFilePointerTable(FILE* fin) +{ + // bool debug = false; + // m_FileName == "FrameData"; + // if(debug) + // std::cout << "=============================================\n " << m_FileName << std::endl; + + size_t readerChunkSize = static_cast(m_Reader->getChunkSize()); + size_t readerUsableChunkSize = static_cast(m_Reader->getUsableChunkSize()); + + int32_t err = 0; + float denom = std::floor(readerUsableChunkSize / 4.0f); + float beforeCeil = m_ChunkCount / denom; + auto chunkCount = static_cast(::ceil(beforeCeil)); + std::vector buffer(chunkCount * readerUsableChunkSize, 0x00); // Allocate a complete buffer to read the table + uint8_t* bufferPtr = buffer.data(); + auto ui32Ptr = reinterpret_cast(buffer.data()); + size_t totalBytesRead = 0; + if(chunkCount > 1) + { + uint32_t nextChunk = m_PointerTableInit; + for(int i = 0; i < chunkCount; i++) + { + size_t offset = readerChunkSize * nextChunk + 280; + SFS_UTIL_FSEEK(fin, offset, SEEK_SET); + nextChunk = SFSUtils::readScalar(fin, err); + SFS_UTIL_FSEEK(fin, 28, SEEK_CUR); + + if(fread(bufferPtr, 1, readerUsableChunkSize, fin) != readerUsableChunkSize) + { + m_IsValid = false; + return; + } + bufferPtr += readerUsableChunkSize - 1; + totalBytesRead += readerUsableChunkSize; + // if(debug) + // { + // std::cout << " " << i << " | File Pos Start: " << offset + 32 << "\t" + // << "File Pos End: " << offset + 32 + readerUsableChunkSize << "\t" + // << "Num. UInt32s: " << totalBytesRead / 4 << std::endl; + + // if(totalBytesRead / 4 > 230687) + // { + // std::cout << ui32Ptr[230687] * readerChunkSize + 312ULL << "\t" << ui32Ptr[230688] * readerChunkSize + 312ULL << std::endl; + + // neolib::hex_dump(ui32Ptr + 230687, 8, std::cout); + // } + // } + } + } + else + { + size_t offset = readerChunkSize * m_PointerTableInit + 312ULL; + SFS_UTIL_FSEEK(fin, offset, SEEK_SET); + if(fread(bufferPtr, 1, readerUsableChunkSize, fin) != readerUsableChunkSize) + { + m_IsValid = false; + return; + } + } + bufferPtr = buffer.data(); + m_FilePointerTable.resize(m_ChunkCount); + // size_t maxIndex = buffer.size() / 4; + // if(maxIndex < m_ChunkCount) + // { + // std::cout << "We will walk off the end of the array." << std::endl; + // } + // bool messedUp = false; + for(size_t i = 0; i < m_ChunkCount; i++) + { + + auto value = static_cast(ui32Ptr[i]); + m_FilePointerTable[i] = value * static_cast(readerChunkSize) + 312ULL; + // if(m_FilePointerTable[i] > 18661284985 && !messedUp) + // { + // std::cout << "===========================" << std::endl; + // messedUp = true; + // } + // if(debug) + // { + // std::cout << " m_FilePointerTable[" << i << "] = " << m_FilePointerTable[i] << std::endl; + // } + } +} + +// ----------------------------------------------------------------------------- +std::vector SFSNodeItem::extractFile() const +{ + std::vector data(m_FileSize, 0); + if(m_FileSize == 0) + { + return data; + } + if(m_Directory) + { + return data; + } + const std::string& inputFilePath = m_Reader->getInputFile(); + FILE* fn = fopen(inputFilePath.c_str(), "rb"); + if(nullptr == fn) + { + return data; + } + + if(m_ChunkCount == 1) + { + SFS_UTIL_FSEEK(fn, m_FilePointerTable[0], SEEK_SET); + if(fread(data.data(), 1, m_FileSize, fn) != m_FileSize) + { + return data; + } + } + else + { + size_t dataSize = 0; + size_t chunkSize = static_cast(m_Reader->getUsableChunkSize()); + uint8_t* destPtr = data.data(); + for(size_t i = 0; i < m_ChunkCount; i++) + { + SFS_UTIL_FSEEK(fn, m_FilePointerTable[i], SEEK_SET); + if(fread(destPtr, 1, chunkSize, fn) != chunkSize) + { + data.assign(m_FileSize, 0); + return data; + } + destPtr += chunkSize; + dataSize += chunkSize; + if(m_FileSize - dataSize < chunkSize) + { + chunkSize = m_FileSize - dataSize; + } + } + } + fclose(fn); + return data; +} + +// ----------------------------------------------------------------------------- +int32_t SFSNodeItem::writeFile(const std::string& outputfile) const +{ + if(m_FileSize == 0) + { + return -1; + } + if(m_Directory) + { + return -2; + } + + FILE* out = fopen(outputfile.c_str(), "wb"); + if(out == nullptr) + { + return -3; + } + + const std::string& inputFilePath = m_Reader->getInputFile(); + FILE* fn = fopen(inputFilePath.c_str(), "rb"); + if(nullptr == fn) + { + fclose(out); + return -4; + } + if(m_ChunkCount == 1) + { + std::vector data(m_FileSize, 0); + SFS_UTIL_FSEEK(fn, m_FilePointerTable[0], SEEK_SET); + if(fread(data.data(), 1, m_FileSize, fn) != m_FileSize) + { + fclose(out); + return -5; + } + fwrite(data.data(), m_FileSize, 1, out); + std::cout << m_FileName << std::endl; + } + else + { + uint64_t dataSize = 0; + uint64_t chunkSize = static_cast(m_Reader->getUsableChunkSize()); + std::vector data(chunkSize, 0); + uint8_t* destPtr = data.data(); + + float progress = 0.0f; + float currentProgress = 0.0f; + + // std::cout << m_FileName << std::endl; + for(size_t i = 0; i < m_ChunkCount; i++) + { + currentProgress = static_cast(i) / static_cast(m_ChunkCount); + if(currentProgress > progress) + { + progress = progress + 0.01f; + std::cout << m_FileName << " " << m_FileSize << " [" << static_cast(currentProgress * 100.0f) << "%]\r"; + std::cout.flush(); + } + size_t filePointer = m_FilePointerTable[i]; + SFS_UTIL_FSEEK(fn, filePointer, SEEK_SET); + size_t numBytesRead = fread(destPtr, 1, chunkSize, fn); + if(numBytesRead != chunkSize) + { + std::cout << "Not Enough Bytes Read: " << m_FilePointerTable[i] << " Needed " << chunkSize << " Got " << numBytesRead << std::endl; + } + fwrite(destPtr, chunkSize, 1, out); + dataSize += chunkSize; + if(m_FileSize - dataSize < chunkSize) + { + chunkSize = m_FileSize - dataSize; + } + } + std::cout << std::endl; + } + + fclose(out); + fclose(fn); + return 0; +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::setParentNode(SFSNodeItem* parent) +{ + m_ParentObject = parent; +} + +// ----------------------------------------------------------------------------- +SFSNodeItem* SFSNodeItem::getParentNode() +{ + return m_ParentObject; +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::addChildNode(const SFSNodeItem::Pointer& child) +{ + m_Children[child->getFileName()] = child; + child->setParentNode(this); +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::removeChildNode(const std::string& name) +{ + if(m_Children.find(name) != m_Children.end()) + { + SFSNodeItem::Pointer item = m_Children[name]; + item->setParentNode(nullptr); + m_Children.erase(name); + } +} + +// ----------------------------------------------------------------------------- +size_t SFSNodeItem::childCount() +{ + return m_Children.size(); +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::clearChildren() +{ + + for(const auto& child : m_Children) + { + child.second->setParentNode(nullptr); + } + + m_Children.clear(); +} + +// ----------------------------------------------------------------------------- +SFSNodeItem::Pointer SFSNodeItem::child(const std::string& name) +{ + if(m_Children.find(name) != m_Children.end()) + { + return m_Children[name]; + } + return SFSNodeItem::Pointer(nullptr); +} + +// ----------------------------------------------------------------------------- +std::map& SFSNodeItem::children() +{ + return m_Children; +} + +// ----------------------------------------------------------------------------- +int32_t SFSNodeItem::getPointerTableInit() const +{ + return m_PointerTableInit; +} + +// ----------------------------------------------------------------------------- +uint64_t SFSNodeItem::getFileSize() const +{ + return m_FileSize; +} + +// ----------------------------------------------------------------------------- +uint64_t SFSNodeItem::getFileCreationTime() const +{ + return m_FileCreationTime; +} + +// ----------------------------------------------------------------------------- +uint64_t SFSNodeItem::getFileModificationTime() const +{ + return m_FileModificationTime; +} + +// ----------------------------------------------------------------------------- +uint64_t SFSNodeItem::getFileLastAccessTime() const +{ + return m_LastAccessTime; +} + +// ----------------------------------------------------------------------------- +uint32_t SFSNodeItem::getPermissions() const +{ + return m_Permissions; +} + +// ----------------------------------------------------------------------------- +int32_t SFSNodeItem::getParentItemIndex() const +{ + return m_ParentItemIndex; +} + +// ----------------------------------------------------------------------------- +bool SFSNodeItem::isDirectory() const +{ + return m_Directory; +} + +// ----------------------------------------------------------------------------- +std::string SFSNodeItem::getFileName() const +{ + return m_FileName; +} + +bool SFSNodeItem::getIsValid() const +{ + return m_IsValid; +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::debug(std::ostream& out) const +{ + out << "--------------------------------------------------------------------" << std::endl; + out << "m_FileName: " << m_FileName << std::endl; + out << "m_PointerTableInit: " << m_PointerTableInit << std::endl; + out << "m_FileSize: " << m_FileSize << std::endl; + out << "m_ChunkCount: " << m_ChunkCount << std::endl; + out << "m_FileCreationTime: " << m_FileCreationTime << std::endl; + out << "m_FileModificationTime: " << m_FileModificationTime << std::endl; + out << "m_LastAccessTime: " << m_LastAccessTime << std::endl; + out << "m_Permissions: " << m_Permissions << std::endl; + out << "m_ParentItemIndex: " << m_ParentItemIndex << std::endl; + out << "m_String176: " << m_String176 << std::endl; + out << "m_Directory: " << m_Directory << std::endl; + out << "m_String3: " << m_String3 << std::endl; + out << "m_String32: " << m_String32 << std::endl; +} + +// ----------------------------------------------------------------------------- +void SFSNodeItem::printTree(std::ostream& out, int32_t level) const +{ + std::string indent(level, ' '); + out << indent << m_FileName << std::endl; + for(const auto& item : m_Children) + { + if(item.second->isDirectory()) + { + item.second->printTree(out, level + 2); + } + else + { + indent = std::string(level + 4, ' '); + out << indent << item.second->getFileName() << ": " << item.second->getFileSize() << std::endl; + } + } +} diff --git a/src/SFSNodeItem.h b/src/SFSNodeItem.h new file mode 100644 index 0000000..f5a678f --- /dev/null +++ b/src/SFSNodeItem.h @@ -0,0 +1,216 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#pragma once + +#include +#include +#include +#include +#include +#include + +class SFSReader; + +/** + * @brief The SFSNodeItem class holds all the information for an individual file within the SFS file + */ +class SFSNodeItem +{ + +public: + SFSNodeItem() = default; + explicit SFSNodeItem(const uint8_t* raw_string, FILE* fin, SFSReader* reader); + ~SFSNodeItem(); + + /** + * + */ + using Pointer = std::shared_ptr; + + + SFSNodeItem(const SFSNodeItem&) = default; // Copy Constructor + SFSNodeItem(SFSNodeItem&&) = default; // Move Constructor + SFSNodeItem& operator=(const SFSNodeItem&) = default; // Copy Assignment Not Implemented + SFSNodeItem& operator=(SFSNodeItem&&) = default; // Move Assignment Not Implemented + + /** + * @brief getPointerTableInit + * @return + */ + int32_t getPointerTableInit() const; + + /** + * @brief getFileSize + * @return + */ + uint64_t getFileSize() const; + + /** + * @brief getFileCreationTime + * @return + */ + uint64_t getFileCreationTime() const; + + /** + * @brief getFileModificationTime + * @return + */ + uint64_t getFileModificationTime() const; + + /** + * @brief getFileLastAccessTime + * @return + */ + uint64_t getFileLastAccessTime() const; + + /** + * @brief getPermissions + * @return + */ + uint32_t getPermissions() const; + + /** + * @brief getParentItemIndex + * @return + */ + int32_t getParentItemIndex() const; + + /** + * @brief isDirectory + * @return + */ + bool isDirectory() const; + + /** + * @brief getFileName + * @return + */ + std::string getFileName() const; + + bool getIsValid() const; + + /** + * @brief extractFile + * @return + */ + std::vector extractFile() const; + + /** + * @brief writeFile + * @param outputfile + * @return + */ + int32_t writeFile(const std::string& outputfile) const; + + /** + * @brief debug + * @param out + */ + void debug(std::ostream& out) const; + + /** + * @brief printTree + * @param out + */ + void printTree(std::ostream& out, int32_t level) const; + + /** + * @brief setParentNode + * @param parent + */ + void setParentNode(SFSNodeItem* parent); + + /** + * @brief getParentNode + * @return + */ + SFSNodeItem* getParentNode(); + + /** + * @brief addChildNode + * @param child + */ + void addChildNode(const Pointer &child); + + /** + * @brief removeChildNode + * @param name + */ + void removeChildNode(const std::string& name); + + /** + * @brief childCount + * @return + */ + size_t childCount(); + + /** + * @brief clearChildren + */ + void clearChildren(); + + /** + * @brief child + * @param name + * @return + */ + Pointer child(const std::string& name); + + /** + * @brief children + * @return + */ + std::map& children(); + + +protected: + /** + * @brief getPointerTableEntryCount + * @return + */ + int32_t getPointerTableEntryCount() const; + + /** + * @brief generateFilePointerTable + */ + void generateFilePointerTable(FILE* fin); + +private: + bool m_IsValid = false; + int32_t m_PointerTableInit = 0; + uint64_t m_FileSize = 0; + uint64_t m_FileCreationTime = 0; + uint64_t m_FileModificationTime = 0; + uint64_t m_LastAccessTime = 0; + uint32_t m_Permissions = 0; + int32_t m_ParentItemIndex = 0; + std::string m_String176; + bool m_Directory = true; + std::string m_String3; + std::string m_FileName = std::string("/"); + std::string m_String32; + + int32_t m_ChunkCount = 0; + + std::vector m_FilePointerTable; + + SFSReader* m_Reader = nullptr; + SFSNodeItem* m_ParentObject = nullptr; + std::map m_Children; +}; diff --git a/src/SFSReader.cpp b/src/SFSReader.cpp new file mode 100644 index 0000000..725b26a --- /dev/null +++ b/src/SFSReader.cpp @@ -0,0 +1,332 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#include "SFSReader.h" + +#if(_MSC_VER >= 1) +#pragma warning(error : 4715) /* Not all control points return a value */ +#pragma warning(error : 4258) /* Nested Variable Definitions are ignored */ + //#ifdef SIMPLib_DISABLE_MSVC_WARNINGS +#pragma warning(disable : 4244) +#pragma warning(disable : 4267) +#pragma warning(disable : 4305) +//#endif +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SFSNodeItem.h" +#include "SFSUtils.hpp" + +namespace BCF +{ +const char k_SFSMagic[8] = {'A', 'A', 'M', 'V', 'H', 'F', 'S', 'S'}; +} + +// ----------------------------------------------------------------------------- +SFSReader::SFSReader() = default; + +// ----------------------------------------------------------------------------- +SFSReader::~SFSReader() = default; + +// ----------------------------------------------------------------------------- +float SFSReader::getVersion() const +{ + return m_Version; +} + +// ----------------------------------------------------------------------------- +uint32_t SFSReader::getChunkSize() const +{ + return m_ChunkSize; +} + +// ----------------------------------------------------------------------------- +uint32_t SFSReader::getUsableChunkSize() const +{ + return m_UsableChunkSize; +} + +const std::string& SFSReader::getInputFile() const +{ + return m_FilePath; +} + +// ----------------------------------------------------------------------------- +SFSNodeItemPtr SFSReader::getRootNode() const +{ + return m_RootNode; +} + +// ----------------------------------------------------------------------------- +int SFSReader::parseFile(const std::string& filepath) +{ + m_FilePath = filepath; + + int32_t err = 0; + FILE* fin = fopen(m_FilePath.c_str(), "rb"); + if(nullptr == fin) + { + std::cout << "Error opening file '" << filepath << "'" << std::endl; + return -2; + } + + std::array sfsmagic = {0, 0, 0, 0, 0, 0, 0, 0}; + + size_t nread = fread(sfsmagic.data(), 1, 8, fin); + if(nread != 8) + { + fclose(fin); + return -3; + } + + for(size_t c = 0; c < 8; c++) + { + if(BCF::k_SFSMagic[c] != sfsmagic[c]) + { + fclose(fin); + return -6; + } + } + + // Seek to the version number + SFS_UTIL_FSEEK(fin, 0x124, SEEK_SET); + + m_Version = SFSUtils::readScalar(fin, err); + if(err == 1) + { + fclose(fin); + std::cout << "Error reading version" << std::endl; + return -4; + } + m_ChunkSize = SFSUtils::readScalar(fin, err); + if(err == 1) + { + fclose(fin); + std::cout << "Error reading chunkSize" << std::endl; + return -5; + } + + m_UsableChunkSize = m_ChunkSize - 32; + // std::cout << "Version: " << m_Version << std::endl; + // std::cout << "chunkSize: " << m_ChunkSize << std::endl; + // std::cout << "usableChunkSize: " << m_UsableChunkSize << std::endl; + + // Read the SFS Tree Root Structure + SFS_UTIL_FSEEK(fin, 320, SEEK_SET); + err = 0; + m_TreeAddress = SFSUtils::readScalar(fin, err); + if(err == -1) + { + m_IsValid = false; + } + m_NumTreeItems = SFSUtils::readScalar(fin, err); + if(err == -1) + { + m_IsValid = false; + } + m_NumChunks = SFSUtils::readScalar(fin, err); + if(err == -1) + { + m_IsValid = false; + } + // std::cout << "TreeAddress: " << m_TreeAddress << std::endl; + // std::cout << "m_NumTreeItems: " << m_NumTreeItems << std::endl; + // std::cout << "m_NumChunks: " << m_NumChunks << std::endl; + + // Create all the headers to convert into SFSNodeItems + int32_t fileTreeChunks = static_cast(std::ceil((m_NumTreeItems * 512.0f) / (m_ChunkSize - 32.0f))); + // std::cout << "fileTreeChunks: " << fileTreeChunks << std::endl; + std::vector rawTreeBuffer; + if(fileTreeChunks == 1) + { + // file tree does not exceed one chunk in bcf: + size_t offset = m_ChunkSize * m_TreeAddress + 312; + SFS_UTIL_FSEEK(fin, offset, SEEK_SET); + size_t rawTreeCount = 512 * m_NumTreeItems; + rawTreeBuffer.resize(rawTreeCount * sizeof(int32_t)); + + nread = fread(rawTreeBuffer.data(), 4, rawTreeCount, fin); + if(nread != rawTreeCount) + { + std::cout << "sfsReader::parseFile(" << __LINE__ << ") error reading bytes: " << ferror(fin) << std::endl; + // std::cout << "sfsReader::parseFile(" << __LINE__ << ") EOF Value: " << feof(fin) << std::endl; + perror("sfsReader::parseFile Error"); + err = -1; + } + } + else + { + uint32_t treeAddress = m_TreeAddress; + int32_t numTreeItemsInChunk = static_cast(std::floor((m_ChunkSize - 32.0f) / 512.0f)); + size_t bytesToRead = numTreeItemsInChunk * 512; + rawTreeBuffer.resize(fileTreeChunks * numTreeItemsInChunk * 512); // Allocate all the space for the header + uint8_t* rawTreeBufferPtr = rawTreeBuffer.data(); + for(int32_t i = 0; i < fileTreeChunks; i++) + { + SFS_UTIL_FSEEK(fin, m_ChunkSize * treeAddress + 280, SEEK_SET); + treeAddress = SFSUtils::readScalar(fin, err); // strct_unp(' indexNodeMap; + m_RootNode = std::make_shared(); + // SFSNodeItemPtr(new SFSNodeItem); + + // Read all the 512 byte headers for each tree item + for(uint32_t i = 0; i < m_NumTreeItems; i++) + { + // std::cout << "--------------- SFSTreeItem -------------------" << std::endl; + SFSNodeItemPtr item = std::make_shared(rawTreeBuffer.data() + (i * 512), fin, this); + indexNodeMap[i] = item; + if(item->getParentItemIndex() == -1) + { + m_RootNode->addChildNode(item); + } + else + { + SFSNodeItemPtr parent = indexNodeMap[item->getParentItemIndex()]; + parent->addChildNode(item); + } + } + + // m_RootNode->printTree(std::cout, 0); + + fclose(fin); + fin = nullptr; + + return err; +} + +// ----------------------------------------------------------------------------- +void saveFile(const std::string& outputDir, const SFSNodeItemPtr& node) +{ + int err = 0; + + std::map children = node->children(); + for(const auto& child : children) + { + if(child.second->isDirectory()) + { + std::string path = outputDir + "/" + child.second->getFileName(); + SFSUtils::mkdir(path, true); + saveFile(path, child.second); + } + else + { + std::string outputPath = outputDir + "/" + child.second->getFileName(); + std::cout << "Saving File: " << outputPath << std::endl; + err = child.second->writeFile(outputPath); + } + } +} + +// ----------------------------------------------------------------------------- +void SFSReader::extractAll(const std::string& outputPath) const +{ + SFSUtils::mkdir(outputPath, true); + saveFile(outputPath, m_RootNode); +} + +// ----------------------------------------------------------------------------- +std::vector split(const std::string& s, char delimiter) +{ + std::vector tokens; + std::string token; + std::istringstream tokenStream(s); + while(std::getline(tokenStream, token, delimiter)) + { + tokens.push_back(token); + } + return tokens; +} + +// ----------------------------------------------------------------------------- +int32_t SFSReader::extractFile(const std::string& outputPath, const std::string& sfsPath) const +{ + SFSUtils::mkdir(outputPath, true); + + std::vector tokens = split(sfsPath, '/'); + + SFSNodeItemPtr node = m_RootNode; + for(const auto& token : tokens) + { + node = node->child(token); + if(node.get() == nullptr) + { + break; + } + if(node->isDirectory()) + { + std::string path = outputPath + "/" + node->getFileName(); + SFSUtils::mkdir(path, true); + } + } + + if(node.get() == nullptr) + { + std::cout << "Path does not exist in SFS file. '" << sfsPath << "'" << std::endl; + return -10; + } + std::string fullpath = outputPath + "/" + sfsPath; + return node->writeFile(fullpath); +} + +// ----------------------------------------------------------------------------- +bool SFSReader::fileExists(const std::string& sfsPath) const +{ + if(sfsPath.empty()) + { + return false; + } + std::vector tokens = split(sfsPath, '/'); + + SFSNodeItemPtr node = m_RootNode; + for(const auto& token : tokens) + { + node = node->child(token); + if(nullptr == node.get()) + { + return false; + } + } + + if(node == m_RootNode) + { + return false; + } + return node.get() != nullptr; +} diff --git a/src/SFSReader.h b/src/SFSReader.h new file mode 100644 index 0000000..f1af7e3 --- /dev/null +++ b/src/SFSReader.h @@ -0,0 +1,114 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#pragma once + +#include +#include +#include +#include + +class SFSNodeItem; +using SFSNodeItemPtr = std::shared_ptr; + +/** + * @brief The SFSReader class + */ +class SFSReader +{ +public: + SFSReader(); + ~SFSReader(); + + SFSReader(const SFSReader&) = delete; // Copy Constructor Not Implemented + SFSReader(SFSReader&&) = delete; // Move Constructor Not Implemented + SFSReader& operator=(const SFSReader&) = delete; // Copy Assignment Not Implemented + SFSReader& operator=(SFSReader&&) = delete; // Move Assignment Not Implemented + + /** + * @brief openFile + * @param filepath + * @return + */ + int parseFile(const std::string& filepath); + + /** + * @brief getVersion + * @return + */ + float getVersion() const; + + /** + * @brief getChunkSize + * @return + */ + uint32_t getChunkSize() const; + + /** + * @brief getUsableChunkSize + * @return + */ + uint32_t getUsableChunkSize() const; + + /** + * @brief getInputFile + * @return + */ + const std::string& getInputFile() const; + + /** + * @brief getRootNode + * @return + */ + SFSNodeItemPtr getRootNode() const; + + /** + * @brief extractAll This will extract all files within the SFS file into a designated folder. + * @param outputPath + */ + void extractAll(const std::string& outputPath) const; + + /** + * @brief extractFile This will extract a specific file within the SFS File + * @param outputPath + * @param sfsPath + * @return Error code + */ + int32_t extractFile(const std::string& outputPath, const std::string& sfsPath) const; + + /** + * @brief Checks if the given path exists in the BCF archive + * @param sfsPath The path to check + * @return + */ + bool fileExists(const std::string& sfsPath) const; + +private: + std::string m_FilePath; + bool m_IsValid = false; + + float m_Version = 0.0f; + uint32_t m_ChunkSize = 0; + uint32_t m_UsableChunkSize = 0; + + uint32_t m_TreeAddress = 0; + uint32_t m_NumTreeItems = 0; + uint32_t m_NumChunks = 0; + + SFSNodeItemPtr m_RootNode; +}; diff --git a/src/SFSUtils.hpp b/src/SFSUtils.hpp new file mode 100644 index 0000000..1a65905 --- /dev/null +++ b/src/SFSUtils.hpp @@ -0,0 +1,374 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#pragma once + + +#include +#include +#include +#include +#include + + +#if defined (_MSC_VER) +#include +#define UNLINK _unlink +#define SFS_UTIL_PATH_MAX MAX_PATH +#define SFS_UTIL_GET_CWD _getcwd +#else +#define UNLINK ::unlink +#include +#define SFS_UTIL_PATH_MAX PATH_MAX +#define SFS_UTIL_GET_CWD ::getcwd +#endif + + +#include + +#if defined (_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#define SFS_UTIL_STATBUF struct _stati64 // non-ANSI defs +#define SFS_UTIL_STATBUF4TSTAT struct _stati64 // non-ANSI defs +#define SFS_UTIL_STAT _stati64 +#define SFS_UTIL_FSTAT _fstati64 +#define SFS_UTIL_FSEEK _fseeki64 +#define SFS_UTIL_STAT_REG _S_IFREG +#define SFS_UTIL_STAT_DIR _S_IFDIR +#define SFS_UTIL_STAT_MASK _S_IFMT +#if defined(_S_IFLNK) +#define SFS_UTIL_STAT_LNK _S_IFLNK +#endif + +#elif defined (__APPLE__) + +#define SFS_UTIL_STATBUF struct stat +#define SFS_UTIL_STATBUF4TSTAT struct stat +#define SFS_UTIL_STAT stat +#define SFS_UTIL_FSTAT fstat +#define SFS_UTIL_FSEEK fseek +#define SFS_UTIL_STAT_REG S_IFREG +#define SFS_UTIL_STAT_DIR S_IFDIR +#define SFS_UTIL_STAT_MASK S_IFMT +#define SFS_UTIL_STAT_LNK S_IFLNK + +#else +#define SFS_UTIL_STATBUF struct stat +#define SFS_UTIL_STATBUF4TSTAT struct stat +#define SFS_UTIL_STAT stat +#define SFS_UTIL_FSTAT fstat +#define SFS_UTIL_FSEEK fseek +#define SFS_UTIL_STAT_REG S_IFREG +#define SFS_UTIL_STAT_DIR S_IFDIR +#define SFS_UTIL_STAT_MASK S_IFMT +#define SFS_UTIL_STAT_LNK S_IFLNK +#endif + + +/** + * @brief The SFSUtils class + */ +class SFSUtils +{ + + public: + + + template + static T readScalar(FILE *f, int32_t &err) + { + T value = 0x00; + size_t nread = fread(&value, sizeof(T), 1, f); + if(nread != 1) + { + err = -1; + } + return value; + } + + + static void readArray(FILE* f, size_t byteCount, void* data, int32_t &err) + { + size_t nread = fread(data, 1, byteCount, f); + if(nread != byteCount) + { + std::cout << "readArray(" << __LINE__ << ") error reading bytes: " << ferror(f) << std::endl; + std::cout << "readArray(" << __LINE__ << ") EOF Value: " << feof(f) << std::endl; + perror("SFSUtils::readArray Error"); + err = -1; + } + } + + +#if defined (WIN32) + static const char Separator = '\\'; +#else + static const char Separator = '/'; +#endif + static const char UnixSeparator = '/'; + static const char Dot = '.'; + + // ----------------------------------------------------------------------------- + static std::string fromNativeSeparators(const std::string &fsPath) + { + std::string path(fsPath); +#if defined (WIN32) + for (int i=0; i<(int)path.length(); i++) { + if (path[i] == Separator ) + path[i] = UnixSeparator; + } +#endif + return path; + } + + // ----------------------------------------------------------------------------- + static std::string toNativeSeparators(const std::string &fsPath) + { + std::string path(fsPath); +#if defined (WIN32) + for (int i=0; i<(int)path.length(); i++) { + if (path[i] == UnixSeparator ) + path[i] = Separator; + } +#endif + return path; + } + +#if defined (WIN32) + // ----------------------------------------------------------------------------- + static bool isDirPath(const std::string &folderPath, bool *existed) + { + std::string fsPath = folderPath; + if (fsPath.length() == 2 &&fsPath.at(1) == ':') + fsPath += Separator; + + DWORD fileAttrib = INVALID_FILE_ATTRIBUTES; + fileAttrib = ::GetFileAttributesA(fsPath.c_str() ); + + if (existed) + *existed = fileAttrib != INVALID_FILE_ATTRIBUTES; + + if (fileAttrib == INVALID_FILE_ATTRIBUTES) + return false; + + return (fileAttrib & FILE_ATTRIBUTE_DIRECTORY) ? true : false; + } +#endif + + + + + // ----------------------------------------------------------------------------- + static bool exists(const std::string &fsPath) + { + int error; + std::string dirName(fsPath); + // Both windows and OS X both don't like trailing slashes so just get rid of them + // for all Operating Systems. + if (dirName[dirName.length() - 1] == Separator) { + dirName = dirName.substr(0, dirName.length() - 1); + } + SFS_UTIL_STATBUF st; + error = SFS_UTIL_STAT(dirName.c_str(), &st); + return (error == 0); + } + + + // ----------------------------------------------------------------------------- + static std::string cleanPath(const std::string &fsPath) + { + if (fsPath.length() == 0) + {return fsPath;} + std::string path(fsPath); + char slash = '/'; + char dot = '.'; + if (Separator != UnixSeparator) + { + path = fromNativeSeparators(path); + } + + // Peel off any trailing slash + if (path[path.length() -1 ] == slash) + { + path = path.substr(0, path.length() -1); + } + + std::vector stk; + std::string::size_type pos = 0; + std::string::size_type pos1 = 0; + + pos = path.find_first_of(slash, pos); + pos1 = path.find_first_of(slash, pos + 1); +#if defined (WIN32) + // Check for UNC style paths first + if (pos == 0 && pos1 == 1) + { + pos1 = path.find_first_of(slash, pos1 + 1); + } else +#endif + if (pos != 0) + { + stk.push_back(path.substr(0, pos)); + } + // check for a top level Unix Path: + if (pos == 0 && pos1 == std::string::npos) + { + stk.push_back(path); + } + + + while (pos1 != std::string::npos) + { + if (pos1 - pos == 3 && path[pos+1] == dot && path[pos+2] == dot) + { + // std::cout << "Popping back element" << std::endl; + if (!stk.empty()) + { + stk.pop_back(); + } + } + else if (pos1 - pos == 2 && path[pos+1] == dot ) + { + + } + else if (pos + 1 == pos1) { + + } + else { + stk.push_back(path.substr(pos, pos1-pos)); + } + pos = pos1; + pos1 = path.find_first_of(slash, pos + 1); + if (pos1 == std::string::npos) + { + stk.push_back(path.substr(pos, path.length() - pos)); + } + } + std::string ret; + for(const auto& str : stk) + { + ret.append(str); + } + ret = toNativeSeparators(ret); +#if defined (WIN32) + if (ret.length() > 2 + && isalpha(ret[0]) != 0 + && islower(ret[0]) != 0 + && ret[1] == ':' && ret[2] == '\\') + { + //we have a lower case drive letter which needs to be changed to upper case. + ret[0] = toupper(ret[0]); + } +#endif + return ret; + } + + // ----------------------------------------------------------------------------- + static bool mkdir(const std::string &name, bool createParentDirectories) + { +#if defined (WIN32) + std::string dirName = name; + if (createParentDirectories) { + dirName = toNativeSeparators(cleanPath(dirName)); + // We specifically search for / so \ would break it.. + std::string::size_type oldslash = std::string::npos; + if (dirName[0] == '\\' && dirName[1] == '\\') + { + // Don't try to create the root fsPath of a UNC fsPath; + // CreateDirectory() will just return ERROR_INVALID_NAME. + for (unsigned int i = 0; i < dirName.size(); ++i) { + if (dirName[i] != Separator) + { + oldslash = i; + break; + } + } + if (oldslash != std::string::npos) { + oldslash = dirName.find(Separator, oldslash); + } + } + for (std::string::size_type slash=0; slash != std::string::npos; oldslash = slash) { + slash = dirName.find(Separator, oldslash+1); + if (slash == std::string::npos) { + if(oldslash == dirName.length()) + break; + slash = dirName.length(); + } + if (slash != std::string::npos) { + std::string chunk = dirName.substr(0, slash); + bool existed = false; + if (!SFSUtils::isDirPath(chunk, &existed) && !existed) + { + if (!::CreateDirectoryA(chunk.c_str(), 0) ) { return false; } + } + } + } + return true; + } + return (!::CreateDirectoryA(dirName.c_str(), 0) == 0); +#else + + std::string dirName = name; + if (createParentDirectories) + { + dirName = SFSUtils::cleanPath(dirName); + std::string::size_type slash = 0; + for (std::string::size_type oldslash = 1; slash != std::string::npos; oldslash = slash) + { + slash = dirName.find(Separator, oldslash + 1); + if (slash == std::string::npos) + { + if (oldslash == dirName.length()) {break;} + slash = dirName.length(); + } + if (slash != std::string::npos) + { + std::string chunk = dirName.substr(0, slash); + if (!SFSUtils::exists(chunk)) + { + SFS_UTIL_STATBUF st; + if(SFS_UTIL_STAT(chunk.c_str(), &st) != -1) + { + if ((st.st_mode & S_IFMT) != S_IFDIR) + { + return false; + } + } + else if (::mkdir(chunk.c_str(), 0777) != 0) + { + return false; + } + } + } + } + return true; + } +#if defined(__APPLE__) // Mac X doesn't support trailing /'s + if (dirName[dirName.length() - 1] == '/') + { + dirName = dirName.substr(0, dirName.length() - 1); + } +#endif + return (::mkdir(dirName.c_str(), 0777) == 0); +#endif + } + +}; diff --git a/src/bcf2hdf5.cpp b/src/bcf2hdf5.cpp new file mode 100644 index 0000000..94b4fc8 --- /dev/null +++ b/src/bcf2hdf5.cpp @@ -0,0 +1,88 @@ +// Qt Includes +#include +#include +#include +#include + +#include "BcfHdf5Convertor.h" + +#include + +int main(int argc, char* argv[]) +{ + QString version("1.0.0"); + // Instantiate the QCoreApplication that we need to get the current path and load plugins. + QCoreApplication* app = new QCoreApplication(argc, argv); + QCoreApplication::setOrganizationName("BlueQuartz Software"); + QCoreApplication::setOrganizationDomain("bluequartz.net"); + QCoreApplication::setApplicationName("bcf2hdf5"); + QCoreApplication::setApplicationVersion(version); + + QCommandLineParser parser; + QString str; + QTextStream ss(&str); + ss << "bcf2hdf5 (" << version << "): This application will convert a Bruker Nano .bcf file into a somewhat compatible HDF5 file."; + parser.setApplicationDescription(str); + parser.addHelpOption(); + parser.addVersionOption(); + + QCommandLineOption bcfFileArg(QStringList() << "b" + << "bcf", + "Input BCF ", "file"); + parser.addOption(bcfFileArg); + + QCommandLineOption hdf5FileArg(QStringList() << "o" + << "output", + "Output HDF5 ", "file"); + parser.addOption(hdf5FileArg); + + QCommandLineOption reorderArg(QStringList() << "r" + << "reorder", + "Reorder Data inside of HDF5 file. This can increase final file size significantly. true or false.", "file"); + parser.addOption(reorderArg); + + QCommandLineOption flipPatternArg(QStringList() << "f" + << "flip", + "Flip the patterns across the X Axis (Vertical Flip). true or false.", "file"); + parser.addOption(flipPatternArg); + + // Process the actual command line arguments given by the user + parser.process(*app); + + if(argc != 9) + { + std::cout << "7 Arguments are required. Use --help for more information." << std::endl; + return EXIT_FAILURE; + } + + QString inputFile = parser.value(bcfFileArg); + if(inputFile.isEmpty()) + { + std::cout << "Input file was empty. Use --help for more information." << std::endl; + return EXIT_FAILURE; + } + QString outputFile = parser.value(hdf5FileArg); + if(outputFile.isEmpty()) + { + std::cout << "Output file was empty. Use --help for more information." << std::endl; + return EXIT_FAILURE; + } + + QString reorder = parser.value(reorderArg); + QString flipPatterns = parser.value(flipPatternArg); + + BcfHdf5Convertor convertor(inputFile, outputFile); + convertor.setReorder(reorder == "true"); + convertor.setFlipPatterns(flipPatterns == "true"); + convertor.execute(); + int32_t err = convertor.getErrorCode(); + if(err < 0) + { + std::cout << convertor.getErrorMessage().toStdString() << ": " << convertor.getErrorCode() << std::endl; + } + + std::cout << "Complete" << std::endl; + + /* code */ + return err; +} diff --git a/src/unbcf.cpp b/src/unbcf.cpp new file mode 100644 index 0000000..d64676f --- /dev/null +++ b/src/unbcf.cpp @@ -0,0 +1,70 @@ +/* ============================================================================ + * Copyright (c) 2019 BlueQuartz Software, LLC + * All rights reserved. + * + * This library is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with any project and source this library is coupled. + * If not, see . + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +#include + +#include "SFSReader.h" +#include "SFSNodeItem.h" + + + +/** + * @brief This will upack all the files within an SFS file archive. This will NOT support + * SFS files that have compression or encryption enabled. + * @param argc + * @param argv + * @return + */ +int main(int argc, char const *argv[]) +{ + if(argc != 3) + { + std::cout << "Need the input file name and output directory" << std::endl; + return 1; + } + std::string inputFile(argv[1]); + std::string outputDir(argv[2]); + if(outputDir.back() != '/') + { + outputDir = outputDir + '/'; + } + + SFSReader sfsFile; + sfsFile.parseFile(inputFile); + + +#ifdef _WIN32 + const char slash = '\\'; + #else + const char slash = '/'; + #endif + size_t slashIndex = inputFile.find_last_of(slash); + std::string baseName = inputFile.substr(slashIndex + 1); + size_t dotIndex = baseName.find_last_of('.'); + baseName = baseName.substr(0, dotIndex); + + std::string outputPath = outputDir + baseName; + //sfsFile.extractFile(outputPath, "EBSDData/SEMImage"); + std::cout << "Extracting to " << outputPath << std::endl; + sfsFile.extractAll(outputPath); + + std::cout << "Complete" << std::endl; + + /* code */ + return 0; +}