diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4125888
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,29 @@
+
+doc/*.vsdx
+doc/*.~vsdx
+
+working/*.un~
+
+*.o
+bin/aurora*
+*.swp
+bin/compile_proto
+.DS_Store
+
+bin/
+
+*.log.*
+
+work/aurora*.INFO
+work/aurora*.WARNING
+work/aurora*.ERROR
+work/aurora*.FATAL
+
+work/*/aurora.INFO
+work/*/aurora.WARNING
+work/*/aurora.ERROR
+work/*/aurora.FATAL
+
+work/*/raft.binlog.*
+x64
+doc/~$grpc-issue.xlsx
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..eba6a45
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,37 @@
+cmake_minimum_required(VERSION 3.3)
+project(Aurora)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+option(GLOG_INC_DIR,"Use include" ON)
+
+message("project src:" ${PROJECT_SOURCE_DIR})
+
+file(GLOB_RECURSE ALL_SRC ${PROJECT_SOURCE_DIR}/src *.cc)
+
+foreach(ARG ${ALL_SRC})
+ message("one source:${ARG}")
+endforeach()
+
+list(FILTER ALL_SRC EXCLUDE REGEX .*trivial_lock_deque.cc)
+list(FILTER ALL_SRC EXCLUDE REGEX .*trivial_lock_hash.cc)
+list(FILTER ALL_SRC EXCLUDE REGEX .*trivial_lock_list.cc)
+list(FILTER ALL_SRC EXCLUDE REGEX .*trivial_lock_queue.cc)
+
+set(MAIN_SRC ${ALL_SRC})
+set(TEST_SRC ${ALL_SRC})
+list(FILTER MAIN_SRC EXCLUDE REGEX .*gtest_main.cc)
+list(FILTER TEST_SRC EXCLUDE REGEX .*[/\\]main.cc)
+
+message("main src found:" ${MAIN_SRC})
+message("test src found:" ${TEST_SRC})
+
+# add the executable
+add_executable(aurora ${ALL_SRC})
+add_executable(aurora_test ${ALL_SRC})
+
+
+target_include_directories(aurora PUBLIC "${PROJECT_SOURCE_DIR}/src")
+
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..8224297
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,674 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ aurora
+ Copyright (C) 2019 arthur
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ aurora Copyright (C) 2019 arthur
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..498b6fa
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,169 @@
+
+BUILD_TYPE=debug
+
+CXX = g++
+CXXFLAGS = -std=c++17 -D_RAFT_UNIT_TEST_
+
+ifeq ($(BUILD_TYPE),debug)
+CXXFLAGS += -g3
+else
+ifneq ($(BUILD_TYPE),release)
+$(error input 'BUILD_TYPE' parameter can be either 'debug' or 'release')
+endif
+CXXFLAGS += -O3
+endif
+
+SRCDIR = src
+BINDIR = bin
+OBJDIR = $(BINDIR)/$(BUILD_TYPE)/object
+
+THIRD_PARTY_DIR=./third_party
+
+INC = -I./src \
+ -I./src/protocol \
+ -I$(THIRD_PARTY_DIR)/glog/src \
+ -I$(THIRD_PARTY_DIR)/gflags/build-dir/include \
+ -I$(THIRD_PARTY_DIR)/boost_1_68_0 \
+ -I$(THIRD_PARTY_DIR)/googletest/googletest/include\
+ -I$(THIRD_PARTY_DIR)/grpc/include\
+
+LIB = /usr/local/lib/libglog.a \
+ /usr/local/lib/libgflags.a \
+ /usr/local/lib/libgtest.a \
+ /usr/local/lib/libgflags_nothreads.a \
+ /usr/local/lib/libboost_filesystem.a \
+ /usr/local/lib/libboost_system.a \
+ /usr/local/lib/libboost_thread.a \
+ /usr/local/lib/libprotobuf.a \
+ /usr/local/lib/libgrpc++.a \
+ /usr/local/lib/libgrpc.a \
+ /usr/local/lib/libgrpc++_reflection.a \
+ /usr/local/lib/libtcmalloc.a \
+ -lz -ldl -lpthread \
+
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+ LIB += -latomic
+endif
+
+PROTOS_PATH = ./src/protocol
+
+vpath %.proto $(PROTOS_PATH)
+
+PROTO_FLAG=$(BINDIR)/compile_proto
+MAIN_PROGRAM=$(BINDIR)/$(BUILD_TYPE)/aurora
+MAIN_TEST=$(BINDIR)/$(BUILD_TYPE)/aurora_test
+
+-include prepare $(PROTO_FLAG)
+
+.PHONY: all
+all: system-check $(MAIN_PROGRAM) $(MAIN_TEST)
+
+.PHONY: prepare
+prepare:
+ mkdir -p $(OBJDIR)
+
+ALL_SRC_FILES=$(wildcard src/*.cc src/*/*.cc)
+TPL_CC_FILES=%src/tools/lock_free_deque.cc %src/tools/lock_free_hash.cc \
+ %src/tools/trivial_lock_double_list.cc %src/tools/lock_free_queue.cc \
+ %src/tools/lock_free_queue.cc %src/tools/lock_free_single_list.cc \
+ %src/common/request_base.cc %src/client/client_framework.cc \
+ %src/follower/follower_request.cc %src/leader/connection_pool.cc \
+ %src/leader/client_pool.cc %src/leader/leader_request.cc \
+ %src/service/ownership_delegator.cc %src/candidate/candidate_request.cc \
+ %src/tools/data_structure_base.cc %src/tools/lock_free_unordered_single_list.cc \
+ %src/tools/trivial_lock_list_base.cc %src/tools/lock_free_single_list.cc\
+ %src/tools/trivial_lock_single_list.cc %src/tools/lock_free_hash_specific.cc\
+ %src/common/react_group.cc \
+
+COMPILE_SRC_FILES = $(filter-out $(TPL_CC_FILES), $(ALL_SRC_FILES) )
+
+OBJ = $(patsubst %.cc, $(OBJDIR)/%.o, $(COMPILE_SRC_FILES))
+
+EXE_MAIN_OBJ=%/main.o
+UTEST_MAIN_OBJ=%gtest_main.o
+EXE_OBJ = $(filter-out $(UTEST_MAIN_OBJ), $(OBJ) )
+UTEST_OBJ = $(filter-out $(EXE_MAIN_OBJ), $(OBJ) )
+
+.PHONY:test
+test:$(PROTO_FLAG)
+ @echo "all:" $(ALL_SRC_FILES)
+ @echo "src:" $(COMPILE_SRC_FILES)
+ @echo "object:" $(OBJ)
+
+$(OBJDIR)/%.o: %.cc
+ @mkdir -p $(OBJDIR)/$(dir $<)
+ $(CXX) $(CXXFLAGS) $(INC) -c $< -o $@
+
+$(MAIN_PROGRAM): $(EXE_OBJ)
+ $(CXX) $(CXXFLAGS) $^ $(LIB) -o $@
+
+$(MAIN_TEST): $(UTEST_OBJ)
+ $(CXX) $(CXXFLAGS) $^ $(LIB) -o $@
+
+PROTOC = protoc
+GRPC_CPP_PLUGIN = grpc_cpp_plugin
+GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)`
+
+$(PROTO_FLAG): $(PROTOS_PATH)/raft.proto
+ $(PROTOC) -I $(PROTOS_PATH) --cpp_out=$(PROTOS_PATH) $<
+ $(PROTOC) -I $(PROTOS_PATH) --grpc_out=$(PROTOS_PATH) --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $<
+ touch $@
+
+.PHONY: clean
+clean:
+ rm -rf $(OBJDIR) $(PROTOS_PATH)/*.h $(PROTOS_PATH)/*.cc $(PROTO_FLAG) $(MAIN_PROGRAM) $(MAIN_TEST)
+
+
+# The following is to test your system and ensure a smoother experience.
+# They are by no means necessary to actually compile a grpc-enabled software.
+
+PROTOC_CMD = which $(PROTOC)
+PROTOC_CHECK_CMD = $(PROTOC) --version | grep -q libprotoc.3
+PLUGIN_CHECK_CMD = which $(GRPC_CPP_PLUGIN)
+HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false)
+ifeq ($(HAS_PROTOC),true)
+HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false)
+endif
+HAS_PLUGIN = $(shell $(PLUGIN_CHECK_CMD) > /dev/null && echo true || echo false)
+
+SYSTEM_OK = false
+ifeq ($(HAS_VALID_PROTOC),true)
+ifeq ($(HAS_PLUGIN),true)
+SYSTEM_OK = true
+endif
+endif
+
+system-check:
+ifneq ($(HAS_VALID_PROTOC),true)
+ @echo " DEPENDENCY ERROR"
+ @echo
+ @echo "You don't have protoc 3.0.0 installed in your path."
+ @echo "Please install Google protocol buffers 3.0.0 and its compiler."
+ @echo "You can find it here:"
+ @echo
+ @echo " https://github.com/google/protobuf/releases/tag/v3.0.0"
+ @echo
+ @echo "Here is what I get when trying to evaluate your version of protoc:"
+ @echo
+ -$(PROTOC) --version
+ @echo
+ @echo
+endif
+ifneq ($(HAS_PLUGIN),true)
+ @echo " DEPENDENCY ERROR"
+ @echo
+ @echo "You don't have the grpc c++ protobuf plugin installed in your path."
+ @echo "Please install grpc. You can find it here:"
+ @echo
+ @echo " https://github.com/grpc/grpc"
+ @echo
+ @echo "Here is what I get when trying to detect if you have the plugin:"
+ @echo
+ -which $(GRPC_CPP_PLUGIN)
+ @echo
+ @echo
+endif
+ifneq ($(SYSTEM_OK),true)
+ @false
+endif
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..2728953
--- /dev/null
+++ b/README.md
@@ -0,0 +1,148 @@
+
+
+
+
+
+
+
+## Table of Contents
+
+* [About the Project](#about-the-project)
+* [Getting Started](#getting-started)
+ * [Prerequisites](#prerequisites)
+ * [Building](#building)
+ * [Usage](#usage)
+* [Contributing](#contributing)
+* [License](#license)
+* [Contact](#contact)
+* [Donation](#donation)
+
+
+## About The Project
+
+**Aurora is a [Raft](https://raft.github.io) based K-V database**. The basics of the current architecture is :
+
+
+
+*Note : The name `Aurora` **has nothing to do with** neither [Apache Aurora](http://aurora.apache.org/) nor [Amazon aurora](https://aws.amazon.com/rds/aurora/).*
+
+It's inspired by the [raft protocol](https://raft.github.io) which often being known as a contrast of another widely known protocol : `paxos`. Aurora comes along with all of raft dominant features supported:
+* **Log Replication**: A two phase commit data replication solution.
+* **Leader Election** : A majority & term based failover solution.
+* **Membership Change**: A smart approach dealing with cluster topology changing.
+
+Besides the consensus protocol, it also comes with a local storage subsystem whose idea is inspired and being the same with [leveldb](https://github.com/google/leveldb) : to enhance blind writing operation throughput.
+
+Last but not least, it's implemented by using the modern cpp(11/14/17) which may contributes to the popularization of the new lanaguage standards. After all, newer is probably better.
+
+
+## Getting Started
+
+Project root directory explanation:
+* src : source code.
+* doc : documents.
+* bin : binaries & objects after successfully building.
+* working : running directory of aurora.
+* third_party : the third party dependencies.
+
+Aurora has some basic components you need to firstly building before running.
+
+### Prerequisites
+
+* [protobuf](https://github.com/protocolbuffers/protobuf). >=3.0.0.
+* [grpc](https://github.com/grpc/grpc). >=1.8.x.
+* [boost](https://www.boost.org/). >=1.64.0.
+* [glog](https://github.com/google/glog). >=0.3.3.
+* [gflags](https://github.com/gflags/gflags). >=2.2.0.
+* [gtest](https://github.com/google/googletest). >=1.7.0.
+* [gperftools](https://github.com/gperftools/gperftools). >=7.0.
+
+Create a `third_party` directory and build the above dependencies under that.
+
+*Note: How to build the dependencies is beyond the scope and you may need consulting the documents for each of them. Making sure the headers and libraries are correctly installed on your system. That maybe boring but can't get around.*
+
+After successfully built all the above dependencies, your working directory should looks something like this:
+```
+|-- src
+|-- doc
+|-- bin
+|-- working
+|-- third_party
+ |-- boost_1_68_0
+ |-- protobuf
+ |-- grpc
+ |-- glog
+ |-- gflags
+ |-- googletest
+```
+
+### Building
+Now, you are ready to compile aurora itself.
+
+* unix & linux & osx:
+```console
+cd aurora && make -j4
+```
+
+> Note: some gcc versions(like gcc (GCC) 8.3.1) don't fully support std::atomic, thus you might need to install `libatomic` manually.
+
+* windows:
+see [building under windows](doc/windows.md)
+
+
+## Usage
+
+First , take a look at the configure files:
+* `election.config` : inner usage for election.
+* `membership-change.config` : inner usage for membership change.
+* `topology.config` : setup a cluster:
+ * `leader` :leader node.
+ * `followers` :follower nodes.
+ * `candidates` :follower nodes.
+ * all nodes are with format : `xx.xx.xx.xx:port`,like `192.168.0.100`.
+
+*Only `topology.config` is intending to be controlled by users, the others are either managed by the system or for debugging purpose.*
+
+Second, after finish configuring `topology.config`, you can start a node by :
+* running command under *nix:
+
+```console
+ cd aurora/working/
+ nohup ../bin/aurora > aurora.log 2>&1 &
+```
+
+* runing command under windows:
+
+```console
+cd aurora\working\
+..\aurora\working\aurora.exe
+```
+
+
+
+
+## Contributing
+It is strongly recommended to read the [developer guide](doc/developer_guide.md) for details.
+
+
+## License
+Distributed under the **GPLv3** License. See the [license file](LICENSE) for more information.
+
+
+## Contact
+Arthur - pplorins@gmail.com
+
+
+## Donation
+This project consumed a lot time away from the author, and if you think it helps, don't hesitate to show your generosity :moneybag: :moneybag: :moneybag:
+
+* paypal: [paypal.me link](paypal.me/arthurCoo)
+* wechat:
+
+ 
+
+* alipay:
+
+ 
+
+
diff --git a/build/.gitignore b/build/.gitignore
new file mode 100644
index 0000000..94548af
--- /dev/null
+++ b/build/.gitignore
@@ -0,0 +1,3 @@
+*
+*/
+!.gitignore
diff --git a/doc/benchmark.md b/doc/benchmark.md
new file mode 100644
index 0000000..e9758ae
--- /dev/null
+++ b/doc/benchmark.md
@@ -0,0 +1,55 @@
+# Benchmark
+
+This is the environment for benchmark.
+
+### Machines:
+
+|machine|OS|Compiler|Cpu|Memory|Disk|
+|--|--|--|--|
+|machineA|win10 64 pro|MSVC 19.00.24215.1 for x86|i7 8(4*2)core 4.0GHZ|16G|SSD|
+|machineB|Mac 64 High Seria 10.13.6|Apple LLVM version 10.0.0 (clang-1000.10.44.4)|i7 8(4*2)core 2.2GHZ|16G|SSD|
+
+### Network:
+Home router: tplink TL-WR880N 450Mbps. Latency:
+* machineA --> machineB : 1~100ms(unstable)
+* machineB --> machineA : 1~5ms(stable)
+
+### Depoly:
+Leader is deployed on machineA, all followers are deployed on machineB.
+
+### Config:
+
+#### leader config:
+--do_heartbeat=false --iterating_wait_timeo_us=2000000 --port=10010 --leader_append_entries_rpc_timeo_ms=5000 --leader_commit_entries_rpc_timeo_ms=5000 --client_cq_num=2 --client_thread_num=2 --notify_cq_num=2 --notify_cq_threads=4 --call_cq_num=2 --call_cq_threads=2 --iterating_threads=2 --client_pool_size=50000
+
+#### follower config:
+--checking_heartbeat=false --iterating_wait_timeo_us=50000 --disorder_msg_timeo_ms=100000 --port=${port} --notify_cq_num=1 --notify_cq_threads=4 --call_cq_num=1 --call_cq_threads=4 --iterating_threads=2
+
+### Logic
+
+Using the `TestLeaderServiceClient.Benchmark` test case in `src\gtest\service\test_leader_service.h` with arguments: --gtest_filter=TestLeaderServiceClient.Benchmark --client_write_timo_ms=10000 --benchmark_client_cq_num=1 --benchmark_client_thread_num_per_cq=1 --value_len=1 --leader_svc_benchmark_req_count=20000
+
+Which is, using asynchronous way of sending 2w requests each time, counting overall write throughput & average write latency. Reading performance is not considered. The influential factors being tested are : **data length** and **number of followers**.
+
+### Result:
+
+#### Taking **data length** as the factor:
+
+
+
+
+
+
+
+#### Taking **number of followers** as the factor:
+
+
+
+
+
+
+
+### Bottleneck analysis:
+If we remove all the logics leaving only the skeleton(where leader broadcasting requests to all its followers) left, we'll get a result of ~2w throughput & ~500ms latency. After a more detailed investigation, you'll find that it's `UnaryAsyncClient::EntrustRequest()::this->m_reader->Finish` that made the watershed, probably because it would trigger the broadcasting process to the followers which further bring a performance drawback. So the **bottleneck is on the grpc framework itself** under my experiments. Better practices for how to utilize grpc is needed.
+
+
diff --git a/doc/developer_guide.md b/doc/developer_guide.md
new file mode 100644
index 0000000..3c893fa
--- /dev/null
+++ b/doc/developer_guide.md
@@ -0,0 +1,1015 @@
+
+# Developer guide
+
+
+## Table of Contents
+
+* [Architecture Overview](#architecture-overview)
+ * [Overview](#overview)
+ * [About the RPC](#about-the-rpc)
+ * [RPC framework](#rpc-framework)
+ * [Asynchronous in GRPC](#asynchronous-in-grpc)
+ * [RPC interfaces](#rpc-interfaces)
+* [Details Design](#details-design)
+ * [Data Structures](#data-structure)
+ * [Trivial lock double list](#trivial-lock-double-list)
+ * [Trivial lock single list](#trivial-lock-single-list)
+ * [Lockfree hash](#lockfree-hash)
+ * [Lockfree deque](#lockfree-deque)
+ * [Lockfree MPMC queue](#lockfree-mpmc-queue)
+ * [Lockfree priority queue](#lockfree-priority-queue)
+ * [Basic workflow](#basic-workflow)
+ * [Follower workflow](#follower-workflow)
+ * [Leader workflow](#leader-workflow)
+ * [Connection pool](#connection-pool)
+ * [Client pool](#client-pool)
+ * [Workflow anatomy](#workflow-anatomy)
+ * [Asynchronous Framework](#asynchronous-framework)
+ * [Threading model](#threading-model)
+ * [Parallel Replicating](#parallel-replicating)
+ * [CGG problem](#the-cgg-problem)
+ * [Lifetime management](#lifetime-management)
+ * [Global Timer](#global-timer)
+ * [About The Binlog](#about-the-binlog)
+ * [Meta data](#meta-data-part)
+ * [Data part](#data-part)
+ * [Supported operations](#supported-operation)
+ * [Storage Layer](#storage-layer)
+ * [GC](#storage-gc)
+ * [Election](#election)
+ * [The basics](#the-basics)
+ * [About prevoting](#about-prevoting)
+ * [The live lock problem & solution](#the-live-lock-problem--solution)
+ * [Membership Change](#membership-change)
+
+* [Contributing](#contributing)
+ * [Future Work](#future-work)
+ * [Naming Conventions](#naming-conventions)
+ * [Unit Test](#unit-test)
+ * [Development history](#dev-his)
+
+
+
+## Architecture Overview
+
+### Overview
+
+As you can see in the [README](../README.md) file , aurora's cluster structure has nothing special than a
+typical raft cluster ,it has a leader exposed to the clients serving their read & write(only
+[blind writing](https://en.wikipedia.org/wiki/Blind_write) is supported for now) requests.
+
+
+
+### About the RPC
+
+#### RPC framework
+
+The RPC framework for each running node is [google's grpc](https://github.com/grpc/grpc), main benefits enjoyed by choosing it from the others are :
+* taking the advantage of the event-driven asynchronous IO model.
+* utilizing multi cores with its multithreading model which is even customizable.
+* the sophisticated serialization solution : [protobuf](https://github.com/protocolbuffers/protobuf).
+* rich documents come in many ways.
+
+If we choose the grpc as the underlying network library , one important thing should be clearly understood, that is the difference of its `synchronous model` and `asynchronous model`. In short:
+* synchronous : user cannot manage the application level threads.
+* asynchronous : It's user's duty to control the application level threads.
+
+In the synchronous model, the fixed threading model makes programmers' life easier by telling them that you just need to fill in the virtual RPC interfaces. This is pretty good in many ways, but not for all.
+
+In the asynchronous model, by allowing the programmers to customize the threading model, a grpc node can be fully event-driven whereas the synchronous model cannot. This further is achieved by **utilizing the `CompletionQueue`** .[This talk](https://groups.google.com/forum/#!topic/grpc-io/DuBDpK96B14) is great
+helpful and worth reading carefully. For more information about the how does the grpc core's polling mechanism work, look at [this](https://github.com/grpc/grpc/blob/master/doc/core/epoll-polling-engine.md).
+
+#### Asynchronous in GRPC
+
+There is no abundant materials telling people about how to desing an asynchronous server except [the simple official example](https://grpc.io/docs/tutorials/async/helloasync-cpp/). It's a time consuming work to find out the correct way of writing an asynchronous c++ grpc server, especially for the *bidirectional streaming* version. Talking about it in detail is beyond this topic, here are some examples one can consulting into when he wants to have a basic understanding of how it works:
+* a simple asynchronous grpc c++ server, says *SA*, check [this](https://gist.github.com/ppLorins/d75e60dbbbbd84e218928d9fe6781869).
+* a bidirectional stream asynchronous grpc c++ server, says *SB*, check [this](https://gist.github.com/ppLorins/6e4cc625c2c5b8fd16ced3172b1ada09).
+* a mixed asynchronous version of *SA* and *SB*, check [this](https://gist.github.com/ppLorins/d3392111992097c68454e9381d53b446).
+* a proxy server for forwarding the unary and bidirectional stream request to *SA* and *SB*, this is the most complicated one for it demonstrates how to do client reacting together with server reacting, and this is exactly the way how does the aurora's leader works, check [this](https://gist.github.com/ppLorins/6b492beadecd07eddcd0b548ea0d12c9).
+
+And there are several implicit behaviors(or even pitfalls) you need to pay attention to when developing, here are a list of them:
+* `CompletinQueue` delivering messages to the application in a reversed order as opposite to client entrusting, check [this](https://github.com/grpc/grpc/issues/19658). This can lead to message coming in a bad disorder sequence, increasing the possibility of `AppendEntries` timeout on the follower side since the order plays an important role in the whole `Log Replication` process.
+* c++ async client unexpectly timeout if I don't call `CQ::Next()`, check [this](https://github.com/grpc/grpc/issues/19573). The downside of this behavior is that it forces us to fetch the RPC result ASAP when there is result waiting to be fetched, otherwise a timeout error would be thrown by grpc eventhough there is actually no timeout happened at all. To adapt, dedicated CQs and threads for client reacting are introduced.
+* calling the server side's [grpc::ServerAsyncReaderWriter< W, R >::Finish](https://grpc.github.io/grpc/cpp/classgrpc_1_1_server_async_reader_writer.html#a75342152acd961b7fcf1317bec0b8c3a) interface will trigger twice notifications immediately on the server side, check [this](https://github.com/grpc/grpc/issues/17222) and [this](https://github.com/grpc/grpc/issues/19159), I suppose their meanings might be:
+
+ * First notification: the final status set by the server has already been sent to the wire.
+ * Second notification: the stream has already been closed now.
+
+ Thus if there are multiple threads polling on the same CQ, a synchronization between them is need.
+
+#### RPC interfaces
+
+The RPC interfaces are defined in [the protocol file](../src/protocol/raft.proto),you can see all the internal logic between the leader and followers, here is a short brief of them:
+
+* For Client & Admin :
+ * `Read` : for client invoking , read request.
+ * `Write` : for client invoking , write request.
+ * `MembershipChange` : for admin invoking , topology changing request.
+
+* For Log Replication :
+ * `AppendEntries` : Log replication 2PC's phaseI.
+ * `CommitEntries` : Log replication 2PC's phaseII.
+ * `SyncData` : re-sync all data from leader to follower,this will erase all existing data first on the follower side .
+
+* For Election :
+ * `PreVote` : an additional step before starting the real election process to prevent few false-failure nodes from disrupting the entire cluster by issuing new rounds of election.
+ * `Vote` : candidate requesting votes from the others.
+ * `HeartBeat` : doing heartbeat.
+
+* For Membership change:
+ * `PropagateMemberChange` : membership change phaseI.
+ * `MemberChangeCommit` : membership change phaseII.
+
+## Details Design
+
+This section describing the fundamental concepts and components.
+
+### Data Structure
+
+Since `grpc` is multi-threaded, the business threads should considering the thread-safety problems for almost all of its operations , therefore some lockfree data structures must be introduced to fit the needs.
+
+*Note : Nearly all of the code for CAS operations in Aurora are using the `std::atomic::compare_exchange_strong` api instead of `std::atomic::compare_exchange_weak`, exchanging for readability with little acceptable performance payment.*
+
+#### Trivial Lock Double List
+
+This is a doubly linked **sorted** list. It support two primary operations:
+* Insert : insert at a proper position based on the new node's value.
+* Cuthead : find the longest continuous sublist from head and cut it off from the list containing it.(This demand is the most peculiar one out of all the data structure operations.)
+
+Relationship between the two operations:
+
+* Insert vs Insert is lock free.
+* Insert vs Cuthead is lock free.
+* Cuthead vs Cuthead need to be exclusive from each other, this is where locking exists and the name prefix `trivial` comes from.
+
+The `TrivialLockDoubleList` always having a `Head Node` representing the minimum value of the template type of the list and a `Tail Node` representing the maximum value. This sets both the lower and upper boundaries for a certain list instance, getting rid of dealing with corner cases for inserting and cutting head operations.
+
+We'll see how they the lockfree semantics are implemented step by step :
+
+##### 1. Inserting at different positions.
+This is the simplest and most common case , no worries about multiple thread doing this at the same time, since there are no conflicts could exist :
+
+
+Insert operations in this case will get finished correctly and independently :
+
+
+##### 2. Inserting at the same position.
+
+Think about how to deal with the following case in a lockfree way ?
+
+
+I believe plenty of solutions might have been invented although I didn't consult any of them: I figured out my own version and the answer cannot being as difficult as you might thought if you following the rules:
+
+###### R1. Steps of race condition operation must each *strictly following a same order*.
+###### R2. Repeat the operations from the right place when conflicts are detected.
+
+*Note: The above two principles are not only applies to `TrivialLockDoubleList` but also all of the other lockfree data structures in aurora.*
+
+Taking a closer look at it :
+
+* **Step 1**, make the `pre` and `next` pointer of the new node pointing to its neighbours. E.g. `Node 13`'s `pri` and `next` pointer pointing to `Node 11` and `Node max` respectively in the above picture.
+
+* **Step 2**, the previous node's `next` pointer switching its target from the next node to new node(CAS). E.g. `Node 11`'s `next` now pointing to `Node 15`, switched from pointing to `Node max`:
+ 
+
+ * If the above CAS fails,meaning there are other threads are **doing exactly the same thing at the same position**, conflicts are detected and the thread inserting `Node 13` will start **iterating from `P_NEXT` again**, conforms [R2](#R2) :
+ 
+
+ * It could fail for a second time for the thread inserting `Node 13` when retrying:
+ 
+ but doesn't matter, just repeat until succeed.
+
+* **Step 3**, the next node's `pri` pointer switching its target from the previous node to new node(CAS). E.g. `Node max`'s `pri` now pointing to `Node 15`, switched from pointing `Node 11`:
+
+
+ * After that, the thread inserting `Node 13` will recognize that current position is not correct anymore(by founding 15 > 13 ), therefore it goes ahead toward `Node min`'s' direction and will eventually find that the new right position for `Node 13`: between `Node 11` and `Node 15`, deciding to insert `Node 13` there.
+
+ * This picture also illustrated that by following the above steps **all the conflicts among those threads will eventually get resolved** as the involved threads will either :
+
+ * **win a CAS contention** : so it can proceed on or
+ * **fail a CAS contention** : it will retry and letting others go ahead.
+
+ Neither of the two behaviors will result in an invalid coming out, the only downside is on performance : the cost of retrying.
+
+This achievement can be generally named as `thread re-conciliating` as the result of following [R1](#R1) and [R2](#R2), and is the **essential factor of how threads could cooperating correctly with each other**.
+
+Finally, reconsidering for a while about the above procedure, there is no point of view where race
+condition could happen, also thanks to the CAS semantics.
+
+*Note : Understanding the knowledge described above especially for [R1](#R1) and [R2](#R2) are critical to read the following data structure sections.*
+
+##### 3. Inserting vs cutting head
+
+###### 3.1 What is cutting head
+
+Cutting head means taking the nodes that are continuous counted from the first one off from the list, leaving the remaining elements as the new list, for example, assume we have a list instance like this :
+
+
+
+starting from `Node 4`, `Node 4` & `Node 5` & `Node 6` are continuous , cutting head will take them off from list, leaving `Node 9` alone inside the original list :
+
+
+
+###### 3.2 General steps
+
+Following [R1](#R1), cutting head operations sharing the same order :
+
+* **Step 1**, set the pointers with CAS like this, this is mainly for clarifing the slicing point (in this case between `Node 6` and `Node 9`):
+ Since the `CutHead` operations are exclusive from each other, not a second `CutHead` operation could be issued by any other threads, only under this protection can we move further.
+
+* **Step 2**, change `Node min`'s `next` pointer's target from the first node(`Node 4`) of the list to the first nonadjacent node(`Node 9`) of the list:
+
+
+
+Pay attention to that this CAS operation(modifying `Node min`'s `next` pointer from `Node 4`) could fail due to the fact that there may be threads trying to insert node between `Node min` and `Node 4`,the solutions for this case is :
+
+ * First, revert the modified `next` pointer of `Node 6` to its previous position (pointing to `Node 9`). Checking this operation as **must succeed** otherwise something have went wrong.
+ * Second, [recursively] starting a new `CutHead` operation from all over again.
+
+ Same for the CAS operation of inserting `Node 3`: It could also fail when modifying `Node min`'s `next` pointer from `Node 4` in which case a new insertion will be recursively triggered, since you can't insert a node to a already cut off list, and `Node 3` will eventually being inserted between `Node min` and `Node 9` as is in the original list. All the recursive invocation are incurred by conflicts operations happened at boundaries, so we call this kind of problem as `boundary trouble`. We'll see similiar scenarios later in the `TrivialLockSingleList` section.
+
+* **Step 3**,after **Step 2** succeed, change `Node 9`'s `pri` pointer from pointing to the last continuous node (`Node 6`) to pointing to the `Head Node` :
+
+
+ * If the third CAS operation succeed, congratulations, your luck is not that bad and got the `CutHead` operation almost accomplished with the final step of modify `Node 4`'s `pre` pointer to pointing to null and get the result like this:
+
+
+ Yet there is still a situation where the third CAS operation could fail, but this time I'm not going to tell you the answers right now, trying to imagine it out yourself, doing a practice and having a break :).
+
+ *one minute later...*
+
+ *two minutes later...*
+
+ *X minutes later...*
+
+ *Okay, now I suppose you've really tried :).*
+
+ * Thinking about a scenario where the last continuous node (`Node 7`) is the node that being inserted by another thread at the moment we're doing the third CAS and that insert operation is not completed yet :
+ 
+
+ We can extend the continuous list to containing `Node 7` even it hasn't been completely inserted since the thread which is doing the insertion **MUST** have modified its previous node (from the insertion point of view which is `Node 6`)'s `next` pointer to pointing to the new node (`Node 7`) from pointing to the first nonadjacent node (`Node 9`), yet hasn't modified the first nonadjacent node (`Node 9`)'s `pre` pointer from pointing to its previous node (`Node 6`) to pointing to the new node (Node 7) :
+ 
+
+ *Although making that sounds like a tongue twister,I still would like to describe it in a formal way, once you are confusing about the description, just ignore it and look at the picture above.*
+
+ In this case, the third CAS operation would fail. But it's quite simple to cope with : the cutting head thread just looping on the CAS, it will get finished after the inserting operation has finished :
+ 
+
+ Okay, we've done cutting head from a list now, finally! The result will the same as expected:
+
+ 
+
+###### 3.3 The node lossing problem
+
+Okay, here comes the last one issue we should consider about: there may have unfinished insertions even **after** we've successfully cut some nodes off from the original list:
+
+ 
+
+ If we start using(iterating) the cut off list immediately, we may lost `Node 4.5` and `Node 5.5` which are not yet finished inserting, this is certainly unacceptable. The simplest work around is to just sleeping for a little while(~1-9us) that is relatively long enough for the threads to get their job done. But we can't define a proper value for it, leaving this kind of solution as inelegant. How does aurora cope with it is a little bit complicated:
+
+ * the list maintain a Hash with the inserting thread's id as the key, and a flag as value, the flag is for indicating whether the threads is inserting something or not at the moment.
+ * every inserting thread will set the flag before start inserting and erase it after finished inserting.
+ * before `CutHead` returned, the thread will waiting for the flags to be all cleared in a spin manner(with `std::yield` enabled to reduce cpu wasting).
+
+ This can ensure that only after all the inserting threads are done with their jobs, can the `CutHead` function return. But this solution still not being good enough: there can be threads(says `threads X`) inserting in the original list(not the cut off list) at the moment while `CutHead` thread is on its waiting, `threads X` could prevent `CutHead` from returning forever:
+
+ 
+
+ So we have to distinguish the threads which are truly relevant to the cut off list from `threads X`. But how can we achieve this? Well, note that the inserting value of each inserting threads is the best factor to do this: we can judge whether a thread is relevant by compare the value with the last element in the cut off list. For now the Hash's value isn't a flag anymore, it has become the value being inserted. However, since the value is a template one, recording(copying) it may has variable overhead, so aurora make a compromise : recording a snapshot(pointer of the node immediately after `Head Node`) of the list when threads start inserting rather than the value being inserted, this solution will still prevent most of the `threads X` from disturbing the cutting head operation, which is good enough for us even though it may has `false positive`(uncorrectly judging some unrelevant threads as relevant) effect on some cases.
+
+ *Note:the hash used here is also lockfree, which will introduced soon later.*
+
+###### 3.4 Recap
+
+By following [R1](#R1) and [R2](#R2) and enjoying its positive consequence of `thread re-conciliating`, conflict operations like `insert` and `cuthead` can running correctly, this implementing the lockfree semantics. And this is also all about the lockfree in aurora.
+
+##### 4. Deleting from the list.
+There are two type of deleting elements from a set :
+* physically : the element is no longer exists in the set, cannot be iterated for, disappear in memory.
+* logically : the element still exists in the set, can be found by iterating, but with a `deletion flag` set to true.
+
+It's difficult to understand the correctness of physically deleting in a lockfree way, let alone implementing it. Aurora chooses the easier approach : logically deleting. All things need to keep in mind is that :
+* newly inserted nodes are with the `deletion flag` set to false.
+* delete operations will set the node's `deletion flag` to true.
+* ignore the deleted elements when iterating over it.
+* If encounter deleted nodes when doing `inserte` or `cuthead`, treat them as non-deleted.
+
+The last case will not incur errors, otherwise we cannot easily decide the right position where to `insert` or `cuthead`.
+
+#### Trivial Lock Single List
+
+It would be much easier to understand this another kind of list: `TrivialLockSingleList` if you've already got a correct comprehension of all the above things about `TrivialLockDoubleList`. The demonstrations will be brifely shorted since the designing ideas for these two kind of list are basically the same, you can figure out everything by applying the rule of [R1](#R1) and [R2](#R2).
+
+##### 1. Deleting from the list.
+
+
+
+Conflict insertions like that between `Node 9` and `Node 10` will be resolved by allowing only one of the insertions succeed by the constraint of CAS.
+
+##### 2. Cuting head from the list.
+
+
+
+Nothing special to emphasize, but note the [boundary-trouble](#boundary-trouble) and [node-lossing](#node-lossing) problems also exist here.
+
+#### Lockfree hash
+
+This hash supports two type of usage:
+* unary container, like `std::unordered_set<>`.
+* binary container,aka the kv semantics, like `std::unordered_map<>`.
+
+Compared to `TrivialLockDoubleList`, `LockFreeHash` is simpler :
+
+
+If you have basic knowledge about hash, you can understand this easily.
+
+##### 1. inserting into hash
+* find the right slot by key.
+* insert the new element at head in a CAS manner.
+
+##### 2. deleting from hash
+* find the right slot by key.
+* traverse through the list, setting the required element's `deletion flag` to true if found.Otherwise do nothing.
+
+##### 3. finding from hash
+* find the right slot by key.
+* traverse through the list, find the node with the equal key and its `deletion flag` is false.
+
+Easy peasy japanesey!
+
+##### 4. the mapping operation
+There is a special operation called `map`, it iterating over the hash slots from left to right, within each slot it iterating from top to down, executing the given function (e.g, add every element's key value by 1) to every elements in the hash. But the function could modify the key, changing the right position where the node should be, thus the modified node probably need to move from one slot to another. This is done by first deleting the node at the old position, then inserting it at the right new position.
+
+There is a side effect of moving node from one place to another : the node could potentially be executed by function for more than one time which is probably not what the caller wants. For example, in the above picture, `Node kx` will be executed for the first time when the iteration reaching at `slot 3`, its key get modified (says the new key is `kx'`) by the function, new position is calculated indicating that the node's new position is at `slot 13`, `Node kx'` moving (first deleting then inserting) to the that place, so far so good. But when iteration reaches `slot 13`, it doesn't know the fact that `Node kx'` actually coming from an old node and has ever been executed once, leading to `Node kx'` being executed again, moving to a new slot after `slot 13` and being executed the third time, repeating again and again.
+
+To get around of this trouble, a `tag` is added to each node acting as a `traverse ID`, it's initialized to `0` for each newly inserted node. When a new `map` operation is issued, a random value says `X` will be generated to uniquely indicating this iteration. `X` will be assigned to every the moved node and nodes whose `tag` equals to `x` will be recognized as executed ones during current iteration. Thus no nodes will be executed more than once for one certain `map` operation.
+
+##### 5. Downside
+
+The downside of this design is wasting memory : **the deleted nodes never get physically freed, memory usage will be monotonously increasing**. In aurora it's acceptable since the hash is used for caching some data(like meta data in binlog & sstable) that will be released at a certain point, the hash object will therefore be released together. But if you plan to use the hash in some other scenarios, pay attention to its memory downside.
+
+#### Lockfree deque
+
+The name of `deque` is for history reasons, it's actually a MPMC queue in a variant single linked list form.I failed to figure out an elegant name for it, so just let it go. Not everything is perfect in this world, say nothing of a name.
+
+Okay, MPMC is short for `multiple producers and multiple consumer`, usually used in the context of talking about lockfree stuff. They are many articles about this, and I believe you might have ever investigated it before, if you are confident to know what it is, just skip this section.
+
+The deque is initialized like this:
+
+
+
+A dummy node always be there with both head and tail pointer pointing to it.Producing and consuming are the two basic operations for any MPMC implementations.
+
+##### 1. Producing
+
+Since this is a list, we append the new elements wrapped in a unified node to the end of the list, absolutely in a CAS manner, again.
+
+
+
+In this picture, different color of lines stands for different threads, remember [R1](#R1) and [R2](#R2) ? I suggest taking a looking back if you forgot them, the two fundamental rules will be applied always the time.
+
+The order of producing for each thread is :
+* allocate a new node with its tail points to the dummy node.
+* take a snapshot of the current tail.
+* try to append the new node to that snapshot tail.
+ * if succeed, lucky dog, producing is done.
+ * if failed, means some other threads already successfully inserted, current thread needs to redo the operation from a proper restarting point.
+
+This quite straight forward, after all threads finished their jobs, the deque will be look like this:
+
+
+
+A little weird? Looks more like a ringbuffer in a list form? Yes, it is, **a ringbuffer without capacity limitation, a ringbuffer with a dummy node indicating its empty status**.
+
+##### 2. Consuming
+
+The basic idea of consuming is very like that of producing, and keep in mind that the dummy node is a boundary.
+
+
+
+The overall process of consuming is trying to move `dummy->next` to its next one by one. Details will not be explained, reference the counterpart of producing, they are basically the same.
+
+##### 3. About deque's wait-free
+This is the tricky part of `LockFreeDeque`: To achieve the [wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm) semantic as much as possible, there is no boundary between the nodes being produced and the nodes being consumed. Therefore, a node can be immediately consumed so long as it emerged on the list regardless whether the `tail` pointer has passed over it or not, and a node can also be immediately produced once it became producible(empty), vice versa. This is an aggressive strategy, a double-edged sword, it pushes us more close to `wait-free` but also has its own downside: we cannot freeing a node just after it has been consumed since the node may still being used by other threads which are trying to move the `tail` cursor forward. The freeing operation has to be deferred: we first push the consumed node into a `garbage list`, a dedicated thread will polling periodically from it and the node be physically freed later.
+
+The `deferred freeing` procedure can also be found in the `TrivialLockDoubleList`.
+
+#### Lockfree MPMC queue
+
+This is the standard MPMC queue in a ringbuffer form. Also, for achieving the [wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm) semantic, a `working status` is introduced to each slot. There are several materials explaining the relationship and difference between `lock-free` and `wait-free`, talking about it is beyond the scope. Simply speaking, the `wait-free` semantics can be easily implemented on one unilateral side (producing or consuming), but hard on both sides. With the help of the `working status` the queue has been very close to it.
+
+Working status for a slot:
+* **Empty**: this is an empty slot (the initialized status), producing inside it is welcomed.
+* **Produced**: this slot is filled, consuming is welcomed.
+* **Producing**: this slot is under filling but is not filled, if one thread want to consume, it must wait for the resource to be available,aka to switch to the `produced` status.
+* **Consuming**: this slot is under consuming but is not finished, if one thread want to produce, it must wait for the slot to be empty.
+
+The initial state of the queue is like this, imagine its head & tail are connected as ring :
+
+
+
+##### 1. Queue Producing
+
+* judge if the queue is full.
+* move head to the next position in CAS manner.
+ * if failed:
+ * not due to full, retry at a proper position like before.
+ * due to full, return.
+ * if succeed:
+ * waiting if `working status` of the slot just occupied is not `Empty`
+ * set the `working status` of the slot to `Producing`, in CAS.
+ * producing is a simple pointer set operation, just do it.
+ * set `working statue` to `Produced` in CAS, allowing consuming operations on this slot.
+
+
+
+There is a possible state of the queue described by the above picture, producing threads pushed the head of the queue quickly to `slot 11`, leaving slots [0-10] behind with three possible status of each:
+
+* empty : just after pushing head, haven't change the slot status.
+* producing : changed the slot status, setting the pointer now.
+* produced : finished producing.
+
+**All the three status are valid to the thread consuming this slot**, it just waiting if the status hasn't been changed to `produced` by the producer. After the waiting, it start its own consuming. The behavior of P & C will making slots' status looks like being randomized.
+
+##### 2. Queue Consuming
+
+Consuming is just the counterpart of producing with very similar behaviors, the above knowledge for P is enough for understanding C. There is just a slight difference about judging the boundary, check it on the code.
+
+Consuming:
+
+
+
+##### 3. About queue's waitfree
+
+As previously said, `wait-free` is achieved on unilateral side:
+* produce : no threads have to wait until some other threads to finish producing, CAS pushing head to advance quickly.
+* consume : no threads have to wait until some other threads to finish consuming, CAS pushing tail to advance quickly.
+
+For both sides, it has been very close to `wait-free`:
+
+* produce-consume : consumers can always do consuming once a slot become available to be consumed with only one exception that, as showing in the above picture, some consumers have to stuck on `Node 6` & `Node 7` & `Node 8` to wait for their availability whereas `Node 9` & `Node 10` are already available for them. And this will not happen, if there are enough consumers. This is what `very-close-to` mean.
+
+#### Lockfree priority queue
+
+This is just a combination of a fixed size of the above MPMC queues with each representing a specific task type. It's an array for iterating by multiple PC threads.
+
+
+
+---
+This is all about the data structures in aurora.
+
+#### Basic workflow
+
+*Note: The `workflow` in the next few sections is meaning for the `writing workflow`.*
+
+Like the example shown in the [official document](https://grpc.io/docs/tutorials/async/helloasync-cpp/), aurora also wraps each request and its related variable & functions into a class called `RequestBase`. A concrete request class derives from which subclass of `RequestBase` depending on what kind of reqeust it represents. The basic hierarchy is:
+
+
+
+Leaf nodes are the classes represeting a concrete requests, their multi-level parents contain basic logics about how to complete a RPC.
+
+There are three main components in the asynchronous mode:
+* grpc [CompletionQueue](https://grpc.github.io/grpc/cpp/classgrpc__impl_1_1_completion_queue.html), the fundamental reactor, all event driven mechanisms are based on it, there can be many instances serving for the server at the same time.
+* The wrapped class of `RequestBase`, each request will entrust its IO intent to one of the CQs.
+* The threads polling on the CQ, these are the working threads and are the limited resources worthy to be treasured.
+
+In aurora's implementation, the most complicated part is the process of how to write a value to the state machine, obviously it's also the core component of the raft protocol. We'll take a deep look at the whole process.
+
+#### Follower workflow
+
+First, let's see how does the follower dealing with the `AppendEntries` requests:
+
+When an appending log request comes, the working thread first compare its `previous entity ID`, says `pre_id` with the `Last Replicate LogID`, says `LRL`, of the binlog:
+
+ * If `pre_id` < `LRL`, means a log conflict, starting the log reverting process.
+ * If `pre_id` > `LRL`, an out of order request has just come, pushing it into a `pending_list` which is a `TrivialLockSingleList` and designed to hold such kind of requests.
+ * If `pre_id` = `LRL`, good, this is the ideal case, cut all the entries in the above `pending_list` which are smaller than the current request's logID and inserting them to another list, says `committing_list`, finally appending them together to the binlog tail.
+
+The `committing_list` is designed for holding the logs which are succesfully appended but not yet committed for the commit phase. The committing operation will be done when a commit request arrived later.
+
+But how to deal with the disorder logs in the case of `pre_id` > `LRL`? Well, there are dedicated threads(says `Thread-X`) wating for new elements pushed into `pending_list` and then checking if there are any elements in the list satisfy `element's log ID <= LRL` which means the request has already been properly processed, if someone do satisfy the requirement, return the result for that reqeust. Since there are multiple threads doing this work, some CAS-based control flags are introduced. The weird thing here is that the requst was processed by another thread instead of the thread which firstly adopted it, the duty of finishing the request has been taken over! Yeah, this is on purpose, as it has been said before that the working threads is the most valuable resource, we cannot let them waiting on anything, otherwise the whole server performance will get easily downgraded or even stuck, the worse thing we don't want to see.
+
+*Note: The above checking for the condition of `element's log ID <= LRL` reveals another important fact that the status of the binglog, more specifically the `LRL`, is the only criteria that should be used on judging whether a request has been successfully processed or not, because it stands for the persistence of a request.*
+
+
+
+Okay, after the iterating thread found some request could be returned, it will invoke the corresponding methods wrapped in the request objects. And then start a new round of waiting, here is a diagram showing how does the threads interacting with each other:
+
+
+
+*`Thread 1-3` are the front threads processing, `Thread-X` is the backend one.*
+
+Now, we've got an detailed view of how does follower work on appending log entries.
+
+#### Leader workflow
+
+Before we stepping further into how does the leader work, we need to first have a glance at the two kind of pools used by leader:
+
+##### Connection pool
+
+In grpc, a connection is represented by a distinguishable(different channel arguments) `::grpc::Channel` object. Since HTTP2 is multiplexed we can share one `::grpc::Channel` object among several `Stub` objects(which stands for a `stream` in HTTP2), here is [detailed](https://stackoverflow.com/questions/47022097/should-i-share-grpc-stubs-or-channels/56375224#56375224) discussion about this.
+
+And aurora maintains a configurable fixed number of connection between each pair of combinations. These tcp long connections will be reused each time when it's needed.
+
+##### Client pool
+
+Clients are used on the leader side, all the RPCs leader issued to the followers are done through them. They may share the same channel, the same `CompletionQueue`, and different clients can customize their own reacting procedures depending on their own RPC logics. It's a high frequently used resources in the view of leader, allocating each time we need it will bring heavy overhead, so they are organized into a pool.
+
+The pool itself is an object pool powered by `LockFreeDeque`, support general opertions `Fetch` & `Back` for a pool. One thing to note here is the lifetime management stuff:
+* it takes part in its associated request object's lifetime mangement.
+* after being fetched from the object pool, the client object could become an orphan(no one is responsible for its lifetime) and being released sometime if we don't extend its life.
+
+`Lifetime management` is an smart mechanism to extend an object's life and recycle it at a proper time. Go [there](#lifetime) to get the detailed explanations.
+
+##### Workflow anatomy
+
+Okay, for now, let's look at how does the leader serve writing requests. First we need to understand some basic concepts:
+* `phaseX statistic` : a counter for the succeed/fail/... numbers in a replicating process and is maintained for each writing request .
+* `implicitly fail` : a `negative false` case indecator, basically standing for the *timeout* case.
+* `execution stream` : a series of jobs need to be done for dealing with a `AppendEntries` RPC resposne. Each stream corresponds to a certain response, different streams can proceeding parallel in different threads.
+* `determined result` : a majority result(whether success or fail) for a writing request, depending on which the leader can return a result to the client.
+
+First, leader generates a GUID(gloabl unique ID) for each writing reqeust, after then push the wrapped reqeust into the `LeaderView::m_entity_list` which is used for holding the unfinished requests. Then start the [parallel replicating](#parallel-replicating) phase which issues a `AppendEntries` RPC to each of the followers and the nodes who are trying to join in the cluster at the moment but still in a `JointConsensus`(not finished joining) state. The issuing requests job is done by the `CompletionQueue` of grpc, something implicit here is that the CQ is driven by the applications threads which means that only until there exists some threads polling(invoking `Next` or `AsyncNext`) on it, then the CQ will senting the requests out on the wire.
+
+*Digression: I just heard the classical `Dreamtale` from my sound*:
+> As the last ship sailed toward the distant horizon, I sat there watching on a rock, my mind slowly drifting away, forming in to my dreamtale...
+
+*It just woke my memories up which has been left behind 10+ years ago, in that beautiful campus. Can't holding myself from writing down this...*
+
+Since the leader working in a *multiple thread* and *asynchronous* manner, the next processes of dealing with responses for the above requests will be split into serveral independent `execution stream`s. One thread can taking the obligation of executing more than one stream as long as there exists unadopted responses in the CQ when the thread comes back from executing one stream and polling on it again.
+
+The most complicated thing in the current workflow is how does the threads work together, here is a diagram in the time sequence view to illustrate the above steps:
+
+
+
+Before explaining the `execution stream` in detail, we'd better know some features and temporary variables that will be needed:
+
+* `phaseII_ready_list`: a variable which is used for holding followers indicators that need to do the committing jobs later.
+* `permission flag` : a variable acting as a switch which ensure only one thread could take it(by successfully finished a CAS operation) and then step further to do the succeeding jobs.
+* `group commit` : This is a feature commonly used in many database product, which is intended to reduce the overhead caused by frequently committing requests especially in some two phase committing scenarios, the `Log Replication` here is just one of them.
+So you'll see not every succeed `AppendEntries` response will trigger a corresponding `CommitEntries` later, they will get grouped into one `CommitEntries` request when accumulated for a certain number.
+
+The `execution stream` contains the following steps:
+* update the `phaseI_statistic`.
+* judge if the current write request could get a `determined result` after the above updating:
+ * `Yes` : trying to get the `permission flag`:
+ * Got : Step futher.
+ * Lost : do nothing.
+ * `No`: judge if the number of logs which are replicated but not yet committed has reached the limitation of `group commit`.
+ * not reached : do nothing.
+ * reached : push the current follower entity into the `phaseII_ready_list`.
+
+For now, there is only thread for a certain write request could reach here, and it's the one who will trying to commit the written value to the leader's state machine. It will:
+* entrust all commit requests gathered before.
+* trying to CutHead a list of logs off from the `entity_pending_list` which is in the global leader view.
+ * if nothing is cut off, mean other threads with a greater log has done this. Push the current write request to another list(says ListX) which is used for holding such kind of requests.
+ * if something is cut off, appending them to the binlog and apply to the state machine.
+
+*Note: if one log got majority confirmed, all its preceeding logs must also be majority confirmed, this is guaranteed by the raft protocol, so ListX has a strong reason to waiting on all its elements to be appended to binlog finally.*
+
+The ListX here is essentially the same as the [pending_list](#pending-listX) mentioned before in the [Follower workflow](#follower-workflow) section. And also there are dedicated threads(says `Thread-X`) iterating over it in a high frequence checking if there are any entries satisfies some certain conditions and return a result to client if someone does. The interaction between the thread(says `Thread-Y`) who successfully appended item to binlog and `Thread-X` is very like that of the follower's : `Thread-Y` notify `Thread-X` that the `LRL` has advanced and there may exist some requests that can be returned to the client.
+
+Ok, let's recap with an overview about what the whole `execution stream` procedure look like:
+
+
+
+Finally, I'd like to emphasize an special design for getting rid of [this implicit behavior of grpc](#client-timeout-if-dont-fetch) introduced before, in short, client side CQ will report a timeout error if you don't fetch the result quickly. What prevent us from fetching it out ASAP is that the polling threads may being busy at other things, no time to taking account the about-to timeout responses and also can't fetching it with a higher priority than other tasks remained in the CQ. Thus we have to give dedicated CQs & threads to meet the special requirement of client responses:
+
+
+
+This is the design for the above purpose, the `Backend-CQ-*` & their corresponding threads(says `threads Y`) are **only** responsible for reacting with client responses, and not the other way around. Further, to keep the work of `threads Y` as simple as possible which also contributes to fetching responses ASAP, `threads Y` will wrap the reacting info by which can find the remaining works into a struct and push that struct into a [priority queue](priorify-queue)(says `PriQueue X`), so at last, it will be the background threads(says `threads Z`) who are polling on the `PriQueue X` will indeed do the subsequent jobs. So threads of `F1-Fn` in the previous picture not the threads directly polling on the CQ, they are actually `threads Z`, a little winding.
+
+### Asynchronous Framework
+
+#### Threading model
+
+Let's recap all kinds of threads in aurora server:
+
+* Grpc inner threads: related to grpc inner designed, we don't need to spend too much time on it.
+* Woking threads: as mentioned above, these are the real wokers doing RPC jobs. All network io operations need to do within the threads are entrusted into a `CompeletionQueue` and thus turned into asynchronous:
+* Background MPMC threads: these are the consumers quickly fetching jobs from the MPMC queue:
+
+
+
+* Golbal timer thread: there is a special thread periodically doing the jobs registered in a heap, details are [here](#global-timer).
+* Follower background thread : As described above, this is the dedicated thread to iterating over the disorder message list. It plays an important role in the `AppendEntries` RPC.
+* Leader background thread : the leader dedicated thread to iterating over the `LeaderView::m_entity_list`, checking and return a result to client if necessary.
+
+#### The CGG problem
+
+CGG shorts for `Concurrent generating GUID`.In raft protocol, each log entry has its own globally unique ID, either the `log index` or the `[log term + log index]` pair will acting as the global factor. But, what's the problem with CGG? Well, there will be no problems with it under normal situations, but there will be problems if we got an issue and the GUID still keep increasing monotonically: considering that one log entry get failed when replicating to the followers due to network unstable, and since GUID increasing monotonically, all its subsequent logs will get a greater ID, these ID will get failed again, because raft force all successfully written IDs to be continuous but their preceeding one is lost. For this kind scenarios, **there will be no changes for the leader to recover, a single error would disable the entire cluster from serving new writing requests**.
+
+Thus we **MUST** have a mechanism to make the leader being recoverable from a single failure, more specifically, to tune the `Last Released GUID` (says `LRG`) back to the last successfully replciated LogID if we detecting a failure.
+
+There are two kinds of failures for the leader during replicating:
+* `explicit failure` : the requesting side knows exactly about that its request cannot be done on the receiver side, e.g., the receiver send a msg back telling that your request is invalid or the request triggered some unexpected logic on the receier side and therefore aborted. There are few cases for this kind of failure.
+* `implicit failure` : the sender didn't get a response from the receiver within a certain period of time and resulting in a timeout error. This is the most possible kind of failures in the real production environment.
+
+The `implicit failure` is worthy to be taken care of carefully: we'd better not immediately telling the client that:"your write request failed" simply by encountering an `implicit failure`, because it may actually succeed: the response just arrived a little later than the timeout deadline but it indeed has been executed on the receiver side, this is a common `false negative` scenario for all the timeout errors. One way to mitigate the disruptions from it is that **making decisions only after the `LRG` got a determined result**.
+
+Why it relies on `LRG`? Let's see this:
+
+
+
+There are four status for a GUID or a log entry in the leader side:
+* Committed : applied to the state machine.
+* Replicated : successfully replicated to the majority of the cluster but not yet committed.
+* Sent but not confirmed : trying to be replicated to the majority, but hasn't been finished.
+* Not Released : not yet generated.
+
+And according to raft safety guarantee, we can conclude that :
+
+**1> a failed log means all its succeeding logs also failed** :
+
+
+
+In this case, we cannot say that the preceeding undetermined logs(2 & 3) also failed just because of `log4` failed. Threads processing `log 2 & 3` will indeed waiting on a condition variable(CV) which will be notified after `LRG` resolved to determine its own result.
+
+**2> a succeeded log means all its preceeding logs also succeeded** :
+
+
+
+If we can be sure that the `LRG` succeeded, then all its preceeding undetermined logs can be also deducted to succeed.
+
+So, to determine the result for the logs in the `Sent but not confirmed` state, we can firstly determind the last log's result, this process also being called as `LastLogResolve` in aurora:
+
+
+
+As said before, this strategy cannot ensure that all the `implicit failure` logs could get a accurate judgement on whether it's failed or not, but it can help reducing the `false negative` cases.
+
+So the final steps dealing with an `implicit failure` are:
+* Step1. Set server status to `halted` to stop receiving new incoming requests, this is to stop server from releasing new GUIDs.
+* Step2. Waiting for the last GUID to be generated.
+* Step3. Waiting for the log with the last released ID to be resolved.
+* Step4. Determine the failed log final result by compare the `LRL` with current logID:
+ * `current_log_id` > `LRL` : return a negative result to clients.
+ * `current_log_id` < `LRL` : the current log actually succeed, return a positive result to clients.
+
+The reason for `Step2` is that there is a tiny time windows where the server status has been set to `halted` but the `LRG` hasn't been generated due to the multiple thread environment. Thus we need to sleep for a little while to get the real *last* one when need it.
+
+For `Step3`, there is a `gap` varibale indicates how many request are there between `LRL` and `last GUID` at the moment server status is set to `halted`, and another variable `gap_processed` which stands for the #requests that has already been processed(whether successfully or unsuccessfully) within that section.
+
+And the `waiting` operation means for waiting in a spin manner for the following condition to be satisfied: `gap_processed` >= `gap`. Adding that both `gap_processed` and `gap` are relative accurate values, we can't make them absolutely right because the simultaneously way of working. So there is a patch mending the inaccuration: waiting for a configurable value to ensure the `gap_processed` >= `gap` condition finally matched. This kind of waiting is absolutely not a perfect solution, but it's still acceptable considering the time already spent on dealing with the CGG problem.
+
+(~~waiting on a CV with a reasonable timeout value after which elapsed the leader will just return a negative result to the client anyway. The waiting condition on the CV is very like that in the [leader & follower's dedicated thread](#bg-thread) : waiting for the `LRL` to be advanced over its current logID~~).
+
+#### Lifetime management
+
+There are scenarios where we need to extend the lifetime for certain objects :
+
+* `Write` represents for the request that are currently being processed by the leader. There will be a result return to the client immediately after the request get a majority result, but there are still other jobs to do at the moment, like finish the replication to the remain followers and start phaseII when necessary. All these subsequent jobs need a living `Write` object.
+* `Write` & `AppendEntries` both need a living object to finish the `checking & return` related logic in the backend thread.
+* some clients like `AppendEntriesAsyncClient` & `CommitEntriesAsyncClient` need to callback into the corresponding living request objects.
+* some background tasks like `CutEmptyContext` & `DisorderMessageContext` need the request context of the iving request objects.
+
+The simplest way to achieve life extention is to use a `shared_ptr`, besides the automatic extending of object's lifetime we can also facilitate the thread safe property of its control block, which is great helpful in the multi-thread environment. There is one thing need to pay attention to : we must have an initiate `shared_ptr` object to hold the ownership and acting as the original assigner for the first ownership copy request, since there is no fixed originations to serve such kind of requests and paly the role as a sustainable holder.
+
+The solution for the initiate `shared_ptr` is to group the lifetime management logics into a wrapper of `OwnershipDelegator`, some core concepts:
+* the inherited classes will share the management right with the wrapper.
+* the ownership can be copied out as a form of extending the object's lifetime.
+* the inherited classes can either be destructed by its own destructor or by the wrapper, but **NOT** both.
+
+The basic structure of the wrapper looks like this :
+```c++
+template
+class OwnershipDelegator {
+public:
+
+ OwnershipDelegator() { this->m_p_shp_delegator = new std::shared_ptr(nullptr, [](T *p) {}); }
+
+ virtual ~OwnershipDelegator() { delete this->m_p_shp_delegator; }
+
+ void ResetOwnership(T *src) noexcept { this->m_p_shp_delegator->reset(src); }
+
+ void ReleaseOwnership() noexcept { this->m_p_shp_delegator->reset(); }
+
+ std::shared_ptr GetOwnership()noexcept { return *this->m_p_shp_delegator; }
+
+ void CopyOwnership(std::shared_ptr from)noexcept { *this->m_p_shp_delegator = from; }
+
+private:
+ std::shared_ptr *m_p_shp_delegator = nullptr;
+};
+```
+
+#### Parallel Replicating
+
+This is a reveiw and re-emphasize on all the above techniques applied to enhance the parallelism and further the throughput. Parallel replicating is the most complicated part of `Log Replication`. To enhance parallelism as much as possible, taking advantage of multicore is imperative: leader duplicating the writing requests to the followers as faster as it can, saturating the downstream nodes, after which the bottle neck of throughput should lying on the disk operations, theoretically.
+
+*Note: After got an unexpected [benchmark result](https://mail.google.com/mail/u/0/#sent/KtbxLxgKJJPkwvGplJvRNNrQlPCVDjKDGV) of grpc, the actualy bottle neck is now on grpc itself.*
+
+Some great efforts have been made to approach this goal:
+* use an asynchronous mode of grpc.
+* support processing disorder message on the follower side.
+* make the processing for write requests as independent as possible.
+ * several lockfree data structures are introduced.
+ * free the working threads when they about to do synchronouos operations(typically are the waiting(on CV) opertions for `LRL` to advance) by using dedicated background threads to finish the inevitable intersection parts of the processing.
+* some general optimizations:
+ * use a channel(connection) pool & client pool to keep long connections between leader and follower.
+ * avoid memory copies as much as possible.
+
+### Global timer
+
+There are some background job like sending heatbeats to follower, checking heartbeats from leader, garbage collecting, etc, need to be periodically executed, thus a dedicated global timer thread is introduced for this purpose.
+
+The task itself is represent with three fields:
+
+* a timestamp indicating the next time it should be executed.
+* a interval_ms representing the intervals between each execution.
+* a function object describing what the task actually is.
+
+Tasks are organized as a *min heap*, and the task that is the most recent to do will be located on the root of the heap.
+
+The timer thread worked in such a way that:
+
+* first, checking the timestamp(say *T*) value of the root node:
+ * if *T* > *now*, means the execution time it's not yet came, continue.
+ * if *T* >= *now*, do the job by invoking the callable function object.
+* after successfully finished a job, checking whether it need to be scheduled again. If so, calculating the next timestamp of its execution by the interval and insert the new task back to the heap.
+
+
+
+Note: *In this design, the thread acting as both a scheduler and an executor, thus task execution may get delayed if its previous task takes a long time to finish. But in the current scenarios, there is no such kind of task registered into the heap, so it's okay to just let it be.*
+
+### About The Binlog
+
+This is the log logging for raft's log entries and also acting as the [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) in the leveled storage subsystem.Let's first take a look at the file format:
+
+
+
+#### Meta Data Part
+
+The meta data is the index of the data part, locating in memory, callers can easily find the corresponding offset of each entry. The meta part is appended to the binlog each time the binlog file is considered large enough to be dumped to disk, since its length varies according to how many records it contains, it must be located at the end of the file rather than at the head of the file. The footer is an identifier identifying the completeness of the file.
+
+#### Data Part
+
+This is just several log entries each serialized by [protobuf](https://github.com/protocolbuffers/protobuf), concatenating together, inside each record, there is an additional 4 bytes to show the length of the serialized buffer.
+
+Basic fields for each log entry :
+* key : the key.
+* val : key's value.
+* ID : to identify the entry itself.
+* pre-ID : to identify the entry it bases on.
+
+Raft's logs are in strict order, each one is based on its previous one, contributing the `consistency semantics` in the distributed environment as a whole, for more details see the [raft paper](https://raft.github.io/raft.pdf).
+
+*Note: The `previous` term above is on an `ID monotonicity` basis, e.g., logA is the previous one of logB, then logA's ID must satisfy LogAID < LogBID - 1, but not necessarily equals.*
+
+#### Supported operation
+
+There are operations binlog need to support :
+* Append : the most basic operation.
+* Revert : in raft, existing log entries could be rewrite for several reasons (like a new leader's log is inconsistent with followers), reverting will be called once in a while.
+* SetHead : in general cases, binlog file must contain at least one entry to be the previous entry of its next following. Thus once a new empty follower joined the cluster and finished synchronizing data from the leader, the last data item just synchronized will be set as the first entry, by the SetHead operation.
+
+### Storage Layer
+
+Let's take an overview first:
+
+
+
+As a KV database where throughput is a critical factor, a leveled manner storage layer has been chosen. You can think of it as a simpler version of the [leveldb](https://github.com/google/leveldb). Just skipping this section if you are already familiar with that.
+
+There are two components in the storage subsystem:
+
+* memory table : a hash based unorderd in-memory data collection, in lockfree manner.
+* SStable : sorted string table with index enabled, nothing specific than the typical [sstable](http://distributeddatastore.blogspot.com/2013/08/cassandra-sstable-storage-format.html).
+
+There are few differences in file format between sstable and binlog. They are sharing the same
+design philosophy:
+* data items starting from at the beginning of the file.
+* meta data and the file's completeness identifier(footer in sstable) are appended after the data zone.
+
+
+
+Some optimization features(like the `minimal key prefix compression`, where the data area is divided in to several smaller groups inside each of the group there is a `common prefix`(usually selected as the first key) for all the keys in that group, the data items' key only stores the tailing part, sharing the common prefix, thus reducing the space requirements for storing data), are not supported yet, since the sstable file size is not a critical factor for now.
+
+Choosing an unorderd data structure acting as the memory table has its own pros & cons compared to a typical choice: an ordered data structure, like a [LSM Tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree).
+
+* Pros:
+ * writing complexity downgraded to O(1) from O(log2N).
+ * can taking the advantage of the existing lockfree feature for writing operations.
+* Cons:
+ * Can't do [rolling merging ](https://blog.acolyer.org/2014/11/26/the-log-structured-merge-tree-lsm-tree/) when migrating data from the upper level(C0) to the lower level(C1) which has a more fine grained control over it.
+
+The memory tables from `memory table 2` to `memory table N` can seldom been seen as they only occur when the previous memory tables still haven't been finished dumping at the moment they are going to be dumped. The dumping process for each memory table can proceed simultaneously, this also bring some tricks into the code:
+
+
+
+First of all, when a memory table(says `M`) finished dumping, it needs to cutoff from its least ancestor table(says `N`) by changing the `next` pointer of `N` from pointing to `M` to pointing to null. But some memory table(s,like `memory table 3` in the above) can be finished dumping earlier than its succeeding ones(`memory table 4` till `memory table N`, says `Q`), if we do cutting off immediately at the moment(says `T0`), there is a time windows starts from `T0` and ends till to all the subsequent memory tables finished dumping(says `T1`), during which querys from clients will not get any data in the tables of `Q`. Thus we have to wait for `T1` reached before cutting off `N` from `M`, this is done by a CAS operation.
+
+The sstables will be periodically merged together from the oldest to the lastest. This is as the same with the typical merging semantics of [LSM Tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree).
+
+Since the storage layer sharing so many concepts and features with leveldb, you might ask why just using the leveldb library instead of implementing an owned version? Well, there two important reasons to do this:
+* we already have a binlog which records exactly what the `WAL` needs, we must to utilize it getting around of leveldb doing a duplicated job.
+* with a customized memory table, we can take the advantage of lockfree feature for writing.
+
+### GC
+
+
+### Election
+
+#### The basics.
+
+Election is the second fundamental component of the entire raft protocol. Detailedly discussing about it is beyond the scope, just keep in mind of the following basic principles:
+* *Term Based* : Different round of elections are distinguished by their terms. Only one leader could be elected for one specific term, whereas **two leaders can existing at the same time in different terms**.
+* *Majority Based* : Contributing to the above principle.
+* *Heartbeat Timeout Triggered*: A follower becoming candidate once it cannot hear from the leader.
+* *Election Timeout* : If one candidate can't win an election in a given term for a specific period of time, it will falling into a random sleep and restarting the election after that.
+* *Respecting With Higher Term*: When a node learns about(by whatsoever means) that there is another node which has a higher term than its own, it will definitely step down to a follower status and conforming instructions from the node it just learned about.
+
+#### About prevoting.
+
+Pre-vote is an additional step for the candidates before they issuing the real voting requests. Purpose of this is to prevent the occasionally disconnected followers from disrupting the entire cluster: consider a network issue occurred between the leader and one (says FX) of the followers, but not all the others. FX will starting an election sending voting requests to the other nodes, but the request will certainly fail since the it's already been disconnected from the cluster, then FX gaining its term and starting a new round election again, this process repeating until network heals. After the healing, all nodes know about FX is requesting vote with a higher term and they step down to followers showing respects to the higher term. Now all the nodes take part in the election, and FX cannot win since it has been disconnected for a period of time during which the logs could have advanced a lot. Each of the other nodes could be elected as the new leader.
+
+But the whole thing is stupid because the old leader is actually without any problem to serve the clients, FX is just a trouble maker.
+
+To work it around, every time a node thinks it has to issue a vote request, it must firstly ask the others that: *do you agree me to start an election*? Only if the node being asked also detected leader's gone, could it response with "Yes", otherwise "No".
+
+#### The live lock problem & solution.
+
+There is one widely known shortcoming of raft's election design: the live lock problem. Besides the one that the [raft paper](https://raft.github.io/raft.pdf) has mentioned:
+
+> **Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly**. To prevent split votes in the first place, election timeouts are chosen randomly from a fixed interval (e.g., 150–300ms). This spreads out the servers so that in most cases only a single server will time out; it wins the election and sends heartbeats before any other servers time out. The same mechanism is used to handle split votes. Each candidate restarts its randomized election timeout at the start of an election, and it waits for that timeout to elapse before starting the next election; this reduces the likelihood of another split vote in the new election.
+
+which can be nearly solved by setting a randomized timeout value before starting each round of election, yet there is still another serious problem I came across, suppose a scenario like this:
+
+The leader crashes at some point leaving the followers with different log entries at the moment:
+
+
+
+`Server 3` & `Server 4` & `Server 5` are all possible to be elected as the new leader since each of them can get a majority vote. But will they definitely being elected out just because of this? What if the servers with a lower index(says ServerLOW) increasing their terms **always faster** than the servers with a higher index(says ServerHIGH)? See below:
+
+
+
+*Note: the sleeping times in each server doesn't need to be the same with each other, they are drawn as equal just for convenience.*
+
+In the picture, all nodes turned into candidate state and going issue an election. ServerLOW *always* have relatively smaller sleeping intervals compared to its ServerHIGH between each round of election(represented by green), leading to each time ServerHIGH starting to do election under a certain `term X`, they will found that `term X` cannot get a majority vote due to one of the two reasons:
+* the term has already been voted by its ServerLOW to themselves as mentioned in the paper:
+
+> To begin an election, a follower increments its current term and transitions to candidate state. It then votes for itself and issues RequestVote RPCs in parallel to each of the other servers in the cluster.
+
+ * the lastest log is earlier than the nodes it sending the vote request to.
+
+This process continues resulting into either none of nodes in `Set 2` could win an election or nodes in `Set 2` win an election but with a long time. Neither of them are wanted by the applications.
+
+Here we named the above abnormal phenomenon as `diff-speed-caused abortion`, shorted as `DSCA`.
+
+For `DSCA`'s happening, the election existing sequence for each term doesn't need to be exactly the same as the above picture illustrated. It just need to make sure that when each of the nodes in `Set 2` start their elections, the nodes in ServerLOW can just rightly prevent its success. E.g., when `Server 3` start electing, just needing one member from `Server 1` & `Server 2` refuse the vote, written as `[1,2] !-> #1`, the other cases are:
+* `[1,2,3] !-> #2`, for `Server 4`.
+* `[1,2,3,4] !-> #3`, for `Server 5`.
+
+You might retort to say that this is still an extreme case and will rarely to be seen. For the precondition of that *each server with different log entries when the leader crashes (as the pictured described)*, right, it is not common to see. But we'd better focus on the election protocol itself instead of presuming anything before it. And for the protocol itself, I tried to give an formula to describe the relations between the probability of the `DSCA` and several other factors(e.g., the lower and upper bound of sleeping interval as Smin and Smax, the time elapsed as T, the number server nodes as N), joined together as:
+
+
+
+
+
+, but found it was **a quite complicated mathematic problem** (contributions are very welcomed). However, in [this unit test](../src/gtest/election/test_election.h), I indeed came across that phenomenon many times which impelling me to take an action, finally a solution worked out like this:
+* Each candidate locally recording the terms which are carried by the PRC `preVote` and `vote` when they are invoked, as `used terms`.
+* When a candidate issuing a new round of an election, it increasing its term by skipping over the `used terms` it recorded.
+* If an election fail, the candidate set its term's value back to the value where it started.
+
+For example, a candidate says C0 initially with a term value *3*, but it knows about that *term 4* and *term 5* have already been used by the other candidates as their new electing terms, thus C0 will start its next round of election by using *term 6* instead of the adjacent *term 4*, mitigating the conflicts.
+
+Under my test, this solution perfectly solved the `DSCA` problem. Now, it's almost impossible for the terms to be conflict with each other during each round of elections.
+
+### Membership change
+
+Now we're coming to the last but most hard to understand part of the raft protocol. I suggest you to read the chapter of `6 Clustermembershipchanges` in the raft paper in depth before reading this section because of there are several differences in aurora's implementation, you need to have those basic knowledges to understand why these differences are introduced and still being correct.
+
+The phrase *membership change* means that the whole cluster being available to the clients and can perform the functions of `log replication` and `leader election` correctly while it's changing the topology by adding/removing nodes from/to it. Essentially, this is a two phase commit procedure, and it the paper it was suggested to be done with inserting the configure changing events into the log entries, this approach is great in the perspective of taking advantage of the existing binlog's version control(the changing event itself is represented as a log entry) and quite easy to implement. But in aurora's implementation, that *2PC* process is extracted out as an independent module to keep the binlog related logic as pure as possible. This brings some extra work to do:
+
+* two separate RPC interfaces(`PropagateMemberChange` & `MemberChangeCommit`) are needed.
+* each server in the cluster maintains a separate configure(`membership-change.config`) for its membership status.
+* the candidates take the version of membership configure into account when deciding whether to vote for a node or not.
+
+The version of a membership configure plays an important role since it's used to find out which configure is newer in elections. It simulates the logID as if the changing events are transferred with log entries.
+
+Another difference in aurora is that the log entries are firstly being replicated to the majorities and will not be appended to the local binlog until they got majority confirmed, this order is the reversed version of that in the paper, and this reversing also applies in the membership change module: the Cnew will be broadcasted to the cluster at first, then the leader waiting for the majority confirmation, only after that the Cold,new would take effect.
+
+Now, let's recap why the `joint consensus` is needed, first thinking about a scenario without the *joint consensus*:
+
+
+
+The cluster wants to shrink from 7 nodes (leader & F1-F6) to 3 nodes (leader & F1-F2), the leader replicate the change event to F1-F6, and suddenly crashes leaving only F1 receives that change event. Now, in F2-F6's point of view the cluster still having 7 nodes but there are only 3 nodes in from F1's view. As a consequence, F1 could be elected as the new leader with votes from F1 & F2, in the meantime F3 could also win an election under the same term with votes from F3 & F4 & F5 & F6. Similar cases could also be enumerated **as long as Cold and Cnew can making unilateral decisions at the same time**. This is why `joint consensus` are introduced in raft.
+
+The next difference is that aurora simplifies the status transition from a process of
+
+Cold --> 2PC --> Cold,new --> 2PC --> Cnew , to :
+
+Cold --> Cold,new --> Cnew
+
+The omitted 2PCs are indeed unnecessary, raft using it may because of just making the boundaries clear. The new approach is now looking like this :
+
+
+
+Note the starting point of where Cnew `can making decisions alone` exists later the original one from the raft paper.
+
+Some other differences for dealing with problems in `membership change` are:
+* new servers may not initially store any log entries : leader will first syncing data to all the new nodes and **won't start replicating the configure change event until the new nodes are all fully synced**.
+* to prevent removed servers (those not in C-new) from disrupting the cluster : after committing the configure change event, a node will **shutdown itself once detected that it's no longer in the new cluster**, making life easier.
+
+## Contributing
+
+There are about 20k lines of code in the first version in aurora. But they are in a well organized directory structure, each directory & file & class name are straight forward, I believe you can get the point of a code snippet just by taking a look at its upper wrapper's name.
+
+### Future work
+
+#### Improvements
+
+In raft's design, all read & writes requests are forwarded to the leader, it's an obvious bottle neck of the whole system. There are several architecture level optimizations we can do.
+
+##### 0. support cmake
+
+Awkward...But it hasn't been supported yet...
+
+##### 1. (done) improve working threads' way of polling
+At present, for the `AppendEntries` interface on the follower side, the working threads have to block on a condition variable when some out of order messages arrived, and will be and only be notified when some certain conditions(in this scenario it's the accomplishments of processing its preceding messages) matched. This resulting into the working threads wasting their time, and getting worse as the number of the out of order message arrivals increasing, this can be get around by inventing some new polling mechanisms for the working threads :
+* there is a back ground task queue designed to hold uncompleted `AppendEntries` requests.
+* the working threads wrapping the current job into a task and delivering it to the above queue when they found they need to wait on a CV, and then go back to polling on the CQ.
+* there are dedicated threads ready to handle the wrapped task and take over the waiting and its subsequent jobs.
+
+This won't help in reducing the average latency for requests, but can releasing the occupied working threads, further improve overall throughput, also introducing a more complex program design, of course.
+
+*Data sharding is an old topic in database design, you can seek for other articles about this topic.*
+
+##### 2. LockFreeDeque dynamic shrink & expand
+
+This is for the varing requests payloads a leader may face. There is a time slice between the client is fetched and client is returned back to the client pool which is implemented by the `LockFreeDeque`, otherwise there is risk of exhuasting client resources during heavy enough payloads.
+
+##### 3. Storage.
+
+###### 3.1 SSTable do not maintain index data in memory.
+
+In the current design of the storage layer, each sstable object maintains the index part(the offset for each key) for each sstable file in memory. Thus could result in unlimited memory usage if data grows always faters than GC's reclaim. This is a legacy problem for an original intention of accelerating sstable access. A better approach is the compromise of the two factor of *memory usage* and *access speed* : only maintain the first several sstables' index data in memory, and leave them alone for the others.
+
+Besides, we also need a way to stop sstable files' size from growing unlimitedly as the consequence of [periodically merging](#storage-gc).
+
+###### 3.2 Disk space
+
+The meta data part of a SSTable is relatively large for now. We need to figure out a data compression solution for the meta part to save disk space usage.
+
+###### 3.3 Memory table
+
+###### 3.3.1 Dump speed
+
+It will cost ~3.5 seconds(win10 debug) to dump a 20K items memory table to a SSTable. The iterating process in the `LockFreeHash::GetOrderedByKey::_rb_tree` is the bottle neck, haven't figure out a way to get around of using the `_rb_tree` temporary variable to customize the order comparator.
+
+###### 3.3.2 Limit by memory usage.
+
+There is only a `memory_table_max_item` config to control when to dump the memory to SSTable, but is hard to decide the number for the users. Thus something like `memory_table_max_memory_mb` is needed to give an option that can be used to control by size of memory it used.
+
+##### 4. Binlog GC.
+
+Number of binlog files will increasing as data grows, in the current design, only at the startup time for the server can we know which binlog files can be deleted, manually. We need an elegant way to delete the big files as promptly as possible to free dick space.
+
+##### 5. Bi-directional streaming rpc.
+
+The communications between leader & followers are now in a way of unary rpc. Throughput is kinda lower compared to the way of bi-directional streaming rpc. It's worth improving it in that way, and this can alos mitigate [this problem](https://github.com/grpc/grpc/issues/19658).
+
+#### New features
+
+##### 1. apportion reading requests
+
+Several possible solutions:
+
+* Followers itself can also serve the read requests by query the lastest committed value for a key to the majority of the cluster. But this will multiply the network traffic by N times.
+
+* Adding slave(s) for the leader, committed data will be replicated to these nodes in either a strong consistency or an eventual consistency manner, depending on the needs. Those nodes don't take participate in the election and log replication phase. This is a more feasible choice.
+
+##### 2. data slicing with multi-raft
+
+This is the ultimate solution for horizontal scalability. The data set that aurora served will be sliced into different pieces, upon each of them there will be a raft instance running. Therefore we can deploying many raft instances among a fixed number of machines, any node in the cluster can acting as the leader for a certain set of data, at the meanwhile acting as a follower for another set of data. Different instances running independently from each other, as a whole, making use of the bindwidth of the local network as much as possible. The challenging part is the slicing strategy, different applications get different inclinations. In general, consistent hashing should be the compromised and default choice.
+
+##### 3. multiple value format
+
+The value formats can be diversified to a wide range of choices, like list, set, ordered set in redis, consider the follow things when plan to import any of them:
+* how to storage data in this format on disk in an efficient way.
+* how to storage data in this format in memory.
+* design suitable operations on this format.
+
+##### 4. [distributed] transactions
+
+This is a very big topic and really hard to get right. Just list some considerations :
+* redo & undo mechanisms on a single node transaction view.
+* 2PC on a global transaction view.
+* a global unique monotonic transactions ID.
+
+##### 5. dockerize
+
+Can be released & deployed as a whole by being a docker service.
+
+### Naming Conventions
+
+Aurora conforms to the [google C++ style guide](https://google.github.io/styleguide/cppguide.html) with some exceptions of:
+* local variables are prefixed with a single underscore.
+* class & function names are in camel style.
+
+### Unit Test
+ Make sure all the unit cases related to your modification are passed before committing. Add new test cases if new code or module are added.
+
+### Development history
+Consult [this](https://gitlab.com/pplorins/aurora/blob/master/doc/dev_log.md) to see the very detailed development history, it remains open most of the time.
+
+
diff --git a/doc/glossary.md b/doc/glossary.md
new file mode 100644
index 0000000..7f0d174
--- /dev/null
+++ b/doc/glossary.md
@@ -0,0 +1,7 @@
+|Term|Short For|Scenarios|
+|--|--|--|
+|LRL|Last Replicated Log|Leader and Follower server and theirs binlog status|
+|ID-LRL|ID of the Last Replicated Log|ID of the Leader and Follower server and theirs binlog status|
+|ID-LCL|ID of the Last Committed Log|ID of the Leader and Follower server and theirs binlog status|
+|LRG|Last Released GUID|The lastest generated guid for write operation in Leader|
+|CV|Condition Variable,especially refers to the std::condition_variable|many multiple threading places|
\ No newline at end of file
diff --git a/doc/images/MPMC-thread.png b/doc/images/MPMC-thread.png
new file mode 100644
index 0000000..1bbe49a
Binary files /dev/null and b/doc/images/MPMC-thread.png differ
diff --git a/doc/images/benchmark-latency-datalen.png b/doc/images/benchmark-latency-datalen.png
new file mode 100644
index 0000000..f70ef44
Binary files /dev/null and b/doc/images/benchmark-latency-datalen.png differ
diff --git a/doc/images/benchmark-latency-followers.png b/doc/images/benchmark-latency-followers.png
new file mode 100644
index 0000000..2ddb2f7
Binary files /dev/null and b/doc/images/benchmark-latency-followers.png differ
diff --git a/doc/images/benchmark-throughput-datalen.png b/doc/images/benchmark-throughput-datalen.png
new file mode 100644
index 0000000..2760431
Binary files /dev/null and b/doc/images/benchmark-throughput-datalen.png differ
diff --git a/doc/images/benchmark-throughput-followers.png b/doc/images/benchmark-throughput-followers.png
new file mode 100644
index 0000000..09ac59e
Binary files /dev/null and b/doc/images/benchmark-throughput-followers.png differ
diff --git a/doc/images/binlog-coordinating.png b/doc/images/binlog-coordinating.png
new file mode 100644
index 0000000..6bd3e3c
Binary files /dev/null and b/doc/images/binlog-coordinating.png differ
diff --git a/doc/images/cgg-1.png b/doc/images/cgg-1.png
new file mode 100644
index 0000000..9ab41f6
Binary files /dev/null and b/doc/images/cgg-1.png differ
diff --git a/doc/images/cgg-2.png b/doc/images/cgg-2.png
new file mode 100644
index 0000000..1ebf150
Binary files /dev/null and b/doc/images/cgg-2.png differ
diff --git a/doc/images/cgg-3.png b/doc/images/cgg-3.png
new file mode 100644
index 0000000..ea4e60e
Binary files /dev/null and b/doc/images/cgg-3.png differ
diff --git a/doc/images/cgg-4.png b/doc/images/cgg-4.png
new file mode 100644
index 0000000..0430b03
Binary files /dev/null and b/doc/images/cgg-4.png differ
diff --git a/doc/images/datalen-factor.png b/doc/images/datalen-factor.png
new file mode 100644
index 0000000..b0c2e25
Binary files /dev/null and b/doc/images/datalen-factor.png differ
diff --git a/doc/images/deque-pop-1.png b/doc/images/deque-pop-1.png
new file mode 100644
index 0000000..1e2f084
Binary files /dev/null and b/doc/images/deque-pop-1.png differ
diff --git a/doc/images/deque-pop-2.png b/doc/images/deque-pop-2.png
new file mode 100644
index 0000000..44d6e36
Binary files /dev/null and b/doc/images/deque-pop-2.png differ
diff --git a/doc/images/deque-push-1.png b/doc/images/deque-push-1.png
new file mode 100644
index 0000000..56fd351
Binary files /dev/null and b/doc/images/deque-push-1.png differ
diff --git a/doc/images/deque-push-2.png b/doc/images/deque-push-2.png
new file mode 100644
index 0000000..5c1b916
Binary files /dev/null and b/doc/images/deque-push-2.png differ
diff --git a/doc/images/deque-push-3.png b/doc/images/deque-push-3.png
new file mode 100644
index 0000000..6f207f1
Binary files /dev/null and b/doc/images/deque-push-3.png differ
diff --git a/doc/images/election-overview-1.png b/doc/images/election-overview-1.png
new file mode 100644
index 0000000..2618870
Binary files /dev/null and b/doc/images/election-overview-1.png differ
diff --git a/doc/images/election-overview-2.png b/doc/images/election-overview-2.png
new file mode 100644
index 0000000..28ca675
Binary files /dev/null and b/doc/images/election-overview-2.png differ
diff --git a/doc/images/follower-iterating-thread.png b/doc/images/follower-iterating-thread.png
new file mode 100644
index 0000000..4883779
Binary files /dev/null and b/doc/images/follower-iterating-thread.png differ
diff --git a/doc/images/followers-factor.png b/doc/images/followers-factor.png
new file mode 100644
index 0000000..19ef3fc
Binary files /dev/null and b/doc/images/followers-factor.png differ
diff --git a/doc/images/formula.png b/doc/images/formula.png
new file mode 100644
index 0000000..d2577c2
Binary files /dev/null and b/doc/images/formula.png differ
diff --git a/doc/images/global-timer-thread.png b/doc/images/global-timer-thread.png
new file mode 100644
index 0000000..a1dc5d6
Binary files /dev/null and b/doc/images/global-timer-thread.png differ
diff --git a/doc/images/guid.png b/doc/images/guid.png
new file mode 100644
index 0000000..f8093a3
Binary files /dev/null and b/doc/images/guid.png differ
diff --git a/doc/images/hash.png b/doc/images/hash.png
new file mode 100644
index 0000000..6371778
Binary files /dev/null and b/doc/images/hash.png differ
diff --git a/doc/images/hierarchy.png b/doc/images/hierarchy.png
new file mode 100644
index 0000000..eccace4
Binary files /dev/null and b/doc/images/hierarchy.png differ
diff --git a/doc/images/list-cuthead-1-1.png b/doc/images/list-cuthead-1-1.png
new file mode 100644
index 0000000..f5d6086
Binary files /dev/null and b/doc/images/list-cuthead-1-1.png differ
diff --git a/doc/images/list-cuthead-1-2.png b/doc/images/list-cuthead-1-2.png
new file mode 100644
index 0000000..c617713
Binary files /dev/null and b/doc/images/list-cuthead-1-2.png differ
diff --git a/doc/images/list-cuthead-1-3.png b/doc/images/list-cuthead-1-3.png
new file mode 100644
index 0000000..edfc6cc
Binary files /dev/null and b/doc/images/list-cuthead-1-3.png differ
diff --git a/doc/images/list-cuthead-1-4.png b/doc/images/list-cuthead-1-4.png
new file mode 100644
index 0000000..2e12f47
Binary files /dev/null and b/doc/images/list-cuthead-1-4.png differ
diff --git a/doc/images/list-cuthead-1-5.png b/doc/images/list-cuthead-1-5.png
new file mode 100644
index 0000000..c405f33
Binary files /dev/null and b/doc/images/list-cuthead-1-5.png differ
diff --git a/doc/images/list-cuthead-2-1.png b/doc/images/list-cuthead-2-1.png
new file mode 100644
index 0000000..1e6fa04
Binary files /dev/null and b/doc/images/list-cuthead-2-1.png differ
diff --git a/doc/images/list-cuthead-2-2.png b/doc/images/list-cuthead-2-2.png
new file mode 100644
index 0000000..30bf458
Binary files /dev/null and b/doc/images/list-cuthead-2-2.png differ
diff --git a/doc/images/list-cuthead-2-3.png b/doc/images/list-cuthead-2-3.png
new file mode 100644
index 0000000..63e43e5
Binary files /dev/null and b/doc/images/list-cuthead-2-3.png differ
diff --git a/doc/images/list-cuthead-2-4.png b/doc/images/list-cuthead-2-4.png
new file mode 100644
index 0000000..45ae616
Binary files /dev/null and b/doc/images/list-cuthead-2-4.png differ
diff --git a/doc/images/list-cuthead-2-5.png b/doc/images/list-cuthead-2-5.png
new file mode 100644
index 0000000..aeb5a09
Binary files /dev/null and b/doc/images/list-cuthead-2-5.png differ
diff --git a/doc/images/list-cuthead-2-6.png b/doc/images/list-cuthead-2-6.png
new file mode 100644
index 0000000..c0d3c86
Binary files /dev/null and b/doc/images/list-cuthead-2-6.png differ
diff --git a/doc/images/list-insert-1-1.png b/doc/images/list-insert-1-1.png
new file mode 100644
index 0000000..3363c5f
Binary files /dev/null and b/doc/images/list-insert-1-1.png differ
diff --git a/doc/images/list-insert-1-2.png b/doc/images/list-insert-1-2.png
new file mode 100644
index 0000000..fc33806
Binary files /dev/null and b/doc/images/list-insert-1-2.png differ
diff --git a/doc/images/list-insert-1-3.png b/doc/images/list-insert-1-3.png
new file mode 100644
index 0000000..613677d
Binary files /dev/null and b/doc/images/list-insert-1-3.png differ
diff --git a/doc/images/list-insert-1-4.png b/doc/images/list-insert-1-4.png
new file mode 100644
index 0000000..71cca5b
Binary files /dev/null and b/doc/images/list-insert-1-4.png differ
diff --git a/doc/images/list-insert-2-1.png b/doc/images/list-insert-2-1.png
new file mode 100644
index 0000000..0456c27
Binary files /dev/null and b/doc/images/list-insert-2-1.png differ
diff --git a/doc/images/list-insert-2-2.png b/doc/images/list-insert-2-2.png
new file mode 100644
index 0000000..abd5049
Binary files /dev/null and b/doc/images/list-insert-2-2.png differ
diff --git a/doc/images/list-insert-2-3.png b/doc/images/list-insert-2-3.png
new file mode 100644
index 0000000..ea714e0
Binary files /dev/null and b/doc/images/list-insert-2-3.png differ
diff --git a/doc/images/list-insert-3-1.png b/doc/images/list-insert-3-1.png
new file mode 100644
index 0000000..af7a738
Binary files /dev/null and b/doc/images/list-insert-3-1.png differ
diff --git a/doc/images/list-insert-3-2.png b/doc/images/list-insert-3-2.png
new file mode 100644
index 0000000..bcc6ad6
Binary files /dev/null and b/doc/images/list-insert-3-2.png differ
diff --git a/doc/images/list-insert-3-3.png b/doc/images/list-insert-3-3.png
new file mode 100644
index 0000000..b26e97f
Binary files /dev/null and b/doc/images/list-insert-3-3.png differ
diff --git a/doc/images/list-insert-3-4.png b/doc/images/list-insert-3-4.png
new file mode 100644
index 0000000..3f88863
Binary files /dev/null and b/doc/images/list-insert-3-4.png differ
diff --git a/doc/images/list-insert-3-5.png b/doc/images/list-insert-3-5.png
new file mode 100644
index 0000000..bbd86f1
Binary files /dev/null and b/doc/images/list-insert-3-5.png differ
diff --git a/doc/images/list-insert-4-1.png b/doc/images/list-insert-4-1.png
new file mode 100644
index 0000000..ec86222
Binary files /dev/null and b/doc/images/list-insert-4-1.png differ
diff --git a/doc/images/list-insert-4-2.png b/doc/images/list-insert-4-2.png
new file mode 100644
index 0000000..0ef89bd
Binary files /dev/null and b/doc/images/list-insert-4-2.png differ
diff --git a/doc/images/list-insert-4-3.png b/doc/images/list-insert-4-3.png
new file mode 100644
index 0000000..3a3d1a5
Binary files /dev/null and b/doc/images/list-insert-4-3.png differ
diff --git a/doc/images/list-insert-4-4.png b/doc/images/list-insert-4-4.png
new file mode 100644
index 0000000..b4eca6b
Binary files /dev/null and b/doc/images/list-insert-4-4.png differ
diff --git a/doc/images/logo/transparent-black-v2.png b/doc/images/logo/transparent-black-v2.png
new file mode 100644
index 0000000..987e125
Binary files /dev/null and b/doc/images/logo/transparent-black-v2.png differ
diff --git a/doc/images/logo/transparent-blue-v2.png b/doc/images/logo/transparent-blue-v2.png
new file mode 100644
index 0000000..19ecb93
Binary files /dev/null and b/doc/images/logo/transparent-blue-v2.png differ
diff --git a/doc/images/logo/transparent-red-v2.png b/doc/images/logo/transparent-red-v2.png
new file mode 100644
index 0000000..1537736
Binary files /dev/null and b/doc/images/logo/transparent-red-v2.png differ
diff --git a/doc/images/membership-change-1.png b/doc/images/membership-change-1.png
new file mode 100644
index 0000000..b28126c
Binary files /dev/null and b/doc/images/membership-change-1.png differ
diff --git a/doc/images/membership-change-2.png b/doc/images/membership-change-2.png
new file mode 100644
index 0000000..2c2e3e9
Binary files /dev/null and b/doc/images/membership-change-2.png differ
diff --git a/doc/images/parallel-replicating.png b/doc/images/parallel-replicating.png
new file mode 100644
index 0000000..500f899
Binary files /dev/null and b/doc/images/parallel-replicating.png differ
diff --git a/doc/images/pay_alipay.png b/doc/images/pay_alipay.png
new file mode 100644
index 0000000..0245339
Binary files /dev/null and b/doc/images/pay_alipay.png differ
diff --git a/doc/images/pay_wechat.png b/doc/images/pay_wechat.png
new file mode 100644
index 0000000..1057a91
Binary files /dev/null and b/doc/images/pay_wechat.png differ
diff --git a/doc/images/polling-CQ-thread.png b/doc/images/polling-CQ-thread.png
new file mode 100644
index 0000000..724fc71
Binary files /dev/null and b/doc/images/polling-CQ-thread.png differ
diff --git a/doc/images/pri-queue.png b/doc/images/pri-queue.png
new file mode 100644
index 0000000..a79d7f9
Binary files /dev/null and b/doc/images/pri-queue.png differ
diff --git a/doc/images/queue-1.png b/doc/images/queue-1.png
new file mode 100644
index 0000000..0019f76
Binary files /dev/null and b/doc/images/queue-1.png differ
diff --git a/doc/images/queue-2.png b/doc/images/queue-2.png
new file mode 100644
index 0000000..a8af807
Binary files /dev/null and b/doc/images/queue-2.png differ
diff --git a/doc/images/queue-3.png b/doc/images/queue-3.png
new file mode 100644
index 0000000..3410ebd
Binary files /dev/null and b/doc/images/queue-3.png differ
diff --git a/doc/images/replicate-callback.png b/doc/images/replicate-callback.png
new file mode 100644
index 0000000..36154cb
Binary files /dev/null and b/doc/images/replicate-callback.png differ
diff --git a/doc/images/singlelist-1.png b/doc/images/singlelist-1.png
new file mode 100644
index 0000000..26f0720
Binary files /dev/null and b/doc/images/singlelist-1.png differ
diff --git a/doc/images/singlelist-2.png b/doc/images/singlelist-2.png
new file mode 100644
index 0000000..dda37d4
Binary files /dev/null and b/doc/images/singlelist-2.png differ
diff --git a/doc/images/storage-binlog.png b/doc/images/storage-binlog.png
new file mode 100644
index 0000000..a262c2c
Binary files /dev/null and b/doc/images/storage-binlog.png differ
diff --git a/doc/images/storage-memory-dump.png b/doc/images/storage-memory-dump.png
new file mode 100644
index 0000000..eeb3c9a
Binary files /dev/null and b/doc/images/storage-memory-dump.png differ
diff --git a/doc/images/storage-overview.png b/doc/images/storage-overview.png
new file mode 100644
index 0000000..1fdd907
Binary files /dev/null and b/doc/images/storage-overview.png differ
diff --git a/doc/images/storage-sstable.png b/doc/images/storage-sstable.png
new file mode 100644
index 0000000..d627a38
Binary files /dev/null and b/doc/images/storage-sstable.png differ
diff --git a/doc/images/system_architecture.png b/doc/images/system_architecture.png
new file mode 100644
index 0000000..bf0f817
Binary files /dev/null and b/doc/images/system_architecture.png differ
diff --git a/doc/images/write-requests-coordinating.png b/doc/images/write-requests-coordinating.png
new file mode 100644
index 0000000..ab90eaf
Binary files /dev/null and b/doc/images/write-requests-coordinating.png differ
diff --git a/doc/windows.md b/doc/windows.md
new file mode 100644
index 0000000..08d7143
--- /dev/null
+++ b/doc/windows.md
@@ -0,0 +1,68 @@
+# build under windows:
+##### 0. Rerquire :
+ * visual studio >= 2015.
+ * vs compiler >= 19.00.24215.1.
+
+*Note: make sure modify the directory prefix to your own for each of the following example configure.*
+
+##### 1. configure preprocessor
+* Go to `Project properties->c/c++->Preprocessor->Proprocessor Definitions` with values:
+ * _CRT_SECURE_NO_WARNINGS
+ * _SCL_SECURE_NO_WARNINGS
+ * _WIN32_WINNT=0x0A00
+ * GLOG_NO_ABBREVIATED_SEVERITIES
+ * GFLAGS_DLL_DEFINE_FLAG=
+ * GTEST_HAS_TR1_TUPLE=0
+ * _HAS_AUTO_PTR_ETC
+* configure `Project properties->c/c++->All options->Additional Options` with `/std:c++latest`.
+
+##### 2. configure Include Directories
+Go to `Project properties->c/c++->General->Additional`, do it for each lib. Example of mine:
+
+```
+C:\Users\95\Documents\Visual Studio 2015\Projects\apollo\raft\src
+D:\third_party\boost_1_64_0
+C:\Users\95\.babun\cygwin\home\arthur\git\glog\src\windows
+C:\Users\95\.babun\cygwin\home\arthur\git\gflags\cmake\build\include
+C:\Users\95\.babun\cygwin\home\arthur\git\protobuf\src
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\include
+C:\Users\95\.babun\cygwin\home\arthur\git\googletest\googletest\include
+```
+
+##### 3. configure Library Directories
+* Go to `Project properties->Linker->General->Additional Library Directories.` Example of mine:
+```
+C:\Users\95\.babun\cygwin\home\arthur\git\googletest\googletest\build\Debug
+D:\third_party\boost_1_64_0\bin.v2\libs_summary
+C:\Users\95\.babun\cygwin\home\arthur\git\glog\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\gflags\cmake\build\lib\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\protobuf\cmake\build\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\build-dir\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\build-dir\third_party\boringssl\crypto\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\build-dir\third_party\boringssl\ssl\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\build-dir\third_party\zlib\Debug
+C:\Users\95\.babun\cygwin\home\arthur\git\grpc\third_party\cares\cares\build-dir\bin\Release
+C:\Users\95\.babun\cygwin\home\arthur\git\gperftools\x64\Release-Patch
+```
+
+* Go to `Project properties->Linker->General->Input->Additional Dependencies.` Example of mine:
+```
+gtest.lib
+grpc.lib
+grpc++.lib
+libprotobuf.lib
+libprotobuf-lite.lib
+gpr.lib
+ws2_32.lib
+gflags_static.lib
+libglog_static.lib
+shlwapi.lib
+zlib.lib
+ssl.lib
+crypto.lib
+cares.lib
+address_sorting.lib
+libtcmalloc_minimal.lib
+```
+
+
diff --git a/src/.gitignore b/src/.gitignore
new file mode 100644
index 0000000..ed71ea0
--- /dev/null
+++ b/src/.gitignore
@@ -0,0 +1,474 @@
+
+#my own addings
+*raft.pb.h
+*raft.pb.cc
+*raft.grpc.pb.h
+*raft.grpc.pb.cc
+
+
+# Created by https://www.gitignore.io/api/vim,git,c++,macos,windows,visualstudio
+# Edit at https://www.gitignore.io/?templates=vim,git,c++,macos,windows,visualstudio
+
+### C++ ###
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+### Git ###
+# Created by git for backups. To disable backups in Git:
+# $ git config --global mergetool.keepBackup false
+*.orig
+
+# Created by git when using merge tools for conflicts
+*.BACKUP.*
+*.BASE.*
+*.LOCAL.*
+*.REMOTE.*
+*_BACKUP_*.txt
+*_BASE_*.txt
+*_LOCAL_*.txt
+*_REMOTE_*.txt
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+### VisualStudio ###
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.iobj
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding add-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+# ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true
+**/wwwroot/lib/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- Backup*.rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# JetBrains Rider
+.idea/
+*.sln.iml
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# End of https://www.gitignore.io/api/vim,git,c++,macos,windows,visualstudio
diff --git a/src/binlog/binlog_meta_data.cc b/src/binlog/binlog_meta_data.cc
new file mode 100644
index 0000000..313f109
--- /dev/null
+++ b/src/binlog/binlog_meta_data.cc
@@ -0,0 +1,322 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "boost/crc.hpp"
+
+#include "config/config.h"
+#include "tools/utilities.h"
+#include "binlog/binlog_meta_data.h"
+
+#define _TYPE_LOG_IDX_OFFSET_ (0x01)
+
+namespace RaftCore::BinLog {
+
+FileMetaData::IdxPair::IdxPair(uint32_t _a, uint64_t _b, uint32_t _c, uint32_t _d, uint32_t _e) {
+ this->m_term = _a;
+ this->m_index = _b;
+ this->m_offset = _c;
+ this->key_crc32 = _d;
+ this->value_crc32 = _e;
+}
+
+bool FileMetaData::IdxPair::operator<(const IdxPair& _other)const noexcept {
+ return this->LogIdentifier::operator<(_other);
+}
+
+bool FileMetaData::IdxPair::operator>(const IdxPair& _other)const noexcept {
+ return this->LogIdentifier::operator>(_other);
+}
+
+bool FileMetaData::IdxPair::operator>(const LogIdentifier& _other)const noexcept {
+ return this->LogIdentifier::operator>(_other);
+}
+
+bool FileMetaData::IdxPair::operator<(const LogIdentifier& _other)const noexcept {
+ return this->LogIdentifier::operator<(_other);
+}
+
+bool FileMetaData::IdxPair::operator<=(const LogIdentifier& _other)const noexcept {
+ return this->LogIdentifier::operator<=(_other);
+}
+
+bool FileMetaData::IdxPair::operator>=(const LogIdentifier& _other)const noexcept {
+ return this->LogIdentifier::operator>=(_other);
+}
+
+bool FileMetaData::IdxPair::operator<(const ::raft::EntityID &_other)const noexcept {
+ if (this->m_term < _other.term())
+ return true;
+
+ if (this->m_term > _other.term())
+ return false;
+
+ return this->m_index < _other.idx();
+}
+
+bool FileMetaData::IdxPair::operator==(const ::raft::EntityID &_other)const noexcept {
+ return (this->m_term == _other.term()) && (this->m_index==_other.idx());
+}
+
+bool FileMetaData::IdxPair::operator!=(const ::raft::EntityID &_other)const noexcept {
+ return !this->operator==(_other);
+}
+
+bool FileMetaData::IdxPair::operator>(const ::raft::EntityID &_other)const noexcept {
+ if (this->m_term > _other.term())
+ return true;
+
+ if (this->m_term < _other.term())
+ return false;
+
+ return this->m_index > _other.idx();
+}
+
+bool FileMetaData::IdxPair::operator==(const IdxPair &_other) const noexcept{
+ return (this->m_term == _other.m_term && this->m_index == _other.m_index);
+}
+
+const FileMetaData::IdxPair& FileMetaData::IdxPair::operator=(const IdxPair &_other) noexcept {
+ this->m_term = _other.m_term;
+ this->m_index = _other.m_index;
+ this->m_offset = _other.m_offset;
+ this->key_crc32 = _other.key_crc32;
+ this->value_crc32 = _other.value_crc32;
+
+ return *this;
+}
+
+std::size_t FileMetaData::IdxPair::Hash() const noexcept {
+ auto h1 = std::hash{}(this->m_index);
+ auto h2 = std::hash{}(this->m_offset);
+ return h1 ^ (h2 << 8);
+}
+
+FileMetaData::FileMetaData() noexcept {}
+
+FileMetaData::~FileMetaData() noexcept {}
+
+void FileMetaData::AddLogOffset(uint32_t term,uint64_t log_index, uint32_t file_offset, uint32_t key_crc32,uint32_t value_crc32) noexcept {
+ //The allocated buf will be released in the destructor.
+ std::shared_ptr _shp_new_record(new IdxPair(term,log_index,file_offset,key_crc32,value_crc32));
+ this->m_meta_hash.Insert(_shp_new_record);
+}
+
+void FileMetaData::AddLogOffset(const TypeOffsetList &_list) noexcept {
+ for (const auto &_item : _list)
+ this->m_meta_hash.Insert(_item);
+}
+
+void FileMetaData::Delete(const IdxPair &_item) noexcept {
+ this->m_meta_hash.Delete(_item);
+}
+
+TypeBufferInfo FileMetaData::GenerateBuffer() const noexcept {
+
+ std::list> _hash_entry_list;
+ this->m_meta_hash.GetOrderedByKey(_hash_entry_list);
+
+ /* Meta data on-disk format,from low offset to high offset:
+ 1> meta data(tlv list)
+ 1) type :1 bytes
+ 2) length : 3 bytes
+ 3) value : vary
+ 2> CRC32 checksum. (4 bytes)
+ 3> length of whole meta data area. (4 bytes)
+ 4> a magic string to identify the end of meta-data area. (variadic bytes) */
+
+ ::raft::LogOffset _log_offset;
+ for (const auto &_ptr_entry : _hash_entry_list) {
+ if (_ptr_entry == nullptr)
+ continue;
+
+ ::raft::LogOffsetItem* _p_item = _log_offset.add_mappings();
+ _p_item->set_log_term(_ptr_entry->m_term);
+ _p_item->set_log_idx(_ptr_entry->m_index);
+ _p_item->set_offset(_ptr_entry->m_offset);
+ _p_item->set_key_crc32(_ptr_entry->key_crc32);
+ _p_item->set_value_crc32(_ptr_entry->value_crc32);
+ }
+
+ std::string _log_offset_buf = "";
+ _log_offset.SerializeToString(&_log_offset_buf);
+
+ uint32_t entry_field_length = (uint32_t)_log_offset_buf.size();
+
+ //Value length cannot exceeds the maximum a three byte long integer can represent.
+ assert(entry_field_length <= 0xFFFFFF);
+
+ static uint32_t _footer_len = (uint32_t)std::strlen(_FILE_META_DATA_MAGIC_STR_);
+ uint32_t _tail_length = 4 * 2 + _footer_len;
+ uint32_t _buf_size = 1 + 3 + entry_field_length + _tail_length;
+ auto _ptr = (unsigned char*)malloc(_buf_size);
+ assert(_ptr!=nullptr);
+
+ //Set meta data 1> 1).
+ unsigned char* _p_cur = (unsigned char*)_ptr;
+ _p_cur[0] = _TYPE_LOG_IDX_OFFSET_;
+ _p_cur++;
+
+ //Set meta data 1> 2).
+ uint32_t _tmp = 0;
+ ::RaftCore::Tools::ConvertToBigEndian(entry_field_length, &_tmp);
+ unsigned char* px = (unsigned char*)&_tmp;
+ CHECK(*px == 0x0); //for it is big endian , the fist byte must be 0x0.
+ px++;
+ std::memcpy(_p_cur,px,_FOUR_BYTES_ - 1);
+ _p_cur += 3;
+
+ //Set meta data 1> 3).
+ std::memcpy(_p_cur,_log_offset_buf.data(),entry_field_length);
+ _p_cur += entry_field_length;
+
+ //Calculate crc32 checksum.
+ uint32_t _crc32_value = ::RaftCore::Tools::CalculateCRC32(_ptr, _buf_size - _tail_length);
+
+ //Set crc32 checksum.
+ ::RaftCore::Tools::ConvertToBigEndian(_crc32_value, &_tmp);
+ std::memcpy(_p_cur,&_tmp,_FOUR_BYTES_);
+ _p_cur += _FOUR_BYTES_;
+
+ //Set meta data area length.
+ ::RaftCore::Tools::ConvertToBigEndian(_buf_size, &_tmp);
+ std::memcpy(_p_cur,&_tmp,_FOUR_BYTES_);
+ _p_cur += _FOUR_BYTES_;
+
+ //Set meta data area magic string.
+ std::memcpy(_p_cur,_FILE_META_DATA_MAGIC_STR_,_footer_len);
+ _p_cur += _footer_len;
+
+ return std::make_tuple(_ptr,_buf_size);
+}
+
+void FileMetaData::ConstructMeta(const unsigned char* _buf, std::size_t _size) noexcept {
+ auto _cur_ptr = _buf;
+ //Examine header.
+ CHECK(*_cur_ptr == _TYPE_LOG_IDX_OFFSET_) << "meta header check fail";
+ _cur_ptr++;
+
+ //Examine the checksum of metadata buf.
+ uint32_t _length = 0x0;
+ std::memcpy((unsigned char*)&_length+1,_cur_ptr,_FOUR_BYTES_ - 1);
+ ::RaftCore::Tools::ConvertBigEndianToLocal(_length, &_length);
+ _cur_ptr += 3;
+
+ //Examine the checksum of metadata buf.
+ ::raft::LogOffset _log_offset;
+ CHECK(_log_offset.ParseFromArray(_cur_ptr, _length)) << "parse meta data buf fail..";
+
+ this->m_meta_hash.Clear();
+ for (const auto &_item : _log_offset.mappings()) {
+ std::shared_ptr _shp_pair(new IdxPair(_item.log_term(),_item.log_idx(),_item.offset(),_item.key_crc32(),_item.value_crc32()));
+ this->m_meta_hash.Insert(_shp_pair);
+ }
+}
+
+int FileMetaData::ConstructMeta(std::FILE* _handler) noexcept {
+
+ //Offset of _handler should be set to 0 before calling this method.
+
+ assert(_handler != nullptr);
+
+ uint32_t _buf_size = ::RaftCore::Config::FLAGS_binlog_parse_buf_size * 1024 * 1024;
+ unsigned char *sz_buf = (unsigned char *)malloc(_buf_size);
+
+ uint32_t _this_read = 0,_total_read = 0;
+ long _total_offset = 0;
+
+ this->m_meta_hash.Clear();
+ do{
+ _this_read = (uint32_t)std::fread(sz_buf,1,_buf_size,_handler);
+ CHECK(_this_read > _FOUR_BYTES_) << "parsing:read raw file fail,file corruption found,actual read:" << _this_read;
+ _total_read += _this_read;
+
+ auto _p_cur = sz_buf;
+
+ //Parsing this buffer.
+ uint32_t _this_offset = 0;
+ do{
+ if (_this_read - _this_offset < _FOUR_BYTES_) {
+ /*Each of the pairs must be read integrally,if current parsing buffer is smaller than 4 bytes,
+ means a partially read happened,need to adjust the reading position . */
+ CHECK(std::fseek(_handler,_total_offset,SEEK_SET)==0) << "seeking binlog fail";
+ break;
+ }
+
+ uint32_t _cur_offset = _total_offset;
+
+ uint32_t _item_buf_len = *(uint32_t*)_p_cur;
+ ::RaftCore::Tools::ConvertBigEndianToLocal(_item_buf_len, &_item_buf_len);
+ if ( _this_read - (_this_offset+_FOUR_BYTES_) < _item_buf_len ) {
+ /*Like the previous if branch which contains the break statement:need position adjusting*/
+ CHECK(std::fseek(_handler,_total_offset,SEEK_SET)==0) << "seeking binlog fail";
+ break;
+ }
+
+ _p_cur += _FOUR_BYTES_;
+ _total_offset += _FOUR_BYTES_;
+ _this_offset += _FOUR_BYTES_;
+
+ ::raft::BinlogItem _binlog_item;
+
+ //Parse binglog_item buffer.
+ CHECK(_binlog_item.ParseFromArray(_p_cur, _item_buf_len)) << "parse file content fail..";
+ _p_cur += _item_buf_len;
+ _total_offset += _item_buf_len;
+ _this_offset += _item_buf_len;
+
+ const auto & _wop = _binlog_item.entity().write_op();
+ uint32_t _key_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_wop.key().data(),(unsigned int)_wop.key().length());
+ uint32_t _value_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_wop.value().data(),(unsigned int)_wop.value().length());
+
+ std::shared_ptr _shp_pair(new IdxPair(_binlog_item.entity().entity_id().term(),
+ _binlog_item.entity().entity_id().idx(), _cur_offset,_key_crc32,_value_crc32));
+ this->m_meta_hash.Insert(_shp_pair);
+
+ //This will always be true, otherwise it would break at previous break statement.
+ } while (_this_offset < _this_read);
+
+ //If _this_read < _buf_size , means we've read to the end of file.
+ } while (_this_read >= _buf_size);
+
+ free(sz_buf);
+
+ return _total_offset == _total_read ? 0 : _total_offset;
+}
+
+void FileMetaData::GetOrderedMeta(TypeOffsetList &_output) const noexcept {
+ this->m_meta_hash.GetOrderedByKey(_output);
+}
+
+void FileMetaData::BackOffset(int offset) noexcept{
+ this->m_meta_hash.Map([&offset](std::shared_ptr &_one)->void{ _one->m_offset -= offset;});
+}
+
+std::string FileMetaData::IdxPair::ToString() const noexcept {
+ const static int _buf_size = 512;
+ char _sz_buf[_buf_size] = { 0 };
+ std::snprintf(_sz_buf,_buf_size,"IdxPair term:%u,idx:%llu,offset:%u",this->m_term,this->m_index,this->m_offset);
+
+ return std::string(_sz_buf);
+}
+
+std::ostream& operator<<(std::ostream& os, const FileMetaData::IdxPair& obj) {
+ os << obj.ToString();
+ return os;
+}
+
+}
\ No newline at end of file
diff --git a/src/binlog/binlog_meta_data.h b/src/binlog/binlog_meta_data.h
new file mode 100644
index 0000000..9c826d9
--- /dev/null
+++ b/src/binlog/binlog_meta_data.h
@@ -0,0 +1,149 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_BIN_LOG_META_DATA_H__
+#define __AURORA_BIN_LOG_META_DATA_H__
+
+#include
+#include
+#include
+#include
+
+#include "protocol/raft.pb.h"
+
+#include "common/comm_defs.h"
+#include "common/log_identifier.h"
+#include "tools/lock_free_hash.h"
+
+#define _FILE_META_DATA_MAGIC_STR_ "!@#$binlog$#@!"
+
+namespace RaftCore::BinLog {
+
+using ::RaftCore::Common::LogIdentifier;
+using ::RaftCore::Common::TypeBufferInfo;
+using ::RaftCore::DataStructure::HashTypeBase;
+
+class FileMetaData final{
+
+public:
+
+ struct IdxPair final : LogIdentifier , HashTypeBase{
+
+ uint32_t m_offset;
+ uint32_t key_crc32;
+ uint32_t value_crc32;
+
+ IdxPair(uint32_t _a, uint64_t _b, uint32_t _c, uint32_t _d, uint32_t _e);
+
+ virtual bool operator<(const IdxPair& _other)const noexcept override;
+
+ bool operator==(const IdxPair &_other) const noexcept override;
+
+ const IdxPair& operator=(const IdxPair &_other) noexcept override;
+
+ std::string ToString()const noexcept;
+
+ virtual std::size_t Hash() const noexcept override;
+
+ virtual bool operator>(const IdxPair& _other)const noexcept;
+
+ virtual bool operator>(const LogIdentifier& _other)const noexcept;
+
+ virtual bool operator<(const LogIdentifier& _other)const noexcept;
+
+ virtual bool operator<=(const LogIdentifier& _other)const noexcept;
+
+ virtual bool operator>=(const LogIdentifier& _other)const noexcept;
+
+ virtual bool operator<(const ::raft::EntityID &_other)const noexcept;
+
+ virtual bool operator==(const ::raft::EntityID &_other)const noexcept;
+
+ virtual bool operator!=(const ::raft::EntityID &_other)const noexcept;
+
+ virtual bool operator>(const ::raft::EntityID &_other)const noexcept;
+ };
+
+ typedef std::list> TypeOffsetList;
+
+public:
+
+ FileMetaData() noexcept;
+
+ virtual ~FileMetaData() noexcept;
+
+ /* This functions will be invoked each time we write a new log entry into the binlog file,
+ namely,in a multiple threading access environment.
+ - Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ void AddLogOffset(uint32_t term,uint64_t log_index, uint32_t file_offset,uint32_t key_crc32,uint32_t value_crc32) noexcept;
+
+ /*- Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ void AddLogOffset(const TypeOffsetList &_list) noexcept;
+
+ /*- Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ void Delete(const IdxPair &_item) noexcept;
+
+ /*This functions will be invoked only in the rotating file scenario,namely,
+ a single thread accessing environment.After return the pointer pointing
+ to the generated buffer, this function will no longer be responsible
+ for the memory it allocated, the caller must take care of(freeing) it.
+
+ - Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ TypeBufferInfo GenerateBuffer() const noexcept;
+
+ /* - Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ void ConstructMeta(const unsigned char* _buf, std::size_t _size) noexcept;
+
+ /* - Support Multiple Thread invoking: False
+ - Will be invoked simultaneously: False */
+ /*Return value:
+ 0 - parsed successfully.
+ >0 - parsed bytes.Remaining bytes are not parsable.Need to be truncated. */
+ int ConstructMeta(std::FILE* _handler) noexcept;
+
+ /* - Support Multiple Thread invoking: True
+ - Will be invoked simultaneously: False */
+ void GetOrderedMeta(TypeOffsetList &_output) const noexcept;
+
+ void BackOffset(int offset) noexcept;
+
+private:
+
+ ::RaftCore::DataStructure::LockFreeHash m_meta_hash;
+
+private:
+
+ FileMetaData(const FileMetaData&) = delete;
+
+ FileMetaData& operator=(const FileMetaData&) = delete;
+
+};
+
+std::ostream& operator<<(std::ostream& os, const FileMetaData::IdxPair& obj);
+
+} //end namespace
+
+
+#endif
diff --git a/src/binlog/binlog_operator.cc b/src/binlog/binlog_operator.cc
new file mode 100644
index 0000000..d75250b
--- /dev/null
+++ b/src/binlog/binlog_operator.cc
@@ -0,0 +1,672 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "boost/crc.hpp"
+#include "boost/filesystem.hpp"
+
+#include "config/config.h"
+#include "tools/utilities.h"
+#include "leader/leader_view.h"
+#include "binlog/binlog_operator.h"
+#include "storage/storage_singleton.h"
+
+namespace RaftCore::BinLog {
+
+//To avoid issues caused by including header files mutually.
+namespace fs = ::boost::filesystem;
+using ::raft::EntityID;
+using ::RaftCore::Leader::LeaderView;
+using ::RaftCore::Common::ConvertID;
+using ::RaftCore::Storage::StorageGlobal;
+
+BinLogOperator::BinLogOperator() {}
+
+BinLogOperator::~BinLogOperator()noexcept {
+ if (this->m_initialized)
+ this->UnInitialize();
+}
+
+void BinLogOperator::Initialize(const char* role, bool file_name) noexcept {
+
+ CHECK(role != nullptr);
+
+ if (file_name)
+ this->m_file_name = role;
+ else
+ this->m_file_name = _AURORA_BINLOG_NAME_ + std::string(".") + role;
+
+ this->m_binlog_handler = std::fopen(this->m_file_name.c_str(),_AURORA_BINLOG_OP_MODE_);
+ CHECK(this->m_binlog_handler != nullptr) << "open binlog file " << this->m_file_name << "fail..,errno:" << errno;
+
+ this->ParseFile();
+
+ std::list> _cur_file_meta;
+ m_p_meta.load()->GetOrderedMeta(_cur_file_meta);
+
+ LogIdentifier _last;
+ _last.Set(0,0);
+ if (!_cur_file_meta.empty())
+ _last.Set(_cur_file_meta.back()->m_term,_cur_file_meta.back()->m_index);
+
+ this->m_last_logged.store(_last, std::memory_order_release);
+ this->m_binlog_status.store(BinlogStatus::NORMAL);
+ this->m_log_num = (uint32_t)_cur_file_meta.size();
+
+ this->m_precede_lcl_inuse.store(0);
+
+ //this->m_term_changed.store(false);
+
+ this->m_initialized = true;
+}
+
+//void BinLogOperator::SetTermMatched() noexcept {
+// //this->m_term_changed = true;
+//}
+
+//bool BinLogOperator::IsTermMatched() noexcept {
+// //return this->m_term_changed;
+//}
+
+void BinLogOperator::AddPreLRLUseCount() noexcept {
+ this->m_precede_lcl_inuse.fetch_add(1);
+}
+
+void BinLogOperator::SubPreLRLUseCount() noexcept {
+ this->m_precede_lcl_inuse.fetch_sub(1);
+}
+
+void BinLogOperator::ParseFile() noexcept{
+
+ //Construct the meta data if not exist, otherwise just load it into RAM
+ m_p_meta.store(new FileMetaData());
+
+ CHECK(std::fseek(this->m_binlog_handler, 0, SEEK_END) == 0) << "seek binlog file "
+ << this->m_file_name << "fail..,errno:" << errno;
+
+ long _file_size = std::ftell(this->m_binlog_handler);
+ CHECK(_file_size >= 0) << "tell binlog file " << this->m_file_name << "fail..,errno:" << errno;
+
+ if (_file_size == 0) {
+ //Empty file,just skip the following parsing meta steps.
+ return;
+ }
+
+ long _file_tail_len = _FOUR_BYTES_ * 3;
+
+ int _minimal_pb_section_size = _FOUR_BYTES_; //Only a four-bytes length field .
+ int _minimal_meta_section_size = _FOUR_BYTES_ + _file_tail_len; //Only the tailing part and a 0-length tlv field.
+
+ CHECK(_file_size >= _minimal_pb_section_size + _minimal_meta_section_size) << "binlog file size not correct:" << _file_size;
+
+ //First read the last 12 bytes area.
+ CHECK(std::fseek(this->m_binlog_handler, (_file_size - _file_tail_len), SEEK_SET) == 0) << "seek binlog file "
+ << this->m_file_name << "fail..,errno:" << errno;
+
+ char sz_last_zone[12] = {0};
+ CHECK(std::fread(sz_last_zone, 1, 12, this->m_binlog_handler)==12) << "read binlog file " << this->m_file_name << "last zone fail..,errno:" << errno;
+
+ //This uncompleted file ending is probably due to an unexpected crash.
+ static std::size_t _footer_len = std::strlen(_FILE_META_DATA_MAGIC_STR_);
+ if (strncmp((const char*)&sz_last_zone[8], _FILE_META_DATA_MAGIC_STR_, _footer_len) != 0) {
+
+ CHECK(std::fseek(this->m_binlog_handler,0, SEEK_SET) == 0) << "seek binlog file "
+ << this->m_file_name << "fail..,errno:" << errno;
+
+ //Since the file has no meta data , we have to construct it all over again.
+ int _parsed_bytes = m_p_meta.load()->ConstructMeta(this->m_binlog_handler);
+ if (_parsed_bytes > 0) {
+ LOG(WARNING) << "binlog incomplete data found , file corruption probably happened,now truncating it.";
+ this->Truncate(_parsed_bytes);
+ }
+
+ return;
+ }
+
+ //Meta data exists , just parse it.
+ uint32_t _meta_data_len = *((uint32_t*)&sz_last_zone[4]);
+ ::RaftCore::Tools::ConvertBigEndianToLocal(_meta_data_len, &_meta_data_len);
+
+ CHECK(std::fseek(this->m_binlog_handler,(_file_size - _meta_data_len), SEEK_SET) == 0) << "seek binlog file "
+ << this->m_file_name << "fail..,errno:" << errno;
+
+ auto _ptr_meta_buf = (unsigned char*)malloc(_meta_data_len);
+ CHECK(std::fread(_ptr_meta_buf, 1,_meta_data_len, this->m_binlog_handler) == _meta_data_len)
+ << "read binlog meta data fail...,errno:" << errno;
+
+ uint32_t crc_in_file = *(uint32_t*)&sz_last_zone[0];
+ ::RaftCore::Tools::ConvertBigEndianToLocal(crc_in_file, &crc_in_file);
+
+ //CRC32 only calculate the 1>meta data area.
+ uint32_t _crc32_value = ::RaftCore::Tools::CalculateCRC32(_ptr_meta_buf, _meta_data_len - 12);
+
+ CHECK(crc_in_file == _crc32_value) << "binlog file meta check checksum failed";
+
+ m_p_meta.load()->ConstructMeta(_ptr_meta_buf,_meta_data_len);
+ free(_ptr_meta_buf);
+}
+
+void BinLogOperator::UnInitialize() noexcept{
+
+ auto _p_tmp = m_p_meta.load();
+ delete _p_tmp;
+ m_p_meta.store(nullptr);
+
+ this->m_initialized = false;
+
+ if (this->m_binlog_handler == nullptr)
+ return ;
+
+ if (!std::fclose(this->m_binlog_handler))
+ return; //normally return.
+
+ //Something bad happened.
+ CHECK(false) << "close binlog file fail...,errno:"<< errno;
+}
+
+void BinLogOperator::Truncate(uint32_t new_size) noexcept {
+
+ CHECK(std::fclose(this->m_binlog_handler)==0) << "truncating file fclose failed";
+ fs::resize_file(fs::path(this->m_file_name),new_size);
+
+ //Re-open file.
+ this->m_binlog_handler = std::fopen(this->m_file_name.c_str(),_AURORA_BINLOG_OP_MODE_);
+ CHECK(this->m_binlog_handler != nullptr) << "re-open binlog file after truncate fail..";
+}
+
+TypeBufferInfo BinLogOperator::PrepareBuffer(const TypeEntityList &input_entities, FileMetaData::TypeOffsetList &offset_list) noexcept {
+
+ unsigned char* _p_buf = nullptr;
+ std::size_t _buf_size = 0;
+
+ auto ms = std::chrono::duration_cast< std::chrono::milliseconds >(
+ std::chrono::system_clock::now().time_since_epoch() );
+
+ uint64_t ts = ms.count();
+
+ std::list _buf_list;
+ uint32_t _cur_offset = 0; //Offset relative to this append operation.
+ for (const auto & _shp_entity : input_entities) {
+
+ auto _binlogitem = ::raft::BinlogItem();
+ _binlogitem.set_allocated_entity(_shp_entity.get());
+ _binlogitem.set_timestamp_ms(ts);
+
+ std::string _output = "";
+ CHECK(_binlogitem.SerializeToString(&_output)) << "SerializeToString fail...,log entity:"
+ << _shp_entity->entity_id().term() << "|" << _shp_entity->entity_id().idx();
+
+ _binlogitem.release_entity();
+
+ const auto & _wop = _shp_entity->write_op();
+ uint32_t _key_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_wop.key().data(),(unsigned int)_wop.key().length());
+ uint32_t _value_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_wop.value().data(),(unsigned int)_wop.value().length());
+
+ uint32_t _this_buf_size = (uint32_t)_output.size() + _FOUR_BYTES_;
+ _buf_size += _this_buf_size;
+ _buf_list.emplace_back(std::move(_output));
+
+ //Recording offset info.
+ offset_list.emplace_back(new FileMetaData::IdxPair(_shp_entity->entity_id().term(),_shp_entity->entity_id().idx(), _cur_offset,_key_crc32,_value_crc32));
+ _cur_offset += _this_buf_size;
+ }
+
+ _p_buf = (unsigned char*)malloc(_buf_size);
+ auto _pcur = _p_buf;
+
+ for (const auto& _buf : _buf_list) {
+ uint32_t _buf_len = (uint32_t)_buf.size();
+ ::RaftCore::Tools::ConvertToBigEndian(uint32_t(_buf_len),&_buf_len);
+ std::memcpy(_pcur,(unsigned char*)&_buf_len,_FOUR_BYTES_);
+ _pcur += _FOUR_BYTES_;
+ std::memcpy(_pcur,_buf.data(),_buf.size());
+ _pcur += _buf.size();
+ }
+
+ return std::make_tuple(_p_buf, _buf_size);
+}
+
+void BinLogOperator::AppendBufferToBinlog(TypeBufferInfo &buffer_info, const FileMetaData::TypeOffsetList &offset_list) noexcept {
+
+ //First,update this file's meta data.
+ long _offset = std::ftell(this->m_binlog_handler);
+ CHECK (_offset >= 0) << "ftell binlog file fail...,errno:"<< errno ;
+
+ for (const auto &_item : offset_list) {
+ _item->m_offset += _offset;
+ }
+
+ m_p_meta.load()->AddLogOffset(offset_list);
+
+ //Second,append buffer to file.
+ unsigned char* _p_buf = nullptr;
+ int _buf_size = 0;
+ std::tie(_p_buf, _buf_size) = buffer_info;
+
+ CHECK(std::fwrite((void*)_p_buf,1,_buf_size,this->m_binlog_handler) == _buf_size) << "fwrite binlog file fail...,errno:"<< errno ;
+ CHECK(std::fflush(this->m_binlog_handler) == 0) << "fflush binlog file fail...,errno:" << errno;
+
+ free(_p_buf);
+}
+
+void BinLogOperator::RotateFile() noexcept {
+
+ /*Since file position indicator is always at the tail ,
+ and file is opened in binary mode , ftell is exactly the file size. */
+ long file_len = std::ftell(this->m_binlog_handler);
+ CHECK(file_len >= 0) << "ftell binlog file fail...,errno:"<< errno ;
+
+ bool _exceed_limits = ((uint32_t)file_len > ::RaftCore::Config::FLAGS_binlog_max_size) ||
+ (this->m_log_num.load() > ::RaftCore::Config::FLAGS_binlog_max_log_num);
+ if (!_exceed_limits)
+ return;
+
+ int _use_count = this->m_precede_lcl_inuse.load();
+ if (_use_count > 0) {
+ LOG(INFO) << "somewhere using the pre-lcl part,use count:" << _use_count;
+ return;
+ }
+
+ //Get the tail meta data need to move to the new binlog file.
+ uint32_t _reserve_counter = 0; //#logs before ID-LCL.
+ std::list> _cur_file_meta;
+ std::list> _new_file_meta;
+ FileMetaData* _p_new_meta = nullptr;
+ m_p_meta.load()->GetOrderedMeta(_cur_file_meta);
+ for (auto _iter = _cur_file_meta.crbegin(); _iter != _cur_file_meta.crend(); ) {
+
+ /*We need to preserve more log entries . There are reasons for this:
+
+ 1. Need to preserve the LCL in the new binlog, because it will be used in reverting
+ log scenario as the 'pre_log_id'.
+ 2. In some extreme cases, the resync log procedure may need log entries from leader's
+ binlog file that are even earlier than the LCL. E.g., a follower coming back to the
+ normal connected status of the cluster after a relatively long status of disconnected.
+ At which point , its logs falling a lot behind the leader's and need to resync a lot of entries.
+ */
+ bool _less_than_LCL = (*_iter)->operator<(StorageGlobal::m_instance.GetLastCommitted());
+ _reserve_counter += (_less_than_LCL) ? 1 : 0 ;
+ bool _overflow_reserve_num = _reserve_counter > ::RaftCore::Config::FLAGS_binlog_reserve_log_num;
+
+ if ( _less_than_LCL && _overflow_reserve_num )
+ break;
+
+ _new_file_meta.emplace_front(*_iter);
+
+ //Delete meta data that are smaller than (ID-LCL - FLAGS_reserve_log_num).
+ m_p_meta.load()->Delete(**_iter);
+
+ //Insert current iterating meta-data item to the new binlog file's meta-data list.
+ if (_p_new_meta == nullptr)
+ _p_new_meta = new FileMetaData();
+ _p_new_meta->AddLogOffset((*_iter)->m_term,(*_iter)->m_index,(*_iter)->m_offset,(*_iter)->key_crc32,(*_iter)->value_crc32);
+
+ _iter++;
+ }
+
+ //If the trailing part actually exists , remove it.
+ unsigned char* _p_tail_logs_buf = nullptr;
+ long _tail_size = 0;
+
+ //Note: _new_file_meta will contain log start from,i.e.,>= (ID-LCL - FLAGS_reserve_log_num).
+ std::shared_ptr _shp_front = _new_file_meta.front();
+
+ //Copy tail meta data to the new binlog file.
+ CHECK(std::fseek(this->m_binlog_handler,_shp_front->m_offset,SEEK_SET)==0);
+
+ long _front_size = std::ftell(this->m_binlog_handler);
+ CHECK(_front_size >= 0) << "tell binlog file " << this->m_file_name << "fail..,errno:" << errno;
+ _tail_size = file_len - _front_size;
+
+ _p_tail_logs_buf = (unsigned char *)malloc(_tail_size);
+
+ CHECK(std::fread(_p_tail_logs_buf,1,_tail_size,this->m_binlog_handler)==_tail_size) << "read binlog file "
+ << this->m_file_name << " tail fail..,errno:" << errno;
+
+ this->Truncate(_shp_front->m_offset);
+
+ //Writing meta data to the end of binlog file, close it after that.
+ uint32_t _buf_size = 0;
+ unsigned char * _p_meta_buf = nullptr;
+ std::tie(_p_meta_buf, _buf_size) = m_p_meta.load()->GenerateBuffer();
+
+ CHECK(std::fwrite(_p_meta_buf, _buf_size, 1, this->m_binlog_handler) == 1) << "fwrite binlog file fail...,errno:"<< errno ;
+ free(_p_meta_buf);
+ CHECK (std::fflush(this->m_binlog_handler) == 0 ) << "fflush meta data to end of binlog file fail...";
+
+ //Rename & re-open.
+ this->RenameOpenBinlogFile();
+
+ //Write the tail logs to the new binlog file
+ if (_p_tail_logs_buf != nullptr) {
+ CHECK(fwrite((void*)_p_tail_logs_buf,_tail_size,1,this->m_binlog_handler) ==1)<< "fwrite binlog file fail...,errno:"<< errno ;
+ CHECK (std::fflush(this->m_binlog_handler) == 0) << "fflush binlog file fail...,errno:" << errno;
+ }
+
+ //New file meta offset are relative to the old binlog file,adjust it to the new binlog file.
+ _p_new_meta->BackOffset(_front_size);
+
+ //Release and re-allocate meta data buf.
+ auto *_p_tmp = m_p_meta.load();
+ delete _p_tmp;
+ m_p_meta.store(_p_new_meta);
+}
+
+bool BinLogOperator::AppendEntry(const TypeEntityList &input_entities,bool force) noexcept{
+
+ //TODO: remove test code
+ //this->m_last_logged.store(ConvertID(input_entities.back()->entity_id()), std::memory_order_release);
+ //return true;
+
+ /* Prerequisite: There is no way for two or more threads calling this method
+ parallel with the same pre_log(guaranteed by the previous generating
+ guid step). Otherwise, terrible things could happen. */
+
+ if (!force && this->m_binlog_status.load() != BinlogStatus::NORMAL) {
+ LOG(ERROR) << "binlog status wrong:" << int(this->m_binlog_status.load());
+ return false;
+ }
+
+ if (input_entities.empty())
+ return true;
+
+ //For some legacy reason , we represent pre_log like this...
+ const auto &pre_log = input_entities.front()->pre_log_id();
+
+ //Log offsets to update.
+ FileMetaData::TypeOffsetList _offset_list;
+ auto _buf_info = this->PrepareBuffer(input_entities,_offset_list);
+
+ {
+ std::unique_lock _mutex_lock(m_cv_mutex);
+
+ /* Having to be sure that the last log entry which has been written to the binlog file
+ is exactly the one prior to the log that we're currently going to write .
+ Meaning that we have to wait until the last log caught up with the pre_log.
+
+ Figured out two approaches here:
+ A. use CAS weak version to wait.
+ B. use condition variable to wait.
+
+ Since approach A will consume a lot of CPU times ,I choose approach B. */
+ while (!EntityIDEqual(pre_log, this->m_last_logged.load(std::memory_order_consume))) {
+
+ auto wait_cond = [&]()->bool {return EntityIDEqual(pre_log,this->m_last_logged.load(std::memory_order_consume)); };
+
+ /* During high request load,many threads will be blocked here , and only one could
+ go further after some other threads called cv.notify_all */
+ bool waiting_result = m_cv.wait_for(_mutex_lock, std::chrono::microseconds(::RaftCore::Config::FLAGS_binlog_append_file_timeo_us), wait_cond);
+ if (!waiting_result) {
+ LOG(WARNING) << "timeout during append ,current ID-LRL logID: " << this->m_last_logged.load(std::memory_order_consume) << ",waiting on previous id :" << ConvertID(pre_log)
+ << ", this shouldn't exist too much, and will resolve quickly.";
+ /*Just continuous waiting, no need to return false or something like that . Threads who got here must finish the appending process.*/
+ continue;
+ }
+ break;
+ }
+
+ //Note:Only one thread could reading here.
+
+ //The following two steps can be merged into one .
+ this->AppendBufferToBinlog(_buf_info, _offset_list);
+
+ //Rotating binlog file...
+ this->RotateFile();
+ }
+
+ this->m_last_logged.store(ConvertID(input_entities.back()->entity_id()), std::memory_order_release);
+
+ {
+ /* Caution: modifying the conditions must be under the protect of mutex,
+ whether the conditions can be represented by an atomic object or not.
+ Reference:http://en.cppreference.com/w/cpp/thread/condition_variable */
+ std::unique_lock _mutex_lock(m_cv_mutex);
+ //Notifying doesn't need to hold the mutex.
+ m_cv.notify_all();
+ }
+
+ this->m_log_num.fetch_add((uint32_t)input_entities.size());
+
+ return true;
+}
+
+LogIdentifier BinLogOperator::GetLastReplicated() noexcept{
+ return this->m_last_logged.load(std::memory_order_consume);
+}
+
+BinLogOperator::BinlogErrorCode BinLogOperator::RevertLog(TypeMemlogFollowerList &log_list, const LogIdentifier &boundary) noexcept {
+
+ const auto & pre_entity_id = log_list.front()->GetEntity()->pre_log_id();
+
+ //Find the last entry that being consistent with the first element of log_list.
+ std::list> _ordered_meta;
+ m_p_meta.load()->GetOrderedMeta(_ordered_meta);
+
+ auto _criter_meta = _ordered_meta.crbegin();
+ for (; _criter_meta != _ordered_meta.crend(); _criter_meta++) {
+ if ((*_criter_meta)->operator!=(pre_entity_id)) {
+ continue;
+ }
+ break;
+ }
+
+ CHECK(_criter_meta != _ordered_meta.crend()) << "Reverting log : cannot find pre_entity_id,something must be wrong";
+
+ auto _citer_meta = _criter_meta.base();
+ bool _found_consistent_log = false;
+
+ for (auto _citer_log = log_list.cbegin(); _citer_log != log_list.cend();) {
+ const std::string &_key = (*_citer_log)->GetEntity()->write_op().key();
+ const std::string &_value = (*_citer_log)->GetEntity()->write_op().value();
+
+ uint32_t _key_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_key.data(), (unsigned int)_key.length());
+ uint32_t _value_crc32 = ::RaftCore::Tools::CalculateCRC32((void*)_value.data(), (unsigned int)_value.length());
+
+ bool _item_equal = (*_citer_meta)->key_crc32 == _key_crc32 && (*_citer_meta)->value_crc32 == _value_crc32;
+ if ( !_item_equal )
+ break;
+
+ _found_consistent_log = true;
+
+ //Advance input log iterator.
+ _citer_log = log_list.erase(_citer_log);
+
+ //Advance binlog meta iterator.
+ _citer_meta++;
+ if (_citer_meta == _ordered_meta.cend())
+ break;
+ }
+
+ /* There are several scenarios in reverting:
+ 1> log_list & binlog can find a consistent log entry:
+ assuming _log_listX is the sublist of log_list,with its first element is the first inconsistent
+ entry with binlog :
+
+ 1) _log_listX is empty: entries in log_list are all the same as binlog, nothing to revert.
+ 2) _log_listX has no intersection with binlog: entries in log_list will be appended to binlog
+ without any erasing operations.
+ 3) _log_listX has intersection with binlog : the conflict entries in the binlog will be replaced
+ by _log_listX.
+
+ 2> log_list & binlog can't find a consistent consistent log entry:
+ return a NO-CONSITENT error.
+ */
+ if (!_found_consistent_log)
+ return BinlogErrorCode::NO_CONSISTENT_ERROR;
+
+ //No need to revert anything.
+ if (log_list.empty())
+ return BinlogErrorCode::SUCCEED_MERGED;
+
+ const auto &_first_inconsistent_log = log_list.front();
+ if (!EntityIDLarger(_first_inconsistent_log->GetEntity()->entity_id(), boundary))
+ return BinlogErrorCode::OVER_BOUNDARY;
+
+ BinlogStatus _cur_status = BinlogStatus::NORMAL;
+ if (!this->m_binlog_status.compare_exchange_strong(_cur_status,BinlogStatus::REVERTING)) {
+ //Other threads may have already modified the status variable.
+ return BinlogErrorCode::OTHER_ERROR;
+ }
+
+ bool binlog_ended = (_citer_meta == _ordered_meta.cend());
+ std::string _revert_point = binlog_ended ? "end of binlog file" : (*_citer_meta)->ToString();
+ LOG(INFO) << "log reverting start...,detail:" << _revert_point;
+
+ //Only in the following case should we do reverting.
+ if (!binlog_ended) {
+
+ //Remove meta data.
+ auto _remove_iter = _citer_meta;
+ while (_remove_iter !=_ordered_meta.cend()) {
+ m_p_meta.load()->Delete(**_remove_iter);
+ _remove_iter++;
+ }
+
+ //Update ID-LRL and notify the other threads who are waiting on it.
+ std::unique_lock _mutex_lock(m_cv_mutex);
+
+ this->Truncate((*_citer_meta)->m_offset);
+
+ _citer_meta--;
+ LogIdentifier _id_lrl;
+ _id_lrl.Set((*_citer_meta)->m_term,(*_citer_meta)->m_index );
+ this->m_last_logged.store(_id_lrl, std::memory_order_release);
+ m_cv.notify_all();
+ }
+
+ //Appending new entries.
+ std::list> _entities_list;
+ for (const auto& _item : log_list)
+ _entities_list.emplace_back(_item->GetEntity());
+
+ CHECK(AppendEntry(_entities_list,true)) << "AppendEntry to binlog fail,never should this happen,something terribly wrong.";
+
+ //No exceptions could happened here.
+ _cur_status = BinlogStatus::REVERTING;
+ CHECK(this->m_binlog_status.compare_exchange_strong(_cur_status, BinlogStatus::NORMAL)) << "Binlog reverting : status CAS failed,something terribly wrong";
+
+ return BinlogErrorCode::SUCCEED_TRUNCATED;
+}
+
+BinLogOperator::BinlogErrorCode BinLogOperator::SetHead(std::shared_ptr<::raft::Entity> _shp_entity) noexcept {
+
+ BinlogStatus _cur_status = BinlogStatus::NORMAL;
+ if (!this->m_binlog_status.compare_exchange_strong(_cur_status,BinlogStatus::SETTING_HEAD)) {
+ //Other threads may have already modified the status variable.
+ return BinlogErrorCode::OTHER_ERROR;
+ }
+
+ CHECK(std::fclose(this->m_binlog_handler)==0) << "truncating file fclose failed";
+ if (fs::exists(fs::path(this->m_file_name)))
+ CHECK(std::remove(this->m_file_name.c_str())==0);
+ this->m_binlog_handler = std::fopen(this->m_file_name.c_str(), _AURORA_BINLOG_OP_MODE_);
+ CHECK(this->m_binlog_handler != nullptr) << "open binlog file " << this->m_file_name << "fail..,errno:" << errno;
+
+ //Clear and set up new meta info.
+ auto *_p_tmp = m_p_meta.load();
+ delete _p_tmp;
+ m_p_meta.store(new FileMetaData());
+
+ m_zero_log_id.Set(0, 0);
+ this->m_last_logged.store(m_zero_log_id, std::memory_order_release);
+
+ /*Since the binlog file has already just been reset,pre_log_id should be set to the initial id,only after that
+ the data could be written into the binlog file. */
+ auto _p_pre_log_id = _shp_entity->mutable_pre_log_id();
+ _p_pre_log_id->set_term(0);
+ _p_pre_log_id->set_idx(0);
+
+ TypeEntityList _input_list;
+ _input_list.emplace_back(_shp_entity);
+ CHECK(AppendEntry(_input_list,true)) << "AppendEntry to binlog fail,never should this happen,something terribly wrong.";
+
+ //No exceptions could happened here.
+ _cur_status = BinlogStatus::SETTING_HEAD;
+ CHECK(this->m_binlog_status.compare_exchange_strong(_cur_status, BinlogStatus::NORMAL)) << "Binlog setting head : status CAS failed,something terribly wrong";
+
+ return BinlogErrorCode::SUCCEED_TRUNCATED;
+}
+
+bool BinLogOperator::Clear() noexcept {
+ if (this->m_binlog_status.load() != BinlogStatus::NORMAL)
+ return false;
+
+ this->m_last_logged.store(m_zero_log_id, std::memory_order_release);
+
+ this->DeleteOpenBinlogFile();
+
+ auto *_p_tmp = m_p_meta.load();
+ delete _p_tmp;
+ m_p_meta.store(new FileMetaData());
+
+ return true;
+}
+
+void BinLogOperator::DeleteOpenBinlogFile() noexcept {
+
+ CHECK(std::fclose(this->m_binlog_handler)==0);
+
+ LOG(INFO) << "deleting current running binlog:" << this->m_file_name;
+
+ int _ret = std::remove(this->m_file_name.c_str());
+ CHECK(_ret == 0) << ",delete fail,errno:" << errno;
+
+ //Open & create new binlog file.
+ this->m_binlog_handler = std::fopen(this->m_file_name.c_str(),_AURORA_BINLOG_OP_MODE_);
+ CHECK(this->m_binlog_handler != nullptr) << "rotating, fopen binlog file fail...,errno:" << errno;
+}
+
+void BinLogOperator::RenameOpenBinlogFile() noexcept{
+ //Scan binlog files.
+ fs::path _path(".");
+ CHECK (fs::is_directory(_path)) << "scan current directory fail,cannot save current file";
+
+ int max_suffix = 0;
+ for (auto&& x : fs::directory_iterator(_path)) {
+ std::string file_name = x.path().filename().string();
+ std::string::size_type pos = file_name.find(this->m_file_name);
+ if (pos == std::string::npos)
+ continue ;
+
+ int suffix = 0;
+ if (file_name != this->m_file_name)
+ suffix = std::atol(file_name.substr(pos + this->m_file_name.length() + 1).c_str());
+
+ max_suffix = std::max(suffix,max_suffix);
+ }
+
+ CHECK(std::fclose(this->m_binlog_handler)==0);
+
+ //Rename current file.
+ char sz_new_name[1024] = { 0 };
+ std::snprintf(sz_new_name, sizeof(sz_new_name),"%s-%d",this->m_file_name.c_str() ,max_suffix + 1);
+ CHECK (std::rename(this->m_file_name.c_str(), sz_new_name) == 0) << "rename binlog file fail...,errno:" << errno;
+
+ //Open & create new binlog file.
+ this->m_binlog_handler = std::fopen(this->m_file_name.c_str(),_AURORA_BINLOG_OP_MODE_);
+ CHECK (this->m_binlog_handler != nullptr) << "rotating, fopen binlog file fail...,errno:" << errno;
+}
+
+std::string BinLogOperator::GetBinlogFileName() noexcept {
+ return this->m_file_name;
+}
+
+void BinLogOperator::GetOrderedMeta(FileMetaData::TypeOffsetList &_output) noexcept {
+ m_p_meta.load()->GetOrderedMeta(_output);
+}
+
+}
\ No newline at end of file
diff --git a/src/binlog/binlog_operator.h b/src/binlog/binlog_operator.h
new file mode 100644
index 0000000..23ef9f4
--- /dev/null
+++ b/src/binlog/binlog_operator.h
@@ -0,0 +1,171 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_BIN_LOG_OPERATOR_H__
+#define __AURORA_BIN_LOG_OPERATOR_H__
+
+#include
+#include
+#include
+
+#include "protocol/raft.pb.h"
+
+#include "common/comm_defs.h"
+#include "common/log_identifier.h"
+#include "follower/memory_log_follower.h"
+#include "binlog/binlog_meta_data.h"
+
+#define _AURORA_BINLOG_NAME_ "raft.binlog"
+#define _AURORA_BINLOG_NAME_REG_ "raft\\.binlog"
+#define _AURORA_BINLOG_OP_MODE_ "ab+"
+#define _AURORA_BINLOG_READ_MODE_ "rb"
+
+namespace RaftCore::BinLog {
+
+using ::raft::Entity;
+using ::RaftCore::Common::LogIdentifier;
+using ::RaftCore::Follower::TypeMemlogFollowerList;
+using ::RaftCore::Common::TypeEntityList;
+using ::RaftCore::Common::TypeBufferInfo;
+using ::RaftCore::BinLog::FileMetaData;
+
+class BinLogOperator final{
+
+public:
+
+ enum class BinlogErrorCode {
+ UNKNOWN = 0,
+ SUCCEED_TRUNCATED,
+ SUCCEED_MERGED, //Input logs are all existed in the bin log.
+
+ //------Separating line.------//
+ SUCCEED_MAX,
+
+ OVER_BOUNDARY,
+ NO_CONSISTENT_ERROR,
+ OTHER_ERROR,
+ };
+
+ enum class BinlogStatus {
+
+ //Normal status , could be written new log entries
+ NORMAL,
+
+ //Reverting due a log conflict with (probably new) leader's log.
+ REVERTING,
+
+ //Set the first log entry in the binlog file, used in the SyncData scenario.
+ SETTING_HEAD
+ };
+
+public:
+
+ BinLogOperator();
+
+ virtual ~BinLogOperator()noexcept;
+
+ void Initialize(const char* role, bool file_name = false) noexcept;
+
+ void UnInitialize() noexcept;
+
+ //The '_input' parameter must be ordered by log index in ascending order.
+ bool AppendEntry(const TypeEntityList &input_entities,bool force=false) noexcept;
+
+ /*Although it is supported, reverting log actually won't be executed with AppendEntry simultaneously ,
+ which is guaranteed by the callers.*/
+ BinlogErrorCode RevertLog(TypeMemlogFollowerList &log_list, const LogIdentifier &boundary) noexcept;
+
+ /*Although it is supported, SetHead log actually won't be executed with AppendEntry simultaneously ,
+ which is done by the callers.*/
+ BinlogErrorCode SetHead(std::shared_ptr<::raft::Entity> _shp_entity) noexcept;
+
+ bool Clear() noexcept;
+
+ LogIdentifier GetLastReplicated() noexcept;
+
+ std::string GetBinlogFileName() noexcept;
+
+ void GetOrderedMeta(FileMetaData::TypeOffsetList &_output) noexcept;
+
+ //bool IsTermMatched() noexcept;
+
+ //void SetTermMatched() noexcept;
+
+ void AddPreLRLUseCount() noexcept;
+
+ void SubPreLRLUseCount() noexcept;
+
+private:
+
+ void ParseFile() noexcept;
+
+ void DeleteOpenBinlogFile() noexcept;
+
+ void RenameOpenBinlogFile() noexcept;
+
+ void Truncate(uint32_t new_size) noexcept;
+
+ //The following three member functions are helpers of AppendEntry.
+ TypeBufferInfo PrepareBuffer(const TypeEntityList &input_entities, FileMetaData::TypeOffsetList &offset_list) noexcept;
+
+ void AppendBufferToBinlog(TypeBufferInfo &buffer_info, const FileMetaData::TypeOffsetList &offset_list) noexcept;
+
+ void RotateFile() noexcept;
+
+private:
+
+ LogIdentifier m_zero_log_id;
+
+ bool m_initialized = false;
+
+ //std::atomic m_term_changed = false;
+
+ std::atomic m_last_logged;
+
+ std::string m_file_name = "";
+
+ std::FILE *m_binlog_handler = nullptr;
+
+ std::mutex m_cv_mutex;
+
+ std::condition_variable m_cv;
+
+ std::atomic m_p_meta;
+
+ std::atomic m_binlog_status;
+
+ std::atomic m_precede_lcl_inuse;
+
+ std::atomic m_log_num;
+
+private:
+
+ BinLogOperator(const BinLogOperator&) = delete;
+
+ BinLogOperator& operator=(const BinLogOperator&) = delete;
+
+};
+
+
+
+} //end namespace
+
+
+#endif
diff --git a/src/binlog/binlog_singleton.cc b/src/binlog/binlog_singleton.cc
new file mode 100644
index 0000000..1dd684c
--- /dev/null
+++ b/src/binlog/binlog_singleton.cc
@@ -0,0 +1,25 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "binlog/binlog_singleton.h"
+
+namespace RaftCore::BinLog {
+
+BinLogOperator BinLogGlobal::m_instance;
+
+}
\ No newline at end of file
diff --git a/src/binlog/binlog_singleton.h b/src/binlog/binlog_singleton.h
new file mode 100644
index 0000000..a2e40b3
--- /dev/null
+++ b/src/binlog/binlog_singleton.h
@@ -0,0 +1,51 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_BIN_LOG_SINGLETON_H__
+#define __AURORA_BIN_LOG_SINGLETON_H__
+
+#include "binlog/binlog_operator.h"
+
+namespace RaftCore::BinLog {
+
+using ::RaftCore::BinLog::BinLogOperator;
+
+class BinLogGlobal final{
+
+public:
+
+ static BinLogOperator m_instance;
+
+private:
+
+ BinLogGlobal() = delete;
+
+ virtual ~BinLogGlobal() = delete;
+
+ BinLogGlobal(const BinLogGlobal&) = delete;
+
+ BinLogGlobal& operator=(const BinLogGlobal&) = delete;
+
+};
+
+} //end namespace
+
+
+#endif
diff --git a/src/candidate/candidate_request.cc b/src/candidate/candidate_request.cc
new file mode 100644
index 0000000..65b0afa
--- /dev/null
+++ b/src/candidate/candidate_request.cc
@@ -0,0 +1,31 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "candidate/candidate_request.h"
+
+namespace RaftCore::Candidate {
+
+template
+CandidateUnaryRequest::CandidateUnaryRequest() noexcept {}
+
+template
+CandidateUnaryRequest::~CandidateUnaryRequest() noexcept {}
+
+
+}
+
diff --git a/src/candidate/candidate_request.h b/src/candidate/candidate_request.h
new file mode 100644
index 0000000..f7e8d39
--- /dev/null
+++ b/src/candidate/candidate_request.h
@@ -0,0 +1,57 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_CANDIDATE_REQUEST_H__
+#define __AURORA_CANDIDATE_REQUEST_H__
+
+#include
+
+#include "protocol/raft.grpc.pb.h"
+#include "protocol/raft.pb.h"
+
+#include "common/request_base.h"
+
+using ::raft::RaftService;
+using ::grpc::ServerCompletionQueue;
+using ::RaftCore::Common::UnaryRequest;
+
+namespace RaftCore::Candidate {
+
+template
+class CandidateUnaryRequest : public UnaryRequest {
+
+public:
+
+ CandidateUnaryRequest()noexcept;
+
+ virtual ~CandidateUnaryRequest()noexcept;
+
+private:
+
+ CandidateUnaryRequest(const CandidateUnaryRequest&) = delete;
+
+ CandidateUnaryRequest& operator=(const CandidateUnaryRequest&) = delete;
+};
+
+} //end namespace
+
+#include "candidate_request.cc"
+
+#endif
diff --git a/src/candidate/candidate_view.cc b/src/candidate/candidate_view.cc
new file mode 100644
index 0000000..d390467
--- /dev/null
+++ b/src/candidate/candidate_view.cc
@@ -0,0 +1,32 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "candidate/candidate_view.h"
+
+namespace RaftCore::Candidate {
+
+void CandidateView::Initialize() noexcept{
+ CommonView::Initialize();
+}
+
+void CandidateView::UnInitialize() noexcept {
+ CommonView::UnInitialize();
+}
+
+}
+
diff --git a/src/candidate/candidate_view.h b/src/candidate/candidate_view.h
new file mode 100644
index 0000000..d46f641
--- /dev/null
+++ b/src/candidate/candidate_view.h
@@ -0,0 +1,56 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_CANDIDATE_VIEW_H__
+#define __AURORA_CANDIDATE_VIEW_H__
+
+#include
+#include
+
+#include "common/comm_view.h"
+
+namespace RaftCore::Candidate {
+
+using ::RaftCore::Common::CommonView;
+
+//CandidateView is nothing inside, all logic in candidate is in the `Election` module.
+class CandidateView final: public CommonView{
+
+public:
+
+ static void Initialize() noexcept;
+
+ static void UnInitialize() noexcept;
+
+private:
+
+ CandidateView() = delete;
+
+ virtual ~CandidateView() = delete;
+
+ CandidateView(const CandidateView&) = delete;
+
+ CandidateView& operator=(const CandidateView&) = delete;
+
+};
+
+} //end namespace
+
+#endif
diff --git a/src/client/client_base.cc b/src/client/client_base.cc
new file mode 100644
index 0000000..c473ebb
--- /dev/null
+++ b/src/client/client_base.cc
@@ -0,0 +1,35 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "client/client_base.h"
+
+namespace RaftCore::Client {
+
+ClientBase::ClientBase() {}
+
+ClientBase::~ClientBase() {}
+
+void ClientBase::PushCallBackArgs(void* cb_data) noexcept{
+ this->m_callback_args.push_back(cb_data);
+}
+
+void ClientBase::ClearCallBackArgs() noexcept{
+ this->m_callback_args.clear();
+}
+
+}
diff --git a/src/client/client_base.h b/src/client/client_base.h
new file mode 100644
index 0000000..c0a482c
--- /dev/null
+++ b/src/client/client_base.h
@@ -0,0 +1,54 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_CLIENT_BASE_H__
+#define __AURORA_CLIENT_BASE_H__
+
+#include
+
+namespace RaftCore::Client {
+
+//For the prospective common properties .
+class ClientBase {
+
+public:
+
+ ClientBase();
+
+ virtual ~ClientBase();
+
+ void PushCallBackArgs(void* cb_data)noexcept;
+
+protected:
+
+ void ClearCallBackArgs()noexcept;
+
+ std::vector m_callback_args;
+
+private:
+
+ ClientBase(const ClientBase&) = delete;
+
+ ClientBase& operator=(const ClientBase&) = delete;
+};
+
+}
+
+#endif
diff --git a/src/client/client_framework.cc b/src/client/client_framework.cc
new file mode 100644
index 0000000..834c918
--- /dev/null
+++ b/src/client/client_framework.cc
@@ -0,0 +1,210 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "glog/logging.h"
+
+#include "client/client_framework.h"
+
+namespace RaftCore::Client {
+
+template
+ClientFramework::ClientFramework(std::shared_ptr shp_channel) noexcept{
+ this->m_stub = ::raft::RaftService::NewStub(shp_channel);
+ this->Reset();
+}
+
+template
+void ClientFramework::Reset() noexcept {
+ this->m_client_context.reset(new ::grpc::ClientContext());
+}
+
+template
+std::shared_ptr<::raft::RaftService::Stub> ClientFramework::GetStub() noexcept {
+ return this->m_stub;
+}
+
+template
+ClientFramework::~ClientFramework() noexcept{}
+
+template
+ClientTpl::ClientTpl(std::shared_ptr shp_channel) noexcept : ClientFramework(shp_channel) {
+ this->m_shp_request.reset(new T());
+}
+
+template
+ClientTpl::~ClientTpl() noexcept {}
+
+template
+SyncClient::SyncClient(std::shared_ptr shp_channel) noexcept : ClientTpl(shp_channel) {}
+
+template
+SyncClient::~SyncClient() noexcept {}
+
+template
+AsyncClient::AsyncClient(std::shared_ptr shp_channel,
+ std::shared_ptr shp_cq)noexcept : ClientTpl(shp_channel),
+ m_server_cq(shp_cq) {}
+
+template
+AsyncClient::~AsyncClient()noexcept {}
+
+template
+UnarySyncClient::UnarySyncClient(std::shared_ptr shp_channel)noexcept :
+ SyncClient(shp_channel) {}
+
+template
+const R& UnarySyncClient::DoRPC(std::function&)> req_setter,
+ std::function<::grpc::Status(::grpc::ClientContext*,const T&,R*)> rpc, uint32_t timeo_ms,
+ ::grpc::Status &ret_status)noexcept {
+
+ req_setter(this->m_shp_request);
+ std::chrono::system_clock::time_point deadline = std::chrono::system_clock::now() +
+ std::chrono::milliseconds(timeo_ms);
+ this->m_client_context->set_deadline(deadline);
+
+ ret_status = rpc(this->m_client_context.get(), *this->m_shp_request, &this->m_response);
+ return this->m_response;
+}
+
+template
+UnarySyncClient::~UnarySyncClient()noexcept {}
+
+template
+BidirectionalSyncClient::BidirectionalSyncClient(std::shared_ptr shp_channel)noexcept :
+ SyncClient(shp_channel) {}
+
+template
+BidirectionalSyncClient::~BidirectionalSyncClient()noexcept {}
+
+template
+UnaryAsyncClient::UnaryAsyncClient(std::shared_ptr shp_channel,
+ std::shared_ptr shp_cq) noexcept : AsyncClient(shp_channel, shp_cq) {
+ static_assert(std::is_base_of::value, "Q is not a derived from UnaryAsyncClient.");
+}
+
+template
+void UnaryAsyncClient::React(bool cq_result) noexcept {
+
+ if (!cq_result) {
+ LOG(ERROR) << "UnaryAsyncClient got false result from CQ.";
+ this->Release();
+ return;
+ }
+
+ this->Responder(this->m_final_status, this->m_response);
+ this->Release();
+}
+
+template
+void UnaryAsyncClient::Release() noexcept {
+ delete dynamic_cast(this);
+}
+
+template
+void UnaryAsyncClient::EntrustRequest(const std::function&)> &req_setter,
+ const FPrepareAsync &f_prepare_async, uint32_t timeo_ms) noexcept {
+ req_setter(this->m_shp_request);
+
+ std::chrono::time_point _deadline = std::chrono::system_clock::now() +
+ std::chrono::milliseconds(timeo_ms);
+ this->m_client_context->set_deadline(_deadline);
+
+ this->m_reader = f_prepare_async(this->m_client_context.get(), *this->m_shp_request,
+ this->m_server_cq.get());
+ this->m_reader->StartCall();
+ this->m_reader->Finish(&this->m_response, &this->m_final_status, dynamic_cast(this));
+}
+
+template
+UnaryAsyncClient::~UnaryAsyncClient() noexcept {}
+
+template
+BidirectionalAsyncClient::BidirectionalAsyncClient(std::shared_ptr shp_channel,
+ std::shared_ptr shp_cq) noexcept : AsyncClient(shp_channel, shp_cq),
+ m_async_rw(this->m_client_context.get()), m_status(ProcessStage::CONNECT) {
+ static_assert(std::is_base_of::value, "Q is not a derived from BidirectionalAsyncClient.");
+}
+
+template
+BidirectionalAsyncClient::~BidirectionalAsyncClient() noexcept {}
+
+template
+void BidirectionalAsyncClient::React(bool cq_result) noexcept {
+
+ Q* _p_downcast = dynamic_cast(this);
+
+ if (!cq_result && (this->m_status != ProcessStage::READ)) {
+ LOG(ERROR) << "BidirectionalAsyncClient got false result from CQ.";
+ delete _p_downcast;
+ return;
+ }
+
+ switch (this->m_status) {
+ case ProcessStage::READ:
+
+ //Meaning client said it wants to end the stream either by a 'WritesDone' or 'finish' call.
+ if (!cq_result) {
+ this->m_async_rw.Finish(this->m_final_status, _p_downcast);
+ this->m_status = ProcessStage::FINISH;
+ break;
+ }
+
+ this->m_responder(this->m_final_status, this->m_response);
+ break;
+
+ case ProcessStage::WRITE:
+ this->m_async_rw.Read(&this->m_response, _p_downcast);
+ this->m_status = ProcessStage::READ;
+ break;
+
+ case ProcessStage::CONNECT:
+ break;
+
+ case ProcessStage::WRITES_DONE:
+ this->m_async_rw.Finish(this->m_final_status, _p_downcast);
+ this->m_status = ProcessStage::FINISH;
+ break;
+
+ case ProcessStage::FINISH:
+ if (this->m_final_status.error_code() != ::grpc::StatusCode::OK) {
+ LOG(ERROR) << "rpc fail,err code:" << this->m_final_status.error_code()
+ << ",err msg:" << this->m_final_status.error_message();
+ }
+ delete _p_downcast;
+ break;
+
+ default:
+ CHECK(false) << "Unexpected tag " << int(this->m_status);
+ }
+}
+
+template
+void BidirectionalAsyncClient::AsyncDo(std::function&)> req_setter) noexcept {
+ req_setter(this->m_shp_request);
+ this->m_async_rw.Write(this->m_shp_request,dynamic_cast(this));
+ this->m_status = ProcessStage::WRITE;
+}
+
+template
+void BidirectionalAsyncClient::WriteDone() noexcept {
+ this->m_async_rw.WritesDone(dynamic_cast(this));
+ this->m_status = ProcessStage::WRITES_DONE;
+}
+
+}
+
diff --git a/src/client/client_framework.h b/src/client/client_framework.h
new file mode 100644
index 0000000..08a3deb
--- /dev/null
+++ b/src/client/client_framework.h
@@ -0,0 +1,239 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#pragma once
+
+#ifndef __AURORA_CLIENT_FRAMEWORK_H__
+#define __AURORA_CLIENT_FRAMEWORK_H__
+
+#include
+
+#include "grpc/grpc.h"
+#include "grpc++/grpc++.h"
+
+#include "protocol/raft.grpc.pb.h"
+#include "common/react_base.h"
+
+namespace RaftCore::Client {
+
+using ::grpc::CompletionQueue;
+//using ::grpc::ServerCompletionQueue;
+using ::grpc::Channel;
+using ::RaftCore::Common::ReactBase;
+
+template
+using FPrepareAsync = std::function>(
+ ::grpc::ClientContext*,const T&, ::grpc::CompletionQueue*)>;
+
+template
+class ClientFramework {
+
+public:
+
+ ClientFramework(std::shared_ptr shp_channel)noexcept;
+
+ virtual ~ClientFramework()noexcept;
+
+ //Reset a client object for reusing purpose.
+ virtual void Reset() noexcept;
+
+ virtual std::shared_ptr<::raft::RaftService::Stub> GetStub() noexcept final;
+
+protected:
+
+ std::shared_ptr<::grpc::ClientContext> m_client_context;
+
+ std::shared_ptr<::raft::RaftService::Stub> m_stub;
+
+ ::grpc::Status m_final_status;
+
+private:
+
+ ClientFramework(const ClientFramework&) = delete;
+
+ ClientFramework& operator=(const ClientFramework&) = delete;
+};
+
+template
+class ClientTpl : public ClientFramework {
+
+public:
+
+ typedef std::function TypeResponder;
+
+ ClientTpl(std::shared_ptr shp_channel)noexcept;
+
+ virtual ~ClientTpl()noexcept;
+
+protected:
+
+ std::shared_ptr m_shp_request;
+
+ R m_response;
+
+private:
+
+ ClientTpl(const ClientTpl&) = delete;
+
+ ClientTpl& operator=(const ClientTpl&) = delete;
+};
+
+template
+class SyncClient : public ClientTpl {
+
+public:
+
+ SyncClient(std::shared_ptr shp_channel)noexcept;
+
+ virtual ~SyncClient()noexcept;
+
+private:
+
+ SyncClient(const SyncClient&) = delete;
+
+ SyncClient& operator=(const SyncClient&) = delete;
+};
+
+template
+class UnarySyncClient : public SyncClient {
+
+public:
+
+ UnarySyncClient(std::shared_ptr shp_channel)noexcept;
+
+ virtual ~UnarySyncClient()noexcept;
+
+ const R& DoRPC(std::function&)> req_setter,
+ std::function<::grpc::Status(::grpc::ClientContext*,const T&,R*)> rpc, uint32_t timeo_ms,
+ ::grpc::Status &ret_status)noexcept;
+
+private:
+
+ UnarySyncClient(const UnarySyncClient&) = delete;
+
+ UnarySyncClient& operator=(const UnarySyncClient&) = delete;
+};
+
+template
+class BidirectionalSyncClient : public SyncClient {
+
+public:
+
+ BidirectionalSyncClient(std::shared_ptr shp_channel)noexcept;
+
+ virtual ~BidirectionalSyncClient()noexcept;
+
+protected:
+
+ std::shared_ptr<::grpc::ClientReaderWriter> m_sync_rw;
+
+private:
+
+ BidirectionalSyncClient(const BidirectionalSyncClient&) = delete;
+
+ BidirectionalSyncClient& operator=(const BidirectionalSyncClient&) = delete;
+};
+
+template
+class AsyncClient : public ClientTpl, public ReactBase {
+
+public:
+
+ AsyncClient(std::shared_ptr shp_channel,std::shared_ptr shp_cq)noexcept;
+
+ virtual ~AsyncClient()noexcept;
+
+ virtual void Responder(const ::grpc::Status& status, const R& rsp) noexcept = 0;
+
+protected:
+
+ std::shared_ptr m_server_cq;
+
+private:
+
+ AsyncClient(const AsyncClient&) = delete;
+
+ AsyncClient& operator=(const AsyncClient&) = delete;
+};
+
+template
+class UnaryAsyncClient : public AsyncClient {
+
+public:
+
+ UnaryAsyncClient(std::shared_ptr shp_channel,std::shared_ptr shp_cq)noexcept;
+
+ virtual ~UnaryAsyncClient()noexcept;
+
+ void EntrustRequest(const std::function&)> &req_setter,
+ const FPrepareAsync &f_prepare_async, uint32_t timeo_ms) noexcept;
+
+protected:
+
+ virtual void React(bool cq_result) noexcept override;
+
+ virtual void Release() noexcept;
+
+protected:
+
+ std::unique_ptr<::grpc::ClientAsyncResponseReader> m_reader;
+
+private:
+
+ UnaryAsyncClient(const UnaryAsyncClient&) = delete;
+
+ UnaryAsyncClient& operator=(const UnaryAsyncClient&) = delete;
+};
+
+template
+class BidirectionalAsyncClient : public AsyncClient {
+
+public:
+
+ BidirectionalAsyncClient(std::shared_ptr shp_channel, std::shared_ptr shp_cq)noexcept;
+
+ virtual ~BidirectionalAsyncClient()noexcept;
+
+protected:
+
+ virtual void React(bool cq_result) noexcept override;
+
+ void AsyncDo(std::function&)> req_setter) noexcept;
+
+ void WriteDone() noexcept;
+
+protected:
+
+ ::grpc::ClientAsyncReaderWriter m_async_rw;
+
+ enum ProcessStage { READ = 1, WRITE = 2, CONNECT = 3, WRITES_DONE = 4, FINISH = 5 };
+
+ ProcessStage m_status;
+
+private:
+
+ BidirectionalAsyncClient(const BidirectionalAsyncClient&) = delete;
+
+ BidirectionalAsyncClient& operator=(const BidirectionalAsyncClient&) = delete;
+};
+
+} //end namespace
+
+#include "client/client_framework.cc"
+
+#endif
diff --git a/src/client/client_impl.cc b/src/client/client_impl.cc
new file mode 100644
index 0000000..e2984fb
--- /dev/null
+++ b/src/client/client_impl.cc
@@ -0,0 +1,194 @@
+/*
+*
+* Copyright (C) <2019>
+
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+
+* You should have received a copy of the GNU General Public License
+* along with this program. If not, see .
+*/
+
+#include "service/service.h"
+#include "leader/client_pool.h"
+#include "member/member_manager.h"
+#include "election/election.h"
+#include "client/client_impl.h"
+
+namespace RaftCore::Client {
+
+using ::RaftCore::Common::VoteType;
+using ::RaftCore::Service::Write;
+using ::RaftCore::Leader::ClientPool;
+using ::RaftCore::Member::MemberMgr;
+using ::RaftCore::Election::ElectionMgr;
+
+AppendEntriesAsyncClient::AppendEntriesAsyncClient(std::shared_ptr<::grpc::Channel> shp_channel,
+ std::shared_ptr<::grpc::CompletionQueue> shp_cq)
+ : UnaryAsyncClient<::raft::AppendEntriesRequest, ::raft::AppendEntriesResponse,
+ AppendEntriesAsyncClient>(shp_channel, shp_cq) {
+
+ //Give myself a long lived delegator.
+ this->OwnershipDelegator::ResetOwnership(this);
+}
+
+AppendEntriesAsyncClient::~AppendEntriesAsyncClient() {}
+
+void AppendEntriesAsyncClient::Responder(const ::grpc::Status& status,
+ const ::raft::AppendEntriesResponse& rsp) noexcept {
+
+ auto _shp_write = this->OwnershipDelegator::GetOwnership();
+ auto *_p_conn_pool = (ClientPool*)this->m_callback_args[0];
+
+ _shp_write->ReplicateDoneCallBack(status, rsp, _p_conn_pool->GetParentFollower(), this);
+}
+
+void AppendEntriesAsyncClient::Release() noexcept {
+ //Reset myself.
+ this->Reset();
+
+ //Release associated write request.
+ this->OwnershipDelegator::ReleaseOwnership();
+
+ //Push myself back to the connection pool.
+ auto *_p_conn_pool = (ClientPool*)this->m_callback_args[0];
+ auto _shp_copied = this->OwnershipDelegator::GetOwnership();
+ _p_conn_pool->Back(_shp_copied);
+
+ VLOG(90) << "AppendEntriesAsyncClient returned:" << _p_conn_pool->GetParentFollower()->my_addr;
+
+ //Clear my args.
+ this->ClearCallBackArgs();
+}
+
+CommitEntriesAsyncClient::CommitEntriesAsyncClient(std::shared_ptr<::grpc::Channel> shp_channel,
+ std::shared_ptr<::grpc::CompletionQueue> shp_cq)
+ : UnaryAsyncClient<::raft::CommitEntryRequest, ::raft::CommitEntryResponse,
+ CommitEntriesAsyncClient> (shp_channel, shp_cq){
+
+ //Give myself a long lived delegator.
+ this->OwnershipDelegator::ResetOwnership(this);
+}
+
+CommitEntriesAsyncClient::~CommitEntriesAsyncClient() {}
+
+void CommitEntriesAsyncClient::Responder(const ::grpc::Status& status,
+ const ::raft::CommitEntryResponse& rsp) noexcept {
+
+ auto _shp_write = this->OwnershipDelegator::GetOwnership();
+ auto *_p_conn_pool = (ClientPool*)this->m_callback_args[0];
+
+ _shp_write->CommitDoneCallBack(status, rsp, _p_conn_pool->GetParentFollower());
+}
+
+void CommitEntriesAsyncClient::Release() noexcept {
+ //Reset myself.
+ this->Reset();
+
+ //Release associated write request.
+ this->OwnershipDelegator::ReleaseOwnership();
+
+ //Push myself back to the connection pool.
+ auto *_p_conn_pool = (ClientPool*)this->m_callback_args[0];
+
+ auto _shp_copied = this->OwnershipDelegator::GetOwnership();
+ _p_conn_pool->Back(_shp_copied);
+
+ VLOG(90) << "CommitEntriesAsyncClient returned:" << _p_conn_pool->GetParentFollower()->my_addr;
+
+ //Clear my args.
+ this->ClearCallBackArgs();
+}
+
+HeartbeatSyncClient::HeartbeatSyncClient(std::shared_ptr<::grpc::Channel> shp_channel):
+ UnarySyncClient<::raft::HeartBeatRequest, ::raft::CommonResponse>(shp_channel) {}
+
+HeartbeatSyncClient::~HeartbeatSyncClient() {}
+
+WriteSyncClient::WriteSyncClient(std::shared_ptr<::grpc::Channel> shp_channel):
+ UnarySyncClient<::raft::ClientWriteRequest, ::raft::ClientWriteResponse>(shp_channel) {}
+
+WriteSyncClient::~WriteSyncClient() {}
+
+SyncDataSyncClient::SyncDataSyncClient(std::shared_ptr<::grpc::Channel> shp_channel):
+ BidirectionalSyncClient<::raft::SyncDataRequest, ::raft::SyncDataResponse>(shp_channel) {
+ this->m_sync_rw = this->m_stub->SyncData(this->m_client_context.get());
+}
+
+SyncDataSyncClient::~SyncDataSyncClient() {}
+
+auto SyncDataSyncClient::GetInstantiatedReq()noexcept -> decltype(m_shp_request) {
+ if (!this->m_shp_request)
+ this->m_shp_request.reset(new ::raft::SyncDataRequest());
+
+ return this->m_shp_request;
+}
+
+auto SyncDataSyncClient::GetReaderWriter() noexcept -> decltype(m_sync_rw) {
+ return this->m_sync_rw;
+}
+
+::raft::SyncDataResponse* SyncDataSyncClient::GetResponse() noexcept {
+ return &this->m_response;
+}
+
+MemberChangePrepareAsyncClient::MemberChangePrepareAsyncClient(std::shared_ptr<::grpc::Channel> shp_channel,
+ std::shared_ptr<::grpc::CompletionQueue> shp_cq)
+ : UnaryAsyncClient<::raft::MemberChangeInnerRequest, ::raft::MemberChangeInnerResponse,
+ MemberChangePrepareAsyncClient,::grpc::CompletionQueue>(shp_channel, shp_cq){}
+
+MemberChangePrepareAsyncClient::~MemberChangePrepareAsyncClient() {}
+
+void MemberChangePrepareAsyncClient::Responder(const ::grpc::Status& status,
+ const ::raft::MemberChangeInnerResponse& rsp) noexcept {
+ uint32_t _idx = static_cast(reinterpret_cast(m_callback_args[1]));
+ MemberMgr::MemberChangePrepareCallBack(status, rsp, m_callback_args[0], _idx);
+}
+
+MemberChangeCommitAsyncClient::MemberChangeCommitAsyncClient(std::shared_ptr<::grpc::Channel> shp_channel,
+ std::shared_ptr<::grpc::CompletionQueue> shp_cq)
+ : UnaryAsyncClient<::raft::MemberChangeInnerRequest, ::raft::MemberChangeInnerResponse,
+ MemberChangeCommitAsyncClient,::grpc::CompletionQueue>(shp_channel, shp_cq){}
+
+MemberChangeCommitAsyncClient::~MemberChangeCommitAsyncClient() {}
+
+void MemberChangeCommitAsyncClient::Responder(const ::grpc::Status& status,
+ const ::raft::MemberChangeInnerResponse& rsp) noexcept {
+ uint32_t _idx = static_cast(reinterpret_cast(m_callback_args[1]));
+ MemberMgr::MemberChangeCommitCallBack(status, rsp, m_callback_args[0], _idx);
+}
+
+PrevoteAsyncClient::PrevoteAsyncClient(std::shared_ptr<::grpc::Channel> shp_channel,
+ std::shared_ptr<::grpc::CompletionQueue> shp_cq)
+ : UnaryAsyncClient<::raft::VoteRequest, ::raft::VoteResponse,
+ PrevoteAsyncClient,::grpc::CompletionQueue>(shp_channel, shp_cq){}
+
+PrevoteAsyncClient::~PrevoteAsyncClient() {}
+
+void PrevoteAsyncClient::Responder(const ::grpc::Status& status,
+ const ::raft::VoteResponse& rsp) noexcept {
+ uint32_t _idx = static_cast