diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..9c0091d --- /dev/null +++ b/.clang-format @@ -0,0 +1,487 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +--- +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +#AlignEscapedNewlines: Left # Unknown to clang-format-4.0 +AlignOperands: true +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: false +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + #AfterExternBlock: false # Unknown to clang-format-5.0 + BeforeCatch: false + BeforeElse: false + IndentBraces: false + #SplitEmptyFunction: true # Unknown to clang-format-4.0 + #SplitEmptyRecord: true # Unknown to clang-format-4.0 + #SplitEmptyNamespace: true # Unknown to clang-format-4.0 +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +#BreakBeforeInheritanceComma: false # Unknown to clang-format-4.0 +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +#BreakConstructorInitializers: BeforeComma # Unknown to clang-format-4.0 +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +#CompactNamespaces: false # Unknown to clang-format-4.0 +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 8 +ContinuationIndentWidth: 8 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +#FixNamespaceComments: false # Unknown to clang-format-4.0 + +# Taken from: +# git grep -h '^#define [^[:space:]]*for_each[^[:space:]]*(' include/ \ +# | sed "s,^#define \([^[:space:]]*for_each[^[:space:]]*\)(.*$, - '\1'," \ +# | sort | uniq +ForEachMacros: + - 'apei_estatus_for_each_section' + - 'ata_for_each_dev' + - 'ata_for_each_link' + - '__ata_qc_for_each' + - 'ata_qc_for_each' + - 'ata_qc_for_each_raw' + - 'ata_qc_for_each_with_internal' + - 'ax25_for_each' + - 'ax25_uid_for_each' + - '__bio_for_each_bvec' + - 'bio_for_each_bvec' + - 'bio_for_each_integrity_vec' + - '__bio_for_each_segment' + - 'bio_for_each_segment' + - 'bio_for_each_segment_all' + - 'bio_list_for_each' + - 'bip_for_each_vec' + - 'blkg_for_each_descendant_post' + - 'blkg_for_each_descendant_pre' + - 'blk_queue_for_each_rl' + - 'bond_for_each_slave' + - 'bond_for_each_slave_rcu' + - 'bpf_for_each_spilled_reg' + - 'btree_for_each_safe128' + - 'btree_for_each_safe32' + - 'btree_for_each_safe64' + - 'btree_for_each_safel' + - 'card_for_each_dev' + - 'cgroup_taskset_for_each' + - 'cgroup_taskset_for_each_leader' + - 'cpufreq_for_each_entry' + - 'cpufreq_for_each_entry_idx' + - 'cpufreq_for_each_valid_entry' + - 'cpufreq_for_each_valid_entry_idx' + - 'css_for_each_child' + - 'css_for_each_descendant_post' + - 'css_for_each_descendant_pre' + - 'device_for_each_child_node' + - 'drm_atomic_crtc_for_each_plane' + - 'drm_atomic_crtc_state_for_each_plane' + - 'drm_atomic_crtc_state_for_each_plane_state' + - 'drm_atomic_for_each_plane_damage' + - 'drm_connector_for_each_possible_encoder' + - 'drm_for_each_connector_iter' + - 'drm_for_each_crtc' + - 'drm_for_each_encoder' + - 'drm_for_each_encoder_mask' + - 'drm_for_each_fb' + - 'drm_for_each_legacy_plane' + - 'drm_for_each_plane' + - 'drm_for_each_plane_mask' + - 'drm_for_each_privobj' + - 'drm_mm_for_each_hole' + - 'drm_mm_for_each_node' + - 'drm_mm_for_each_node_in_range' + - 'drm_mm_for_each_node_safe' + - 'flow_action_for_each' + - 'for_each_active_drhd_unit' + - 'for_each_active_iommu' + - 'for_each_available_child_of_node' + - 'for_each_bio' + - 'for_each_board_func_rsrc' + - 'for_each_bvec' + - 'for_each_card_components' + - 'for_each_card_links' + - 'for_each_card_links_safe' + - 'for_each_card_prelinks' + - 'for_each_card_rtds' + - 'for_each_card_rtds_safe' + - 'for_each_cgroup_storage_type' + - 'for_each_child_of_node' + - 'for_each_clear_bit' + - 'for_each_clear_bit_from' + - 'for_each_cmsghdr' + - 'for_each_compatible_node' + - 'for_each_component_dais' + - 'for_each_component_dais_safe' + - 'for_each_comp_order' + - 'for_each_console' + - 'for_each_cpu' + - 'for_each_cpu_and' + - 'for_each_cpu_not' + - 'for_each_cpu_wrap' + - 'for_each_dev_addr' + - 'for_each_dma_cap_mask' + - 'for_each_dpcm_be' + - 'for_each_dpcm_be_rollback' + - 'for_each_dpcm_be_safe' + - 'for_each_dpcm_fe' + - 'for_each_drhd_unit' + - 'for_each_dss_dev' + - 'for_each_efi_memory_desc' + - 'for_each_efi_memory_desc_in_map' + - 'for_each_element' + - 'for_each_element_extid' + - 'for_each_element_id' + - 'for_each_endpoint_of_node' + - 'for_each_evictable_lru' + - 'for_each_fib6_node_rt_rcu' + - 'for_each_fib6_walker_rt' + - 'for_each_free_mem_range' + - 'for_each_free_mem_range_reverse' + - 'for_each_func_rsrc' + - 'for_each_hstate' + - 'for_each_if' + - 'for_each_iommu' + - 'for_each_ip_tunnel_rcu' + - 'for_each_irq_nr' + - 'for_each_link_codecs' + - 'for_each_lru' + - 'for_each_matching_node' + - 'for_each_matching_node_and_match' + - 'for_each_memblock' + - 'for_each_memblock_type' + - 'for_each_memcg_cache_index' + - 'for_each_mem_pfn_range' + - 'for_each_mem_range' + - 'for_each_mem_range_rev' + - 'for_each_migratetype_order' + - 'for_each_msi_entry' + - 'for_each_msi_entry_safe' + - 'for_each_net' + - 'for_each_netdev' + - 'for_each_netdev_continue' + - 'for_each_netdev_continue_rcu' + - 'for_each_netdev_feature' + - 'for_each_netdev_in_bond_rcu' + - 'for_each_netdev_rcu' + - 'for_each_netdev_reverse' + - 'for_each_netdev_safe' + - 'for_each_net_rcu' + - 'for_each_new_connector_in_state' + - 'for_each_new_crtc_in_state' + - 'for_each_new_mst_mgr_in_state' + - 'for_each_new_plane_in_state' + - 'for_each_new_private_obj_in_state' + - 'for_each_node' + - 'for_each_node_by_name' + - 'for_each_node_by_type' + - 'for_each_node_mask' + - 'for_each_node_state' + - 'for_each_node_with_cpus' + - 'for_each_node_with_property' + - 'for_each_of_allnodes' + - 'for_each_of_allnodes_from' + - 'for_each_of_cpu_node' + - 'for_each_of_pci_range' + - 'for_each_old_connector_in_state' + - 'for_each_old_crtc_in_state' + - 'for_each_old_mst_mgr_in_state' + - 'for_each_oldnew_connector_in_state' + - 'for_each_oldnew_crtc_in_state' + - 'for_each_oldnew_mst_mgr_in_state' + - 'for_each_oldnew_plane_in_state' + - 'for_each_oldnew_plane_in_state_reverse' + - 'for_each_oldnew_private_obj_in_state' + - 'for_each_old_plane_in_state' + - 'for_each_old_private_obj_in_state' + - 'for_each_online_cpu' + - 'for_each_online_node' + - 'for_each_online_pgdat' + - 'for_each_pci_bridge' + - 'for_each_pci_dev' + - 'for_each_pci_msi_entry' + - 'for_each_populated_zone' + - 'for_each_possible_cpu' + - 'for_each_present_cpu' + - 'for_each_prime_number' + - 'for_each_prime_number_from' + - 'for_each_process' + - 'for_each_process_thread' + - 'for_each_property_of_node' + - 'for_each_registered_fb' + - 'for_each_reserved_mem_region' + - 'for_each_rtd_codec_dai' + - 'for_each_rtd_codec_dai_rollback' + - 'for_each_rtdcom' + - 'for_each_rtdcom_safe' + - 'for_each_set_bit' + - 'for_each_set_bit_from' + - 'for_each_sg' + - 'for_each_sg_dma_page' + - 'for_each_sg_page' + - 'for_each_sibling_event' + - 'for_each_subelement' + - 'for_each_subelement_extid' + - 'for_each_subelement_id' + - '__for_each_thread' + - 'for_each_thread' + - 'for_each_zone' + - 'for_each_zone_zonelist' + - 'for_each_zone_zonelist_nodemask' + - 'fwnode_for_each_available_child_node' + - 'fwnode_for_each_child_node' + - 'fwnode_graph_for_each_endpoint' + - 'gadget_for_each_ep' + - 'genradix_for_each' + - 'genradix_for_each_from' + - 'hash_for_each' + - 'hash_for_each_possible' + - 'hash_for_each_possible_rcu' + - 'hash_for_each_possible_rcu_notrace' + - 'hash_for_each_possible_safe' + - 'hash_for_each_rcu' + - 'hash_for_each_safe' + - 'hctx_for_each_ctx' + - 'hlist_bl_for_each_entry' + - 'hlist_bl_for_each_entry_rcu' + - 'hlist_bl_for_each_entry_safe' + - 'hlist_for_each' + - 'hlist_for_each_entry' + - 'hlist_for_each_entry_continue' + - 'hlist_for_each_entry_continue_rcu' + - 'hlist_for_each_entry_continue_rcu_bh' + - 'hlist_for_each_entry_from' + - 'hlist_for_each_entry_from_rcu' + - 'hlist_for_each_entry_rcu' + - 'hlist_for_each_entry_rcu_bh' + - 'hlist_for_each_entry_rcu_notrace' + - 'hlist_for_each_entry_safe' + - '__hlist_for_each_rcu' + - 'hlist_for_each_safe' + - 'hlist_nulls_for_each_entry' + - 'hlist_nulls_for_each_entry_from' + - 'hlist_nulls_for_each_entry_rcu' + - 'hlist_nulls_for_each_entry_safe' + - 'i3c_bus_for_each_i2cdev' + - 'i3c_bus_for_each_i3cdev' + - 'ide_host_for_each_port' + - 'ide_port_for_each_dev' + - 'ide_port_for_each_present_dev' + - 'idr_for_each_entry' + - 'idr_for_each_entry_continue' + - 'idr_for_each_entry_ul' + - 'inet_bind_bucket_for_each' + - 'inet_lhash2_for_each_icsk_rcu' + - 'key_for_each' + - 'key_for_each_safe' + - 'klp_for_each_func' + - 'klp_for_each_func_safe' + - 'klp_for_each_func_static' + - 'klp_for_each_object' + - 'klp_for_each_object_safe' + - 'klp_for_each_object_static' + - 'kvm_for_each_memslot' + - 'kvm_for_each_vcpu' + - 'list_for_each' + - 'list_for_each_codec' + - 'list_for_each_codec_safe' + - 'list_for_each_entry' + - 'list_for_each_entry_continue' + - 'list_for_each_entry_continue_rcu' + - 'list_for_each_entry_continue_reverse' + - 'list_for_each_entry_from' + - 'list_for_each_entry_from_rcu' + - 'list_for_each_entry_from_reverse' + - 'list_for_each_entry_lockless' + - 'list_for_each_entry_rcu' + - 'list_for_each_entry_reverse' + - 'list_for_each_entry_safe' + - 'list_for_each_entry_safe_continue' + - 'list_for_each_entry_safe_from' + - 'list_for_each_entry_safe_reverse' + - 'list_for_each_prev' + - 'list_for_each_prev_safe' + - 'list_for_each_safe' + - 'llist_for_each' + - 'llist_for_each_entry' + - 'llist_for_each_entry_safe' + - 'llist_for_each_safe' + - 'media_device_for_each_entity' + - 'media_device_for_each_intf' + - 'media_device_for_each_link' + - 'media_device_for_each_pad' + - 'mp_bvec_for_each_page' + - 'mp_bvec_for_each_segment' + - 'nanddev_io_for_each_page' + - 'netdev_for_each_lower_dev' + - 'netdev_for_each_lower_private' + - 'netdev_for_each_lower_private_rcu' + - 'netdev_for_each_mc_addr' + - 'netdev_for_each_uc_addr' + - 'netdev_for_each_upper_dev_rcu' + - 'netdev_hw_addr_list_for_each' + - 'nft_rule_for_each_expr' + - 'nla_for_each_attr' + - 'nla_for_each_nested' + - 'nlmsg_for_each_attr' + - 'nlmsg_for_each_msg' + - 'nr_neigh_for_each' + - 'nr_neigh_for_each_safe' + - 'nr_node_for_each' + - 'nr_node_for_each_safe' + - 'of_for_each_phandle' + - 'of_property_for_each_string' + - 'of_property_for_each_u32' + - 'pci_bus_for_each_resource' + - 'ping_portaddr_for_each_entry' + - 'plist_for_each' + - 'plist_for_each_continue' + - 'plist_for_each_entry' + - 'plist_for_each_entry_continue' + - 'plist_for_each_entry_safe' + - 'plist_for_each_safe' + - 'pnp_for_each_card' + - 'pnp_for_each_dev' + - 'protocol_for_each_card' + - 'protocol_for_each_dev' + - 'queue_for_each_hw_ctx' + - 'radix_tree_for_each_slot' + - 'radix_tree_for_each_tagged' + - 'rbtree_postorder_for_each_entry_safe' + - 'rdma_for_each_port' + - 'resource_list_for_each_entry' + - 'resource_list_for_each_entry_safe' + - 'rhl_for_each_entry_rcu' + - 'rhl_for_each_rcu' + - 'rht_for_each' + - 'rht_for_each_from' + - 'rht_for_each_entry' + - 'rht_for_each_entry_from' + - 'rht_for_each_entry_rcu' + - 'rht_for_each_entry_rcu_from' + - 'rht_for_each_entry_safe' + - 'rht_for_each_rcu' + - 'rht_for_each_rcu_from' + - '__rq_for_each_bio' + - 'rq_for_each_bvec' + - 'rq_for_each_segment' + - 'scsi_for_each_prot_sg' + - 'scsi_for_each_sg' + - 'sctp_for_each_hentry' + - 'sctp_skb_for_each' + - 'shdma_for_each_chan' + - '__shost_for_each_device' + - 'shost_for_each_device' + - 'sk_for_each' + - 'sk_for_each_bound' + - 'sk_for_each_entry_offset_rcu' + - 'sk_for_each_from' + - 'sk_for_each_rcu' + - 'sk_for_each_safe' + - 'sk_nulls_for_each' + - 'sk_nulls_for_each_from' + - 'sk_nulls_for_each_rcu' + - 'snd_array_for_each' + - 'snd_pcm_group_for_each_entry' + - 'snd_soc_dapm_widget_for_each_path' + - 'snd_soc_dapm_widget_for_each_path_safe' + - 'snd_soc_dapm_widget_for_each_sink_path' + - 'snd_soc_dapm_widget_for_each_source_path' + - 'tb_property_for_each' + - 'tcf_exts_for_each_action' + - 'udp_portaddr_for_each_entry' + - 'udp_portaddr_for_each_entry_rcu' + - 'usb_hub_for_each_child' + - 'v4l2_device_for_each_subdev' + - 'v4l2_m2m_for_each_dst_buf' + - 'v4l2_m2m_for_each_dst_buf_safe' + - 'v4l2_m2m_for_each_src_buf' + - 'v4l2_m2m_for_each_src_buf_safe' + - 'virtio_device_for_each_vq' + - 'xa_for_each' + - 'xa_for_each_marked' + - 'xa_for_each_start' + - 'xas_for_each' + - 'xas_for_each_conflict' + - 'xas_for_each_marked' + - 'zorro_for_each_dev' + +#IncludeBlocks: Preserve # Unknown to clang-format-5.0 +IncludeCategories: + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +#IndentPPDirectives: None # Unknown to clang-format-5.0 +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: Inner +#ObjCBinPackProtocolList: Auto # Unknown to clang-format-5.0 +ObjCBlockIndentWidth: 8 +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: true + +# Taken from git's rules +#PenaltyBreakAssignment: 10 # Unknown to clang-format-4.0 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 1 +PenaltyBreakFirstLessLess: 0 +PenaltyBreakString: 1 +PenaltyExcessCharacter: 1 +PenaltyReturnTypeOnItsOwnLine: 1 + +PointerAlignment: Right +ReflowComments: false +SortIncludes: false +#SortUsingDeclarations: false # Unknown to clang-format-4.0 +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +#SpaceBeforeCtorInitializerColon: true # Unknown to clang-format-5.0 +#SpaceBeforeInheritanceColon: true # Unknown to clang-format-5.0 +SpaceBeforeParens: ControlStatements +#SpaceBeforeRangeBasedForLoopColon: true # Unknown to clang-format-5.0 +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp03 +TabWidth: 8 +UseTab: Never +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e87c0d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,94 @@ +# MIT License +# Copyright(c) 2020 Futurewei Cloud +# +# Permission is hereby granted, +# free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +# to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# CMake file +**/cmake-build-debug +**/CMakeCache.txt +**/cmake_install.cmake +**/install_manifest.txt +**/CMakeFiles/ +**/CTestTestfile.cmake +**/Makefile +**/*.cbp +**/CMakeScripts +**/compile_commands.json + +# python +*.swp +__pycache__ + +# Other files to ignore +.vscode/ +cppkafka/ +.DS_Store +AlcorControlAgent +gs_tests +aca_tests +*.pb.h +*.pb.cc +*aca_data.json +*.log \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ea26888 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,17 @@ +# MIT License +# Copyright(c) 2022 Futurewei Cloud +# +# Permission is hereby granted, +# free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +# to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +[submodule "arion-master"] + path = arion-master + url = https://github.com/futurewei-cloud/arion-master.git diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..8b1f980 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/../../../../:\codebase\arion-agent\.idea/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/alcor-control-agent.iml b/.idea/alcor-control-agent.iml new file mode 100644 index 0000000..89b91cc --- /dev/null +++ b/.idea/alcor-control-agent.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/.idea/arion-agent.iml b/.idea/arion-agent.iml new file mode 100644 index 0000000..f08604b --- /dev/null +++ b/.idea/arion-agent.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..f603881 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..79b3c94 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..5aacd03 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..2a7c1b9 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9b826fa --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.10) +project(ArionAgent) + +# CMAKE_BUILD_TYPE can be Debug or Release +set(CMAKE_BUILD_TYPE Debug) +set(CMAKE_CXX_STANDARD 17) + +add_definitions(-w) + +add_subdirectory(src) +#add_subdirectory(test) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e3fe5c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Futurewei Cloud + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 0000000..49cb2f7 --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,36 @@ +# ArionAgent performance +## Overview + +This is the performance test of Arion Agent working with ebpf/XDP as downstream programming module and ArionMaster grpc streaming server as upstream metadata source as an entire system. + + +## Test environment + +This test is between 2 machines of the same lab, they don't belong to the same IP range (means not located on the same rack but same data center): + + 1. Arion master server + + 2. Arion agent (launched on the same machine of Arion Wing which is XDP as gateway network functionality) + + +## Test workflow and scenario #1 - watch from Arion Master only latency + +Latency = ArionAgent finishes receiving N number of neighbors - start to receive Grpc neighbors time (right after watch call to ArionMaster Grpc server finished) + +Watch performance from ArionMaster: + + Watch 5 neighbors performance: 31 us + + Watch 100k neighbors performance: 379,270 us = ~380 ms + + +## Test workflow and scenario #2 - E2E ebpf programming latency + +Latency = Finish ebpf map programming time - start to receive Grpc neighbors time (right after watch call to ArionMaster Grpc server finished) + +E2E programming (watch + programming) performance: + + 100k neighbors performance: 455,059 us = ~455 ms (and if we compare with the watch only 100k neighbors latency of 380ms, we know the overhead of 100k ebpf rule programming is 75ms) + + 1 million neighbors performance: 5,044,295 us = 5,044 ms = 5 seconds + diff --git a/README.md b/README.md index cd29356..8bab261 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ # arion-agent Arion Agent: Local Network Agent on each Arion Wing + + +## Compile + + ./build/build.sh + + +## Sample command to start ArionAgent + + sudo ./build/bin/ArionAgent -a 10.0.0.4 -p 9090 (-a is the ArionMaster grpc server IP, and -p is the server port) + + +## Performance benchmark + +[Benchmark](./PERFORMANCE.md) diff --git a/arion-master b/arion-master new file mode 160000 index 0000000..678f612 --- /dev/null +++ b/arion-master @@ -0,0 +1 @@ +Subproject commit 678f612ea95f3ec197a61837768f6fc7397526b6 diff --git a/build/build.sh b/build/build.sh new file mode 100644 index 0000000..98a277d --- /dev/null +++ b/build/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# MIT License +# Copyright(c) 2022 Futurewei Cloud +# +# Permission is hereby granted, +# free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +# to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +BUILD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "build path is $BUILD" + +# Prepare dependencies +echo "--- prepare dependencies ---" +./build/machine-init.sh + +echo "--- building arion-agent ---" +cmake . && make + +fi diff --git a/build/machine-init.sh b/build/machine-init.sh new file mode 100644 index 0000000..379a778 --- /dev/null +++ b/build/machine-init.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# MIT License +# Copyright(c) 2022 Futurewei Cloud +# +# Permission is hereby granted, +# free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +# to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +BUILD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo "build path is $BUILD" + +rm -rf /var/local/git +mkdir -p /var/local/git + +echo "1--- installing common dependencies ---" && \ + apt-get update -y && apt-get install -y \ + rpcbind \ + rsyslog \ + build-essential \ + make \ + g++ \ + unzip \ + cmake \ + clang-9 \ + llvm-9 \ + libelf-dev \ #for libelf.a + linux-tools-4.15.0-158-generic \ #for bpftool + linux-cloud-tools-4.15.0-158-generic \ #for bpftool + doxygen \ + zlib1g-dev \ + libssl-dev \ + libboost-program-options-dev \ + libboost-all-dev \ + iproute2 \ + net-tools \ + iputils-ping \ + ethtool \ + curl \ + python3 \ + python3-pip \ + netcat \ + libcmocka-dev \ + lcov \ + git \ + autoconf \ + automake \ + dh-autoreconf \ + pkg-config \ + libtool \ + wget \ + uuid-dev +pip3 install httpserver netaddr + +echo "2--- installing grpc dependencies ---" && \ + apt-get update -y && apt-get install -y \ + cmake libssl-dev \ + autoconf git pkg-config \ + automake libtool make g++ unzip + +# installing grpc and its dependencies +GRPC_RELEASE_TAG="v1.24.x" +echo "3--- cloning grpc repo ---" && \ + git clone -b $GRPC_RELEASE_TAG https://github.com/grpc/grpc /var/local/git/grpc && \ + cd /var/local/git/grpc && \ + git submodule update --init && \ + echo "--- installing c-ares ---" && \ + cd /var/local/git/grpc/third_party/cares/cares && \ + git fetch origin && \ + git checkout cares-1_15_0 && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DCMAKE_BUILD_TYPE=Release ../.. && \ + make -j4 install && \ + cd ../../../../.. && \ + rm -rf third_party/cares/cares && \ + echo "--- installing protobuf ---" && \ + cd /var/local/git/grpc/third_party/protobuf && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release .. && \ + make -j4 install && \ + cd ../../../.. && \ + rm -rf third_party/protobuf && \ + echo "--- installing grpc ---" && \ + cd /var/local/git/grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release ../.. && \ + make -j4 install && \ + echo "--- installing google test ---" && \ + cd /var/local/git/grpc/third_party/googletest && \ + cmake -Dgtest_build_samples=ON -DBUILD_SHARED_LIBS=ON . && \ + make && \ + make install && \ + rm -rf /var/local/git/grpc && \ + cd ~ + +echo "4--- installing marl ---" && \ + mkdir -p /var/local/git/marl && \ + cd /var/local/git/marl && \ + git clone https://github.com/google/marl.git && \ + cd /var/local/git/marl/marl && \ + git submodule update --init && \ + mkdir /var/local/git/marl/marl/build && \ + cd /var/local/git/marl/marl/build && \ + cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 && \ + make && \ + cd ~ + +echo "5--- installing ebpf dependencies ---" && \ + cd /var/local/git && \ + git clone https://github.com/futurewei-cloud/zeta && \ + cd zeta && \ + ./build.sh && \ + cd ~ diff --git a/include/grpc_client.h b/include/grpc_client.h new file mode 100644 index 0000000..2d7d1a0 --- /dev/null +++ b/include/grpc_client.h @@ -0,0 +1,62 @@ +// MIT License +// Copyright(c) 2022 Futurewei Cloud +// +// Permission is hereby granted, +// free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +// to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#include + +#include +#include +#include +#include +#include +#include "arionmaster.grpc.pb.h" +#include "bpf.h" +#include "libbpf.h" + +using namespace arion::schema; +using grpc::Status; + +class ArionMasterWatcherImpl final : public Watch::Service { +public: + std::shared_ptr chan_; + + std::unique_ptr stub_; + + explicit ArionMasterWatcherImpl() {} + + void RequestNeighborRules(ArionWingRequest *request, grpc::CompletionQueue *cq); + + void ConnectToArionMaster(); + + void RunClient(std::string ip, std::string port, std::string group, std::string table); + + bool a = chan_ == nullptr; + +private: + std::string server_address; + + std::string server_port; + + std::string group_id; + + std::string table_name_neighbor_ebpf_map; + + int fd_neighbor_ebpf_map = -1; +}; + +struct AsyncClientCall { + arion::schema::NeighborRule reply; + grpc::ClientContext context; + grpc::Status status; + std::unique_ptr > stream; +}; diff --git a/include/logger.h b/include/logger.h new file mode 100644 index 0000000..170d022 --- /dev/null +++ b/include/logger.h @@ -0,0 +1,106 @@ +// MIT License +// Copyright(c) 2022 Futurewei Cloud +// +// Permission is hereby granted, +// free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +// to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#pragma once +#pragma GCC system_header + +#include + +extern bool g_debug_mode; + +#define UNUSED(x) (void)(x) +#define QUOTE(...) #__VA_ARGS__ + +#define LOG_INIT(entity) \ + do { \ + openlog(entity, LOG_CONS | LOG_PID | LOG_NDELAY, LOG_LOCAL1); \ + } while (0) + +#define LOG_CLOSE() \ + do { \ + closelog(); \ + } while (0) + +/* debug-level message */ +#define LOG_DEBUG(f_, ...) \ + do { \ + if (g_debug_mode) { \ + syslog(LOG_DEBUG, "[%s:%d] " f_, __func__, __LINE__, \ + ##__VA_ARGS__); \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* informational message */ +#define LOG_INFO(f_, ...) \ + do { \ + syslog(LOG_INFO, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* normal, but significant, condition */ +#define LOG_NOTICE(f_, ...) \ + do { \ + syslog(LOG_NOTICE, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* warning conditions */ +#define LOG_WARN(f_, ...) \ + do { \ + syslog(LOG_WARNING, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* error conditions */ +#define LOG_ERROR(f_, ...) \ + do { \ + syslog(LOG_ERR, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* critical conditions */ +#define LOG_CRIT(f_, ...) \ + do { \ + syslog(LOG_CRIT, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* action must be taken immediately */ +#define LOG_ALERT(f_, ...) \ + do { \ + syslog(LOG_ALERT, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) + +/* system is unusable */ +#define LOG_EMERG(f_, ...) \ + do { \ + syslog(LOG_EMERG, f_, ##__VA_ARGS__); \ + if (g_debug_mode) { \ + fprintf(stdout, f_, ##__VA_ARGS__); \ + } \ + } while (0) diff --git a/include/util.h b/include/util.h new file mode 100644 index 0000000..f68053f --- /dev/null +++ b/include/util.h @@ -0,0 +1,52 @@ +// MIT License +// Copyright(c) 2022 Futurewei Cloud +// +// Permission is hereby granted, +// free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +// to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef UTIL_H +#define UTIL_H + +#include +#include + +// the number of characters needed to store the HEX form of IP address +#define HEX_IP_BUFFER_SIZE 12 + +// vxlan-generic openflow outport number +#define VXLAN_GENERIC_OUTPORT_NUMBER "100" + +// maximum valid value of a VNI, that (2^24) - 1 +// applicable for VxLAN, GRE, VxLAN-GPE and Geneve +#define MAX_VALID_VNI 16777215 + +#define MAX_VALID_VLAN_ID 4094 + +#define cast_to_nanoseconds(x) chrono::duration_cast(x) +#define cast_to_microseconds(x) chrono::duration_cast(x) +#define us_to_ms(x) x / 1000 // convert from microseconds to millseconds + +static inline long ip4tol(const string ip) { + struct sockaddr_in sa; + if (inet_pton(AF_INET, ip.c_str(), &(sa.sin_addr)) != 1) { + throw std::invalid_argument("Virtual ipv4 address is not in the expected format"); + } + return sa.sin_addr.s_addr; +} + +static inline std::uint8_t getNum(char hexChar) { + if (hexChar >= '0' && hexChar <= '9') { + return hexChar - '0'; + } + return (hexChar - 'A' + 10); +} + +#endif diff --git a/include/xdp/trn_datamodel.h b/include/xdp/trn_datamodel.h new file mode 100644 index 0000000..7ab0321 --- /dev/null +++ b/include/xdp/trn_datamodel.h @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/** + * @file datamodel.h + * @author Sherif Abdelwahab (@zasherif) + * Wei Yue (@w-yue) + * + * @brief Data models between user and kernel space. data propagated + * from control-plane through transitd. + * + * @copyright Copyright (c) 2019-2022 The Authors. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#pragma once + +#include +#include +#include +#include +#define __ALIGNED_64__ __attribute__((aligned(64))) +#define __ALWAYS_INLINE__ __attribute__((__always_inline__)) + +/* maximal string size, including null terminator */ +#define TRAN_MAX_PATH_SIZE 256 +#define TRAN_MAX_ITF_SIZE 20 +#define TRAN_MAX_CLI_JSON_STR 10240 + +#define TRAN_MAX_ZGC_ENTRANCES 128 +#define TRAN_MAX_EP_BATCH_SIZE 360 +#define TRAN_DP_FLOW_TIMEOUT 30 // In seconds + +/* At most 10 chains, size has to be prime and 100x number of chains */ +#define TRAN_MAX_MAGLEV_TABLE_SIZE 10000 +#define TRAN_MAX_FTN 512 +#define TRAN_MAX_CHAIN 128 + +/* Set max total number of endpoints */ +#define TRAN_MAX_NEP 128*1024 +/* Set max number of host IPs a (scaled) endpoint can be mapped to */ +#define TRAN_MAX_REMOTES 64 +#define TRAN_MAX_ITF 128 +#define TRAN_MAX_VETH 2048 +#define TRAN_UNUSED_ITF_IDX -1 + +#define TRAN_SUBSTRT_VNI 0 + +#define TRAN_SUBSTRT_EP 0 +#define TRAN_SIMPLE_EP 1 +#define TRAN_SCALED_EP 2 +#define TRAN_GATEWAY_EP 3 + +/* Size for OAM message queue bpfmap */ +#define TRAN_OAM_QUEUE_LEN 1024 + +/* XDP interface_map keys for packet redirect */ +enum trn_itf_ma_key_t { +TRAN_ITF_MAP_TENANT = 0, // id map to ifindex connected to tenant network +TRAN_ITF_MAP_ZGC, // id map to ifindex connected to zgc network +TRAN_ITF_MAP_MAX +}; + +/* Cache related const */ +#define TRAN_MAX_CACHE_SIZE 1000000 + +/* XDP programs keys in transit XDP tailcall jump table */ +enum trn_xdp_prog_id_t { + TRAN_TRANSIT_PROG = 0, + TRAN_TX_PROG, + TRAN_PASS_PROG, + TRAN_REDIRECT_PROG, + TRAN_DROP_PROG, + TRAN_MAX_PROG +}; + +/* XDP programs roles pass along tail-called bpf programs */ +enum trn_xdp_role_t { + XDP_FWD = 0, + XDP_FTN, + XDP_ROLE_MAX +}; + +/* FTN node position/type in DFT chain */ +enum trn_ftn_type_t { + TRAN_FTN_TYPE_HEAD = 0, + TRAN_FTN_TYPE_MIDDLE, + TRAN_FTN_TYPE_TAIL +}; + +/* Tunnel Interface protocol */ +enum trn_xdp_tunnel_protocol_t { + XDP_TUNNEL_VXLAN = 0, + XDP_TUNNEL_GENEVE +}; + +/* Flow Verdict */ +enum trn_xdp_flow_op_t { + XDP_FLOW_OP_ENCAP = 0, + XDP_FLOW_OP_DELETE, + XDP_FLOW_OP_DROP, + XDP_FLOW_OP_MAX +}; + +/* Tie interface definitions together */ +typedef struct { + int itf_map_key; // from trn_itf_ma_key_t + int itf_xdp_role; // from trn_xdp_role_t + int itf_protocol; // from trn_xdp_tunnel_protocol_t +} trn_xdp_itf_def_t; + +struct dft_t { + __u32 table_len; + __u32 table[TRAN_MAX_MAGLEV_TABLE_SIZE]; +} __attribute__((packed, aligned(4))); + +struct chain_t { + __u32 tail_ftn; +} __attribute__((packed, aligned(4))); + +struct ftn_t { + __u8 position; + __u32 ip; + __u32 next_ip; + unsigned char mac[6]; + unsigned char next_mac[6]; +} __attribute__((packed, aligned(4))); + +typedef struct { + __u32 vni; + __u32 ip; +} __attribute__((packed, aligned(4))) endpoint_key_t; + +typedef struct { + __u32 hip; + unsigned char mac[6]; + unsigned char hmac[6]; +} __attribute__((packed, aligned(4))) endpoint_t; + +typedef struct { + __u32 ip; // IP used for ZGC access + __u16 announced; // non-zero indicates the MAC has been announced locally + __u8 mac[6]; // MAC to be used for ZGC entrance +} __attribute__((packed, aligned(4))) zgc_entrance_t; + +/* Should call it overlay interface */ +struct tunnel_iface_t { + __u32 iface_index; + __u16 ibo_port; + __u8 protocol; // value from trn_xdp_tunnel_protocol_t + __u8 role; // value from trn_xdp_role_t + __u32 num_entrances; // number of valid entries in entrances array + zgc_entrance_t entrances[TRAN_MAX_ZGC_ENTRANCES]; +} __attribute__((packed, aligned(4))); + +typedef struct { + __u32 saddr; + __u32 daddr; + __u16 sport; + __u16 dport; + __u8 protocol; + __u8 vni[3]; +} __attribute__((packed, aligned(4))) ipv4_flow_t; + +struct remote_endpoint_t { + __u32 ip; + unsigned char mac[6]; +} __attribute__((packed)); + +struct scaled_endpoint_remote_t { + /* Addresses */ + __u32 saddr; + __u32 daddr; + + /* ports */ + __u16 sport; + __u16 dport; + + unsigned char h_source[6]; + unsigned char h_dest[6]; +} __attribute__((packed)); + +/* Direct Path oam op data */ +typedef struct { + /* Destination Encap */ + __u32 dip; + __u32 dhip; + __u8 dmac[6]; + __u8 dhmac[6]; + __u16 timeout; /* in seconds */ + __u16 rsvd; +} dp_encap_opdata_t; + +typedef struct { + __u16 len; + struct ethhdr eth; + struct iphdr ip; + struct udphdr udp; + __u32 opcode; // trn_xdp_flow_op_t + + /* OAM OpData */ + ipv4_flow_t flow; // flow matcher + + union { + dp_encap_opdata_t encap; + } opdata; +} __attribute__((packed, aligned(8))) flow_ctx_t; diff --git a/include/xdp/trn_kern.h b/include/xdp/trn_kern.h new file mode 100644 index 0000000..95f6c18 --- /dev/null +++ b/include/xdp/trn_kern.h @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/** + * @file transit_kern.h + * @author Sherif Abdelwahab (@zasherif) + * + * @brief Helper functions, macros and data structures. + * + * @copyright Copyright (c) 2019 The Authors. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "extern/bpf_helpers.h" +#include "extern/jhash.h" + +#include "trn_datamodel.h" + +#ifdef dead_code +#define PRIu8 "hu" +#define PRId8 "hd" +#define PRIx8 "hx" +#define PRIu16 "hu" +#define PRId16 "hd" +#define PRIx16 "hx" +#define PRIu32 "u" +#define PRId32 "d" +#define PRIx32 "x" +#define PRIu64 "llu" // or possibly "lu" +#define PRId64 "lld" // or possibly "ld" +#define PRIx64 "llx" // or possibly "lx" +#endif + +#define TRN_DEFAULT_TTL 64 +#define GEN_DSTPORT 0xc117 // UDP dport 6081(0x17c1) for Geneve overlay +#define VXL_DSTPORT 0xb512 // UDP dport 4789(0x12b5) for VxLAN overlay +#define INIT_JHASH_SEED 0xdeadbeef + +#define TRN_GNV_OPT_CLASS 0x0111 +#define TRN_GNV_RTS_OPT_TYPE 0x48 +#define TRN_GNV_SCALED_EP_OPT_TYPE 0x49 + +/* Scaled endpoint messages type */ +#define TRN_SCALED_EP_MODIFY 0x4d // (M: Modify) + +#ifndef __inline +#define __inline inline __attribute__((always_inline)) +#endif + +struct trn_gnv_scaled_ep_data { + __u8 msg_type; + struct scaled_endpoint_remote_t target; +} __attribute__((packed, aligned(4))); + +struct trn_gnv_scaled_ep_opt { + __be16 opt_class; + __u8 type; + __u8 length : 5; + __u8 r3 : 1; + __u8 r2 : 1; + __u8 r1 : 1; + /* opt data */ + struct trn_gnv_scaled_ep_data scaled_ep_data; +} __attribute__((packed, aligned(4))); + +struct trn_gnv_rts_data { + __u8 match_flow : 1; + struct remote_endpoint_t host; +} __attribute__((packed, aligned(4))); + +struct trn_gnv_rts_opt { + __be16 opt_class; + __u8 type; + __u8 length : 5; + __u8 r3 : 1; + __u8 r2 : 1; + __u8 r1 : 1; + /* opt data */ + struct trn_gnv_rts_data rts_data; +} __attribute__((packed, aligned(4))); + +struct geneve_opt { + __be16 opt_class; + __u8 type; + __u8 length : 5; + __u8 r3 : 1; + __u8 r2 : 1; + __u8 r1 : 1; + __u8 opt_data[]; +}; + +struct genevehdr { + /* Big endian! */ + __u8 opt_len : 6; + __u8 ver : 2; + __u8 rsvd1 : 6; + __u8 critical : 1; + __u8 oam : 1; + __be16 proto_type; + __u8 vni[3]; + __u8 rsvd2; + struct geneve_opt options[]; +}; + +struct vxlanhdr { + /* Big endian! */ + __u8 rsvd1 : 3; + __u8 i_flag : 1; + __u8 rsvd2 : 4; + __u8 rsvd3[3]; + __u8 vni[3]; + __u8 rsvd4; +}; + +struct transit_packet { + void *data; + void *data_end; + + /* interface index */ + struct tunnel_iface_t *itf; + + __u32 itf_idx; + __u32 itf_ipv4; + + __u16 ent_idx; // entrance index in tunnel_iface_t + __u8 itf_mac[6]; + + /* xdp*/ + struct xdp_md *xdp; + + /* Ether */ + struct ethhdr *eth; + + /* IP */ + struct iphdr *ip; + + /* UDP */ + struct udphdr *udp; + + union { + struct { + /* Geneve */ + struct genevehdr *hdr; + struct trn_gnv_rts_opt *rts_opt; + struct trn_gnv_scaled_ep_opt *scaled_ep_opt; + __u32 gnv_hdr_len; + __u32 gnv_opt_len; + } geneve; + struct vxlanhdr *vxlan; + } overlay; + + /* overlay network ID */ + __u32 vni; + __u32 pad1; + + /* Inner ethernet */ + struct ethhdr *inner_eth; + + /* Inner arp */ + struct arphdr *inner_arp; + + /* Inner IP */ + struct iphdr *inner_ip; + + /* Inner udp */ + struct udphdr *inner_udp; + + /* Inner tcp */ + struct tcphdr *inner_tcp; + + flow_ctx_t fctx; // keep this at last + + // TODO: Inner UDP or TCP +} __attribute__((packed, aligned(8))); + +__ALWAYS_INLINE__ +static inline __u32 trn_get_inner_packet_hash(struct transit_packet *pkt) +{ + // TODO: Just the source IP for now, change to the 4-tuples + return jhash_2words(pkt->inner_ip->saddr, 0, INIT_JHASH_SEED); +} + +__ALWAYS_INLINE__ +static __be32 trn_get_vni(const __u8 *vni) +{ + /* Big endian! */ + return (vni[0] << 16) | (vni[1] << 8) | vni[2]; +} + +static void trn_set_vni(__be32 src, __u8 *vni) +{ + /* Big endian! */ + vni[0] = (__u8)(src >> 16); + vni[1] = (__u8)(src >> 8); + vni[2] = (__u8)src; +} + +__ALWAYS_INLINE__ +static inline __u16 trn_csum_fold_helper(__u64 csum) +{ + int i; +#pragma unroll + for (i = 0; i < 4; i++) { + if (csum >> 16) + csum = (csum & 0xffff) + (csum >> 16); + } + return ~csum; +} + +__ALWAYS_INLINE__ +static inline void trn_ipv4_csum_inline(void *iph, __u64 *csum) +{ + __u16 *next_iph_u16 = (__u16 *)iph; +#pragma clang loop unroll(full) + for (int i = 0; i> 1; i++) { + *csum += *next_iph_u16++; + } + *csum = trn_csum_fold_helper(*csum); +} + +__ALWAYS_INLINE__ +static inline void trn_update_l4_csum(__u64 *csum, __be32 old_addr, + __be32 new_addr) +{ + *csum = (~*csum & 0xffff) + ~old_addr + new_addr; + *csum = trn_csum_fold_helper(*csum); +} + +__ALWAYS_INLINE__ +static inline void trn_update_l4_csum_port(__u64 *csum, __be16 old_port, + __be16 new_port) +{ + *csum = (~*csum & 0xffff) + ~old_port + new_port; + *csum = trn_csum_fold_helper(*csum); +} + +__ALWAYS_INLINE__ +static inline int trn_is_mac_equal(unsigned char *s, unsigned char *d) +{ + unsigned short *ss = (unsigned short *)s; + unsigned short *ds = (unsigned short *)d; + + return ((ds[0] == ss[0] && ds[1] == ss[1] && ds[2] == ss[2])? 1 : 0); +} + +__ALWAYS_INLINE__ +static inline void trn_set_mac(void *dst, unsigned char *mac) +{ + unsigned short *d = dst; + unsigned short *s = (unsigned short *)mac; + + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; +} + +__ALWAYS_INLINE__ +static inline void trn_set_dst_mac(void *data, unsigned char *dst_mac) +{ + trn_set_mac(data, dst_mac); +} + +__ALWAYS_INLINE__ +static inline void trn_set_src_mac(void *data, unsigned char *src_mac) +{ + trn_set_mac(data + 6, src_mac); +} + +__ALWAYS_INLINE__ +static inline void trn_swap_src_dst_mac(void *data) +{ + unsigned short *p = data; + unsigned short tmp[3]; + + tmp[0] = p[0]; + tmp[1] = p[1]; + tmp[2] = p[2]; + p[0] = p[3]; + p[1] = p[4]; + p[2] = p[5]; + p[3] = tmp[0]; + p[4] = tmp[1]; + p[5] = tmp[2]; +} + +__ALWAYS_INLINE__ +static inline void trn_set_src_ip(void *data, void *data_end, __u32 saddr) +{ + int off = offsetof(struct iphdr, saddr); + __u32 *addr = data + off; + if ((void *)addr > data_end) + return; + + *addr = saddr; +} + +__ALWAYS_INLINE__ +static inline void trn_set_dst_ip(void *data, void *data_end, __u32 daddr) +{ + int off = offsetof(struct iphdr, daddr); + __u32 *addr = data + off; + if ((void *)addr > data_end) + return; + + *addr = daddr; +} + +__ALWAYS_INLINE__ +static inline void trn_swap_src_dst_ip(struct iphdr *ip, void *data_end) +{ + __u32 tmp = ip->saddr; + trn_set_src_ip(ip, data_end, ip->daddr); + trn_set_dst_ip(ip, data_end, tmp); +} + +__ALWAYS_INLINE__ +static inline void trn_set_sport_udp(void *data, void *data_end, __u16 sport) +{ + int off = offsetof(struct udphdr, source); + __u16 *addr = data + off; + if ((void *)addr + sizeof(__be16) > data_end) + return; + + *addr = sport; +} + +__ALWAYS_INLINE__ +static inline void trn_set_dport_udp(void *data, void *data_end, __u16 dport) +{ + int off = offsetof(struct udphdr, dest); + __u16 *addr = data + off; + if ((void *)addr + sizeof(__be16) > data_end) + return; + + *addr = dport; +} + +__ALWAYS_INLINE__ +static inline void trn_swap_sport_dport_udp(struct udphdr *udp, void *data_end) +{ + __u16 tmp = udp->source; + trn_set_sport_udp(udp, data_end, udp->dest); + trn_set_dport_udp(udp, data_end, tmp); +} + +__ALWAYS_INLINE__ +static inline void trn_set_src_dst_ip_csum(struct iphdr *ip, + __u32 saddr, __u32 daddr, void *data_end) +{ + /* Since the packet destination is being rewritten we also + decrement the TTL */ + ip->ttl--; + + __u64 csum = 0; + trn_set_src_ip(ip, data_end, saddr); + trn_set_dst_ip(ip, data_end, daddr); + csum = 0; + ip->check = 0; + trn_ipv4_csum_inline(ip, &csum); + ip->check = csum; + + bpf_debug("Modified IP Address, src: 0x%x, dst: 0x%x, csum: 0x%x\n", + ip->saddr, ip->daddr, ip->check); +} + +__ALWAYS_INLINE__ +static inline void trn_inner_l4_csum_update(struct transit_packet *pkt, + __u32 old_addr, __u32 new_addr) +{ + if (new_addr == old_addr) + return; + + if (pkt->inner_ip->protocol == IPPROTO_UDP) { + if (!pkt->inner_udp) { + return; + } + + if (pkt->inner_udp + 1 > pkt->data_end) { + return; + } + + __u64 cs = pkt->inner_udp->check; + trn_update_l4_csum(&cs, old_addr, new_addr); + pkt->inner_udp->check = cs; + } + + if (pkt->inner_ip->protocol == IPPROTO_TCP) { + if (!pkt->inner_tcp) { + return; + } + + if (pkt->inner_tcp + 1 > pkt->data_end) { + return; + } + + __u64 cs = pkt->inner_tcp->check; + trn_update_l4_csum(&cs, old_addr, new_addr); + pkt->inner_tcp->check = cs; + } +} + +__ALWAYS_INLINE__ +static inline void trn_set_src_dst_inner_ip_csum(struct transit_packet *pkt, + __u32 saddr, __u32 daddr) +{ + if (pkt->inner_ip + 1 > pkt->data_end) { + return; + } + + __u32 old_saddr = pkt->inner_ip->saddr; + __u32 old_daddr = pkt->inner_ip->daddr; + + __u64 csum = 0; + trn_set_src_ip(pkt->inner_ip, pkt->data_end, saddr); + trn_inner_l4_csum_update(pkt, old_saddr, saddr); + + trn_set_dst_ip(pkt->inner_ip, pkt->data_end, daddr); + trn_inner_l4_csum_update(pkt, old_daddr, daddr); + + csum = 0; + pkt->inner_ip->check = 0; + trn_ipv4_csum_inline(pkt->inner_ip, &csum); + pkt->inner_ip->check = csum; + + bpf_debug( + "Modified Inner IP Address, src: 0x%x, dst: 0x%x, csum: 0x%x\n", + pkt->inner_ip->saddr, pkt->inner_ip->daddr, + pkt->inner_ip->check); +} + +__ALWAYS_INLINE__ +static inline void trn_set_src_dst_port(struct transit_packet *pkt, __u16 sport, + __u16 dport) +{ + if (pkt->fctx.flow.protocol == IPPROTO_TCP) { + if (pkt->inner_tcp + 1 > pkt->data_end) + return; + __u16 old_dport = pkt->inner_tcp->dest; + __u16 old_sport = pkt->inner_tcp->source; + pkt->inner_tcp->source = sport; + pkt->inner_tcp->dest = dport; + // Compute csum + if (old_dport != dport) { + __u64 cs = pkt->inner_tcp->check; + trn_update_l4_csum_port(&cs, old_dport, dport); + pkt->inner_tcp->check = cs - bpf_htons(256); + } + if (old_sport != sport) { + __u64 cs = pkt->inner_tcp->check; + trn_update_l4_csum_port(&cs, old_sport, sport); + pkt->inner_tcp->check = cs - bpf_htons(256); + } + bpf_debug("Modified Inner TCP Ports src: %u, dest: %u, csum: 0x%x\n", + bpf_ntohs(pkt->inner_tcp->source), + bpf_ntohs(pkt->inner_tcp->dest), pkt->inner_tcp->check); + } else if (pkt->fctx.flow.protocol == IPPROTO_UDP) { + if (pkt->inner_udp + 1 > pkt->data_end) + return; + __u16 old_dport = pkt->inner_udp->dest; + __u16 old_sport = pkt->inner_udp->source; + pkt->inner_udp->source = sport; + pkt->inner_udp->dest = dport; + + // Compute csum + if (old_dport != dport) { + __u64 cs = pkt->inner_udp->check; + trn_update_l4_csum_port(&cs, old_dport, dport); + pkt->inner_udp->check = cs - bpf_htons(256); + } + if (old_sport != sport) { + __u64 cs = pkt->inner_udp->check; + trn_update_l4_csum_port(&cs, old_sport, sport); + pkt->inner_udp->check = cs - bpf_htons(256); + } + bpf_debug("Modified Inner UDP Ports src: %u, dest: %u, csum: 0x%x\n", + bpf_ntohs(pkt->inner_udp->source), + bpf_ntohs(pkt->inner_udp->dest), pkt->inner_udp->check); + } else { + return; + } +} + +__ALWAYS_INLINE__ +static inline void trn_reverse_ipv4_tuple(ipv4_flow_t *tuple) +{ + __u32 tmp_addr = tuple->saddr; + __u16 tmp_port = tuple->sport; + + tuple->saddr = tuple->daddr; + tuple->sport = tuple->dport; + + tuple->daddr = tmp_addr; + tuple->dport = tmp_port; +} + +__ALWAYS_INLINE__ +static inline void trn_reset_rts_opt(struct transit_packet *pkt) + +{ + pkt->overlay.geneve.rts_opt->type = 0; + pkt->overlay.geneve.rts_opt->length = 0; + __builtin_memset(&pkt->overlay.geneve.rts_opt->rts_data, 0, + sizeof(struct trn_gnv_rts_data)); +} + +__ALWAYS_INLINE__ +static inline void trn_reset_scaled_ep_opt(struct transit_packet *pkt) + +{ + pkt->overlay.geneve.scaled_ep_opt->type = 0; + pkt->overlay.geneve.scaled_ep_opt->length = 0; + __builtin_memset(&pkt->overlay.geneve.scaled_ep_opt->scaled_ep_data, 0, + sizeof(struct trn_gnv_scaled_ep_data)); +} diff --git a/include/xdp/trn_transit_xdp_maps.h b/include/xdp/trn_transit_xdp_maps.h new file mode 100644 index 0000000..4d25c80 --- /dev/null +++ b/include/xdp/trn_transit_xdp_maps.h @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/** + * @file transit_maps.h + * @author Sherif Abdelwahab (@zasherif) + * + * @brief Defines ebpf maps of transit XDP + * + * @copyright Copyright (c) 2019 The Authors. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#pragma once + +#include + +#include "extern/bpf_helpers.h" +#include "extern/xdpcap_hook.h" + +#include "trn_datamodel.h" + +struct bpf_map_def SEC("maps") jmp_table = { + .type = BPF_MAP_TYPE_PROG_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u32), + .max_entries = TRAN_MAX_PROG, +}; +BPF_ANNOTATE_KV_PAIR(jmp_table, __u32, __u32); + +struct bpf_map_def SEC("maps") endpoints_map = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(endpoint_key_t), + .value_size = sizeof(endpoint_t), + .max_entries = TRAN_MAX_NEP, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(endpoints_map, endpoint_key_t, endpoint_t); + +#if turnOn +struct bpf_map_def SEC("maps") hosted_eps_if = { + .type = BPF_MAP_TYPE_HASH, + .key_size = sizeof(endpoint_key_t), + .value_size = sizeof(int), + .max_entries = TRAN_MAX_NEP, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(hosted_eps_if, endpoint_key_t, int); +#endif + +struct bpf_map_def SEC("maps") if_config_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct tunnel_iface_t), + .max_entries = TRAN_MAX_ITF, + .map_flags = 0, +}; +BPF_ANNOTATE_KV_PAIR(if_config_map, __u32, struct tunnel_iface_t); + +/* Host specific interface map used for packet redirect */ +struct bpf_map_def SEC("maps") interfaces_map = { + .type = BPF_MAP_TYPE_DEVMAP, + .key_size = sizeof(int), + .value_size = sizeof(int), + .max_entries = TRAN_ITF_MAP_MAX, +}; +BPF_ANNOTATE_KV_PAIR(interface_map, int, int); + +#if turnOn +struct bpf_map_def SEC("maps") oam_queue_map = { + .type = BPF_MAP_TYPE_QUEUE, + .key_size = 0, + .value_size = sizeof(flow_ctx_t), + .max_entries = TRAN_OAM_QUEUE_LEN, +}; +BPF_ANNOTATE_KV_PAIR_QUEUESTACK(oam_queue_map, flow_ctx_t); +#endif + +#if turnOn +struct bpf_map_def SEC("maps") fwd_flow_cache = { + .type = BPF_MAP_TYPE_LRU_HASH, + .key_size = sizeof(ipv4_flow_t), + .value_size = sizeof(struct scaled_endpoint_remote_t), + .max_entries = TRAN_MAX_CACHE_SIZE, +}; +BPF_ANNOTATE_KV_PAIR(fwd_flow_cache, ipv4_flow_t, + struct scaled_endpoint_remote_t); + +struct bpf_map_def SEC("maps") rev_flow_cache = { + .type = BPF_MAP_TYPE_LRU_HASH, + .key_size = sizeof(ipv4_flow_t), + .value_size = sizeof(struct scaled_endpoint_remote_t), + .max_entries = TRAN_MAX_CACHE_SIZE, +}; +BPF_ANNOTATE_KV_PAIR(rev_flow_cache, ipv4_flow_t, + struct scaled_endpoint_remote_t); + +struct bpf_map_def SEC("maps") host_flow_cache = { + .type = BPF_MAP_TYPE_LRU_HASH, + .key_size = sizeof(ipv4_flow_t), + .value_size = sizeof(struct remote_endpoint_t), + .max_entries = TRAN_MAX_CACHE_SIZE, +}; +BPF_ANNOTATE_KV_PAIR(host_flow_cache, ipv4_flow_t, + struct remote_endpoint_t); + +struct bpf_map_def SEC("maps") ep_host_cache = { + .type = BPF_MAP_TYPE_LRU_HASH, + .key_size = sizeof(endpoint_key_t), + .value_size = sizeof(struct remote_endpoint_t), + .max_entries = TRAN_MAX_CACHE_SIZE, +}; +BPF_ANNOTATE_KV_PAIR(ep_host_cache, endpoint_key_t, + struct remote_endpoint_t); +#endif + +struct bpf_map_def SEC("maps") xdpcap_hook = XDPCAP_HOOK(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..448f7e7 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,49 @@ +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../build/bin) + +set(SOURCES + ./comm/grpc_client.cpp + ) + +#FIND_LIBRARY(LIBUUID_LIBRARIES uuid) +#link_libraries(/usr/lib/x86_64-linux-gnu/libuuid.so) +link_libraries(/usr/lib/x86_64-linux-gnu/libevent_pthreads.so) +link_libraries(/usr/lib/x86_64-linux-gnu/libpthread.so) +link_libraries(/var/local/git/marl/marl/build/libmarl.a) #this was built by machine-init.sh +link_libraries(/var/local/git/zeta/src/extern/libbpf/src/libbpf.a) #this was built by machine-init.sh +link_libraries(/usr/lib/x86_64-linux-gnu/libelf.a) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/proto3) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/grpc) +include_directories(/var/local/git/marl/marl/include) +include_directories(/var/local/git/zeta/src/extern/libbpf/src) #libbpf.h + +# Find Protobuf installation +# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation. +set(protobuf_MODULE_COMPATIBLE TRUE) +find_package(Protobuf CONFIG REQUIRED) +message(STATUS "Using protobuf ${protobuf_VERSION}") + +# Find gRPC installation +# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. +find_package(gRPC CONFIG REQUIRED) +message(STATUS "Using gRPC ${gRPC_VERSION}") + +set(_GRPC_GRPCPP_UNSECURE gRPC::grpc++_unsecure) +set(_GRPC_CPP_PLUGIN_EXECUTABLE $) + +add_library(ArionAgentLib STATIC ${SOURCES}) +#target_link_libraries(ArionAgentLib event) +target_link_libraries(ArionAgentLib ssl) +target_link_libraries(ArionAgentLib crypto) +target_link_libraries(ArionAgentLib rt) + +add_executable(ArionAgent main.cpp) +target_link_libraries(ArionAgent ArionAgentLib) +target_link_libraries(ArionAgent proto) +target_link_libraries(ArionAgent grpc) +target_link_libraries(ArionAgent ${PROTOBUF_LIBRARY}) +target_link_libraries(ArionAgent ${_GRPC_GRPCPP_UNSECURE}) + +add_dependencies(ArionAgentLib proto grpc) +add_subdirectory(proto3) +add_subdirectory(grpc) diff --git a/src/comm/grpc_client.cpp b/src/comm/grpc_client.cpp new file mode 100644 index 0000000..be808d7 --- /dev/null +++ b/src/comm/grpc_client.cpp @@ -0,0 +1,164 @@ +/* + * + * Copyright 2015 gRPC authors. + * Copyright 2022 The Arion Authors - file modified. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "marl/defer.h" +#include "marl/event.h" +#include "marl/scheduler.h" +#include "marl/waitgroup.h" + +#include "arionmaster.grpc.pb.h" +#include "grpc_client.h" +#include "xdp/trn_datamodel.h" + +using namespace arion::schema; + +void ArionMasterWatcherImpl::RequestNeighborRules(ArionWingRequest *request, + grpc::CompletionQueue *cq) { + grpc::ClientContext ctx; + arion::schema::NeighborRule reply; + + // check current grpc channel state, try to connect if needed + grpc_connectivity_state current_state = chan_->GetState(true); + if (current_state == grpc_connectivity_state::GRPC_CHANNEL_SHUTDOWN || + current_state == grpc_connectivity_state::GRPC_CHANNEL_TRANSIENT_FAILURE) { + printf("%s, it is: [%d]\n", + "Channel state is not READY/CONNECTING/IDLE. Try to reconnnect.", + current_state); + this->ConnectToArionMaster(); + } + + void* got_tag; + bool ok = false; + AsyncClientCall *call = new AsyncClientCall; + + int tag_watch = 1; + printf("Completion queue: initial task, async watch\n"); + call->stream = stub_->AsyncWatch(&call->context, cq, (void*)tag_watch); + + // start time + //std::chrono::_V2::steady_clock::time_point start; + + //std::atomic i(tag_watch + 1); + bool write_done = false; + while (cq->Next(&got_tag, &ok)) { + if (ok) { + if (!write_done) { + printf("Completion queue: initial task response received\n"); + + printf("Completion queue: write async watch ArionWingRequest of [group, revision] to stream\n"); + call->stream->Write(*request, (void*)tag_watch); + + write_done = true; + } else { + call->stream->Read(&call->reply, got_tag); + auto vni = call->reply.tunnel_id(); + auto vpc_ip = call->reply.ip(); + auto vpc_mac = call->reply.mac(); + auto host_ip = call->reply.hostip(); + auto host_mac = call->reply.hostmac(); + //auto arionwing_group = call->reply.arionwing_group(); + //auto rev = call->reply.version(); + int fd = fd_neighbor_ebpf_map; + + if ("" != vpc_ip) { //non-empty rule + marl::schedule([=] { + endpoint_key_t epkey; + epkey.vni = vni; + struct sockaddr_in ep_ip; + inet_pton(AF_INET, vpc_ip.c_str(), &(ep_ip.sin_addr)); + epkey.ip = ep_ip.sin_addr.s_addr; + + endpoint_t ep; + struct sockaddr_in ep_hip; + inet_pton(AF_INET, host_ip.c_str(), &(ep_hip.sin_addr)); + ep.hip = ep_hip.sin_addr.s_addr; + + // handle vpc mac address + std::sscanf(vpc_mac.c_str(), + "%02x:%02x:%02x:%02x:%02x:%02x", + &ep.mac[0], &ep.mac[1], &ep.mac[2], + &ep.mac[3], &ep.mac[4], &ep.mac[5]); + + // handle host mac address + std::sscanf(host_mac.c_str(), + "%02x:%02x:%02x:%02x:%02x:%02x", + &ep.hmac[0], &ep.hmac[1], &ep.hmac[2], + &ep.hmac[3], &ep.hmac[4], &ep.hmac[5]); + + int rc = bpf_map_update_elem(fd, &epkey, &ep, BPF_ANY); + + //i++; + }); + } + } + } + } +} + +void ArionMasterWatcherImpl::ConnectToArionMaster() { + grpc::ChannelArguments args; + // Channel does a keep alive ping every 10 seconds; + args.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, 10000); + // If the channel does receive the keep alive ping result in 20 seconds, it closes the connection + args.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 20000); + // Allow keep alive ping even if there are no calls in flight + args.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1); + + chan_ = grpc::CreateCustomChannel(server_address + ":" + server_port, + grpc::InsecureChannelCredentials(), args); + stub_ = Watch::NewStub(chan_); + + printf("After initiating a new sub to connect to the Arion Master: %s\n", (server_address + ":" + server_port).c_str()); +} + +void ArionMasterWatcherImpl::RunClient(std::string ip, std::string port, std::string group, std::string table) { + printf("Running a grpc client in a separate thread id: %ld\n", std::this_thread::get_id()); + + server_address = ip; + server_port = port; + group_id = group; + table_name_neighbor_ebpf_map = table; + + fd_neighbor_ebpf_map = bpf_obj_get(table_name_neighbor_ebpf_map.c_str()); + if (fd_neighbor_ebpf_map < 0) { + printf("Failed to get xdp neighbor endpoint map fd\n"); + } else { + printf("Got xdp neighbor endpoint map fd %d\n", fd_neighbor_ebpf_map); + } + + this->ConnectToArionMaster(); + // TODO: read from db and starting watcher from last known good revision + grpc::CompletionQueue cq; + ArionWingRequest watch_req; + watch_req.set_group(group_id); + watch_req.set_rev(1); + this->RequestNeighborRules(&watch_req, &cq); +} diff --git a/src/grpc/CMakeLists.txt b/src/grpc/CMakeLists.txt new file mode 100644 index 0000000..285a2cc --- /dev/null +++ b/src/grpc/CMakeLists.txt @@ -0,0 +1,40 @@ + +# Find Protobuf installation +# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation. +set(protobuf_MODULE_COMPATIBLE TRUE) +find_package(Protobuf CONFIG REQUIRED) +message(STATUS "Using protobuf ${protobuf_VERSION}") + +set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) +set(_PROTOBUF_PROTOC $) + +# Find gRPC installation +# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. +find_package(gRPC CONFIG REQUIRED) +message(STATUS "Using gRPC ${gRPC_VERSION}") + +set(_GRPC_GRPCPP_UNSECURE gRPC::grpc++_unsecure) +set(_GRPC_CPP_PLUGIN_EXECUTABLE $) + +# Proto file +get_filename_component(arion_proto "${CMAKE_CURRENT_SOURCE_DIR}/../../arion-master/schema/proto3/*.proto" ABSOLUTE) +get_filename_component(arion_proto_path "${arion_proto}" PATH) + +set(arion_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/arionmaster.pb.cc") +set(arion_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/arionmaster.pb.h") +set(arion_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/arionmaster.grpc.pb.cc") +set(arion_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/arionmaster.grpc.pb.h") +add_custom_command( + OUTPUT "${arion_proto_srcs}" "${arion_proto_hdrs}" "${arion_grpc_srcs}" "${arion_grpc_hdrs}" + COMMAND ${_PROTOBUF_PROTOC} + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${arion_proto_path}" + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${arion_proto}" + DEPENDS "${arion_proto}") + +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}") + +ADD_LIBRARY(grpc ${arion_proto_srcs} ${arion_proto_hdrs} ${arion_grpc_srcs} ${arion_grpc_hdrs}) \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..ef1a22b --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,144 @@ +// MIT License +// Copyright(c) 2022 Futurewei Cloud +// +// Permission is hereby granted, +// free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons +// to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#include "grpc_client.h" + +#include +#include +#include /* for getopt */ +#include +#include + +#include "marl/defer.h" +#include "marl/event.h" +#include "marl/scheduler.h" +#include "marl/waitgroup.h" + +using namespace std; +using std::string; + +// Defines +#define LOGNAME "ArionAgent" +static char EMPTY_STRING[] = ""; + +// Global variables +std::thread *g_grpc_client_thread = NULL; +ArionMasterWatcherImpl *g_grpc_client = NULL; + +string g_arion_master_address = EMPTY_STRING; +string g_arion_master_port = "9090"; +string g_arion_neighbor_table = "/sys/fs/bpf/endpoints_map"; +//TODO: read from goalstate +string g_arion_group = "group1"; + +// total time for goal state update in microseconds +std::atomic_ulong g_total_update_neighbor_time(0); + +bool g_debug_mode = false; +int processor_count = std::thread::hardware_concurrency(); +/* + From previous tests, we found that, for x number of cores, + it is more efficient to set the size of both thread pools + to be x * (2/3), which means the total size of the thread pools + is x * (4/3). For example, for a host with 24 cores, we would + set the sizes of both thread pools to be 16. +*/ +int thread_pools_size = (processor_count == 0) ? 1 : ((ceil(1.3 * processor_count)) / 2); + +static void cleanup() { + printf("%s", "Program exiting, cleaning up...\n"); + + // optional: delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + // stop the grpc client + if (g_grpc_client != NULL) { + delete g_grpc_client; + g_grpc_client = NULL; + printf("%s", "Cleaned up grpc client.\n"); + } else { + printf("%s", "Unable to delete grpc client pointer since it is null.\n"); + } + + if (g_grpc_client_thread != NULL) { + delete g_grpc_client_thread; + g_grpc_client_thread = NULL; + printf("%s", "Cleaned up grpc client thread.\n"); + } else { + printf("%s", "Unable to call delete grpc client thread pointer since it is null.\n"); + } +} + +// function to handle ctrl-c and kill process +static void signal_handler(int sig_num) { + printf("Caught signal: %d\n", sig_num); + + // perform all the necessary cleanup here + cleanup(); + exit(sig_num); +} + +int main(int argc, char *argv[]) { + int option; + int rc = 0; + + printf("%s", "Arion Agent started...\n"); + + // Register input key signal handlers + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + while ((option = getopt(argc, argv, "a:p:d")) != -1) { + switch (option) { + case 'a': + g_arion_master_address = optarg; + break; + case 'p': + g_arion_master_port = optarg; + break; + case 'd': + g_debug_mode = true; + break; + default: //the '?' case when the option is not recognized + printf("Usage: %s\n" + "\t\t[-a Arion Master Server IP Address]\n" + "\t\t[-p Arion Master Server Port]\n" + "\t\t[-d Enable debug mode]\n", + argv[0]); + exit(EXIT_FAILURE); + } + } + + // Create marl scheduler using all the logical processors available to the process. + // Bind this scheduler to the main thread so we can call marl::schedule() + marl::Scheduler::Config cfg_bind_hw_cores; + cfg_bind_hw_cores.setWorkerThreadCount(thread_pools_size * 2); + marl::Scheduler task_scheduler(cfg_bind_hw_cores); + task_scheduler.bind(); + defer(task_scheduler.unbind()); + + // Create a separate thread to run the grpc client of watching Arion Master + g_grpc_client = new ArionMasterWatcherImpl(); + marl::schedule([=] { + g_grpc_client->RunClient(g_arion_master_address, + g_arion_master_port, + g_arion_group, + g_arion_neighbor_table); + }); + + pause(); + cleanup(); + + return rc; +} diff --git a/src/proto3/CMakeLists.txt b/src/proto3/CMakeLists.txt new file mode 100644 index 0000000..77c2807 --- /dev/null +++ b/src/proto3/CMakeLists.txt @@ -0,0 +1,6 @@ +INCLUDE(FindProtobuf) +FIND_PACKAGE(Protobuf REQUIRED) +INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) +file(GLOB ProtoFiles "${CMAKE_CURRENT_SOURCE_DIR}/../../arion-master/schema/proto3/*.proto") +PROTOBUF_GENERATE_CPP(ProtoSources ProtoHeaders ${ProtoFiles}) +ADD_LIBRARY(proto ${ProtoHeaders} ${ProtoSources})