Merge branch 'release-1.100.0'

telstra · Jul 26, 2021 · e166fcb · e166fcb
2 parents 4a67498 + a3aad3c
commit e166fcb
Show file tree

Hide file tree

Showing 64 changed files with 1,146 additions and 437 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,32 @@
 # Changelog
 
+## v1.100.0 (26/07/2021)
+
+### Bug Fixes:
+-  [#4337](https://github.com/telstra/open-kilda/pull/4337) Replace InetSocketAddress with own IpSocket objects [**floodlight**][**storm-topologies**]
+-  [#4381](https://github.com/telstra/open-kilda/pull/4381) [Server42] Support long flowid fixes #3720
+
+### Improvements:
+-  [#4352](https://github.com/telstra/open-kilda/pull/4352) DB cleanup after fix feature toggles
+-  [#4324](https://github.com/telstra/open-kilda/pull/4324) Rework flow monitoring topology to support parallelism (Issue: [#3968](https://github.com/telstra/open-kilda/issues/3968))
+-  [#4361](https://github.com/telstra/open-kilda/pull/4361) use sla.check variable in file with properties for func-tests [**tests**]
+-  [#4370](https://github.com/telstra/open-kilda/pull/4370) refactor MflStatSpec [**tests**]
+
+### Other changes:
+-  [#4362](https://github.com/telstra/open-kilda/pull/4362) Tests: fix case when traffgen cleanup is not being run [**tests**]
+-  [#4373](https://github.com/telstra/open-kilda/pull/4373) Update mirror tests to iterate over all available sw models [**tests**]
+-  [#4378](https://github.com/telstra/open-kilda/pull/4378) [Server42] Fix build dpdk on linux 5.6
+
+For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.99.0...v1.100.0).
+
+### Affected Components:
+network, flow-monitor, server42, fl
+
+### Upgrade notes:
+OrientDB schema have been changed in this release. You need to apply schema migration. Please follow [migration instructions](https://github.com/telstra/open-kilda/tree/develop/docker/db-migration/migrations).
+
+---
+
 ## v1.99.0 (15/07/2021)
 
 ### Features:

diff --git a/confd/templates/atdd-staging/kilda.properties.tmpl b/confd/templates/atdd-staging/kilda.properties.tmpl
@@ -86,3 +86,4 @@ flow.ping.interval=5
 docker.host=localhost
 
 latency.update.interval = {{ getv "/kilda_latency_update_interval" }}
+flow.sla.check.interval.seconds = {{ getv "/kilda_flow_sla_check_interval_seconds" }}
diff --git a/confd/templates/flowmonitoring-topology/flowmonitoring-topology.tmpl b/confd/templates/flowmonitoring-topology/flowmonitoring-topology.tmpl
@@ -14,7 +14,3 @@ bolts:
     parallelism: 1
   - id: "TICK_BOLT"
     parallelism: 1
-  - id: "FLOW_CACHE_BOLT"
-    parallelism: 1
-  - id: "ISL_CACHE_BOLT"
-    parallelism: 1
diff --git a/docker/db-migration/migrations/009-cleanup-config-class.yaml b/docker/db-migration/migrations/009-cleanup-config-class.yaml
@@ -0,0 +1,26 @@
+databaseChangeLog:
+  - changeSet:
+      id: tag
+      author: dpoltavets
+      changes:
+        - tagDatabase:
+            tag: 009-cleanup-config-class
+
+  - changeSet:
+      id: cleanup_config_class
+      author: dpoltavets
+      changes:
+        - sql: "DELETE VERTEX config"
+        - sql: "DROP CLASS config"
+        - sql: "DROP INDEX kilda_configuration.unique"
+        - sql: "UPDATE kilda_configuration REMOVE unique"
+        - sql: "DROP PROPERTY kilda_configuration.unique"
+      rollback:
+        - sql: "CREATE CLASS config IF NOT EXISTS EXTENDS V"
+        - sql: "CREATE PROPERTY config.unique IF NOT EXISTS STRING"
+        - sql: "CREATE INDEX config.unique UNIQUE_HASH_INDEX"
+        - sql: "INSERT INTO config FROM (SELECT FROM kilda_feature_toggles)"
+        - sql: "UPDATE config SET unique='config'"
+        - sql: "CREATE PROPERTY kilda_configuration.unique IF NOT EXISTS STRING"
+        - sql: "CREATE INDEX kilda_configuration.unique UNIQUE_HASH_INDEX"
+        - sql: "UPDATE kilda_configuration SET unique='config'"
diff --git a/docker/db-migration/migrations/root.yaml b/docker/db-migration/migrations/root.yaml
@@ -36,3 +36,6 @@ databaseChangeLog:
   - include:
       relativeToChangelogFile: true
       file: 008-add-kilda-feature-toggles-class.yaml
+  - include:
+      relativeToChangelogFile: true
+      file: 009-cleanup-config-class.yaml
diff --git a/src-cpp/server42/Dockerfile.prebuild b/src-cpp/server42/Dockerfile.prebuild
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 WORKDIR /root
 

diff --git a/src-cpp/server42/docker_install_requirements.sh b/src-cpp/server42/docker_install_requirements.sh
@@ -3,8 +3,7 @@
 echo 'deb http://archive.ubuntu.com/ubuntu/ bionic-proposed main' > /etc/apt/sources.list.d/bionic-proposed.list
 
 apt-get update -y && \
-apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3.7 python3.7-dev wget linux-headers-$(uname -r) patch sudo iproute2 && \
-update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1000 && \
+apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3 python3-dev wget linux-headers-$(uname -r) patch sudo iproute2 git && \
 mkdir -p bin && \
 mkdir -p tools/cmake && \
 cd tools/cmake && \

diff --git a/src-cpp/server42/external/upstream/dpdk/CMakeLists.txt b/src-cpp/server42/external/upstream/dpdk/CMakeLists.txt
@@ -1,14 +1,16 @@
 include(ExternalProject)
 ExternalProject_Add(dpdk_external
         URL
-        http://fast.dpdk.org/rel/dpdk-19.08.2.tar.xz
+        https://github.com/DPDK/dpdk/archive/refs/tags/v19.11.tar.gz
         URL_HASH
-        SHA1=e203bbe12aeebb638c1eb320c4abeb423b3e7aec
+        SHA1=f8dd55689df1baf123942bb308a06141d25698d8
         DOWNLOAD_NO_PROGRESS
         1
         CONFIGURE_COMMAND
         ${CMAKE_MAKE_PROGRAM} config T=x86_64-native-linuxapp-gcc
         COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/dpdk_config.sh <SOURCE_DIR>
+        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/kni.patch <SOURCE_DIR>
+        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/eal_rte_random.patch <SOURCE_DIR>
         COMMAND /bin/sh dpdk_config.sh
         BUILD_COMMAND
         ${CMAKE_MAKE_PROGRAM}

diff --git a/src-cpp/server42/external/upstream/dpdk/dpdk_config.sh b/src-cpp/server42/external/upstream/dpdk/dpdk_config.sh
@@ -1,3 +1,5 @@
 sed -i "s/CONFIG_RTE_PORT_PCAP=n/CONFIG_RTE_PORT_PCAP=y/g" build/.config
 sed -i "s/CONFIG_RTE_LIBRTE_PMD_PCAP=n/CONFIG_RTE_LIBRTE_PMD_PCAP=y/g" build/.config
 sed -i "s/CONFIG_RTE_LIBEAL_USE_HPET=n/CONFIG_RTE_LIBEAL_USE_HPET=y/g" build/.config
+patch -p0 < kni.patch
+patch -p0 < eal_rte_random.patch
diff --git a/src-cpp/server42/external/upstream/dpdk/eal_rte_random.patch b/src-cpp/server42/external/upstream/dpdk/eal_rte_random.patch
@@ -0,0 +1,11 @@
+--- lib/librte_eal/common/rte_random.c	2019-11-29 00:13:49.000000000 +0200
++++ lib/librte_eal/common/rte_random_new.c	2021-07-15 16:26:34.057113279 +0300
+@@ -198,7 +198,7 @@
+ 		return (uint64_t)rdseed_low | ((uint64_t)rdseed_high << 32);
+ #endif
+ 	/* second fallback: seed using rdtsc */
+-	return rte_get_timer_cycles();
++	return rte_get_tsc_cycles();
+ }
+
+ RTE_INIT(rte_rand_init)
diff --git a/src-cpp/server42/external/upstream/dpdk/kni.patch b/src-cpp/server42/external/upstream/dpdk/kni.patch
@@ -0,0 +1,29 @@
+unchanged:
+--- kernel/linux/kni/kni_net.c	2019-11-29 00:13:49.000000000 +0200
++++ kernel/linux/kni/kni_net_new.c	2021-07-06 18:07:17.457994637 +0300
+@@ -623,8 +623,13 @@
+ /*
+  * Deal with a transmit timeout.
+  */
++#ifdef HAVE_TX_TIMEOUT_TXQUEUE
++static void
++kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)
++#else
+ static void
+ kni_net_tx_timeout(struct net_device *dev)
++#endif
+ {
+ 	pr_debug("Transmit timeout at %ld, latency %ld\n", jiffies,
+ 			jiffies - dev_trans_start(dev));
+only in patch2:
+unchanged:
+--- kernel/linux/kni/compat.h	2019-11-29 00:13:49.000000000 +0200
++++ kernel/linux/kni/compat_new.h	2021-07-06 18:04:44.252843148 +0300
+@@ -130,3 +130,7 @@
+ #if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE
+ #define HAVE_IOVA_TO_KVA_MAPPING_SUPPORT
+ #endif
++
++#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE
++#define HAVE_TX_TIMEOUT_TXQUEUE
++#endif
diff --git a/src-cpp/server42/host_install_requirements.sh b/src-cpp/server42/host_install_requirements.sh
@@ -1,8 +1,7 @@
 #!/usr/bin/env bash
 
 sudo apt-get update -y && \
-sudo apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3.7 python3.7-dev wget linux-headers-$(uname -r) && \
-sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1000 && \
+sudo apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3-dev wget linux-headers-$(uname -r) && \
 mkdir -p tools/cmake && \
 cd tools/cmake && \
 wget -nc https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.tar.gz && \

diff --git a/src-cpp/server42/src/PacketGenerator.cpp b/src-cpp/server42/src/PacketGenerator.cpp
@@ -61,13 +61,15 @@ namespace org::openkilda {
         newPacket.addLayer(&newUdpLayer);
 
         FlowPayload payload{};
-
-        size_t length = arg.flow_id.copy(payload.flow_id, sizeof(payload.flow_id) - 1);
-        payload.flow_id[length] = '\0';
-
         payload.direction = arg.direction;
+        payload.flow_id_length = arg.flow_id.size();
+        payload.flow_id_offset = sizeof payload;
 
-        pcpp::PayloadLayer payloadLayer(reinterpret_cast<uint8_t *>(&payload), sizeof(payload), false);
+        std::vector<uint8_t> buffer(sizeof payload + arg.flow_id.size());
+        std::memcpy(buffer.data(), &payload, sizeof payload);
+        std::memcpy(buffer.data() + payload.flow_id_offset, arg.flow_id.c_str(), arg.flow_id.length());
+
+        pcpp::PayloadLayer payloadLayer(buffer.data(), buffer.size(), false);
         newPacket.addLayer(&payloadLayer);
         newPacket.computeCalculateFields();
 

diff --git a/src-cpp/server42/src/Payload.h b/src-cpp/server42/src/Payload.h
@@ -5,8 +5,9 @@ namespace org::openkilda {
     struct FlowPayload {
         int64_t t0;
         int64_t t1;
-        char flow_id[32];
         bool direction;
+        uint8_t flow_id_offset;
+        size_t flow_id_length;
     };
 
     struct IslPayload {

diff --git a/src-cpp/server42/src/Workers.cpp b/src-cpp/server42/src/Workers.cpp
@@ -276,21 +276,22 @@ namespace org::openkilda {
 
                     if (likely(eth->getDestMac() == src_mac && udp)) {
                         auto udpSrcPort = ntohs(udp->getUdpHeader()->portSrc);
-                        if (udpSrcPort == Config::flow_rtt_caught_packet_udp_src_port
-                                && udp->getLayerPayloadSize() == sizeof(FlowPayload)) {
+                        if (udpSrcPort == Config::flow_rtt_caught_packet_udp_src_port) {
                             packet_id++;
                             auto payload = reinterpret_cast<const org::openkilda::FlowPayload *>(udp->getLayerPayload());
                             auto packet = bucket.add_flowlatencypacket();
 
-                            BOOST_LOG_TRIVIAL(debug) << "flow_id: " << payload->flow_id << " "
+                            std::string flow_id(reinterpret_cast<const char*>(payload) + payload->flow_id_offset, payload->flow_id_length);
+
+                            BOOST_LOG_TRIVIAL(debug) << "flow_id: " << flow_id << " "
                                                      << "t0 raw: " << payload->t0 << " "
                                                      << "t1 raw: " << payload->t1 << " "
                                                      << "t0: " << be64toh(payload->t0) << " "
                                                      << "t1: " << be64toh(payload->t1) << " "
                                                      << "packet_id: " << packet_id << " "
                                                      << "direction: " << payload->direction;
 
-                            packet->set_flow_id(payload->flow_id);
+                            packet->set_flow_id(flow_id);
                             packet->set_t0(be64toh(payload->t0));
 
                             if (payload->t1) {
@@ -411,13 +412,12 @@ namespace org::openkilda {
                 auto *udp_layer = parsed_packet.getLayerOfType<pcpp::UdpLayer>();
                 auto udpDstPort = ntohs(udp_layer->getUdpHeader()->portDst);
 
-                if (udpDstPort == Config::flow_rtt_generated_packet_udp_dst_port
-                    && udp_layer->getLayerPayloadSize() == sizeof(FlowPayload)) {
+                if (udpDstPort == Config::flow_rtt_generated_packet_udp_dst_port) {
                     udp_layer->getUdpHeader()->portSrc = htons(Config::flow_rtt_caught_packet_udp_src_port);
 
                     auto *payload = reinterpret_cast<org::openkilda::FlowPayload *>(udp_layer->getLayerPayload());
 
-                    std::string flow_id(payload->flow_id);
+                    std::string flow_id(reinterpret_cast<const char*>(payload) + payload->flow_id_offset, payload->flow_id_length);
 
                     if (!fake_duration.count(flow_id)) {
                         fake_duration[flow_id] = base_latency(rd) * cycles_in_1_ms;

diff --git a/...ology/base-messaging/src/main/java/org/openkilda/messaging/info/event/SwitchInfoData.java b/...ology/base-messaging/src/main/java/org/openkilda/messaging/info/event/SwitchInfoData.java
@@ -44,9 +44,6 @@ public class SwitchInfoData extends CacheTimeTag {
     @JsonProperty("address")
     private String address;
 
-    @JsonProperty("hostname")
-    private String hostname;
-
     @JsonProperty("description")
     private String description;
 
@@ -73,7 +70,7 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state) {
 
     public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address, String hostname,
                           String description, String controller, boolean underMaintenance) {
-        this(switchId, state, address, hostname, description, controller, underMaintenance, null);
+        this(switchId, state, address, description, controller, underMaintenance, null);
     }
 
     /**
@@ -82,7 +79,6 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address,
      * @param switchId    switch datapath id
      * @param state       switch state
      * @param address     switch ip address
-     * @param hostname    switch name
      * @param description switch description
      * @param controller  switch controller
      * @param switchView data for ISL/switch discovery
@@ -92,15 +88,13 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address,
     public SwitchInfoData(@JsonProperty("switch_id") SwitchId switchId,
                           @JsonProperty("state") SwitchChangeType state,
                           @JsonProperty("address") String address,
-                          @JsonProperty("hostname") String hostname,
                           @JsonProperty("description") String description,
                           @JsonProperty("controller") String controller,
                           @JsonProperty("under_maintenance") boolean underMaintenance,
                           @JsonProperty("switch") SpeakerSwitchView switchView) {
         this.switchId = switchId;
         this.state = state;
         this.address = address;
-        this.hostname = hostname;
         this.description = description;
         this.controller = controller;
         this.switchView = switchView;

diff --git a/...opology/base-messaging/src/main/java/org/openkilda/messaging/model/SpeakerSwitchView.java b/...opology/base-messaging/src/main/java/org/openkilda/messaging/model/SpeakerSwitchView.java
@@ -15,6 +15,7 @@
 
 package org.openkilda.messaging.model;
 
+import org.openkilda.model.IpSocketAddress;
 import org.openkilda.model.SwitchFeature;
 import org.openkilda.model.SwitchId;
 
@@ -27,7 +28,6 @@
 import lombok.Value;
 
 import java.io.Serializable;
-import java.net.InetSocketAddress;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -39,10 +39,13 @@ public class SpeakerSwitchView implements Serializable {
     private SwitchId datapath;
 
     @JsonProperty(value = "switch-socket", required = true)
-    private InetSocketAddress switchSocketAddress;
+    private IpSocketAddress switchSocketAddress;
 
     @JsonProperty(value = "speaker-socket", required = true)
-    private InetSocketAddress speakerSocketAddress;
+    private IpSocketAddress speakerSocketAddress;
+
+    @JsonProperty(value = "hostname")
+    String hostname;
 
     // TODO: move to enum
     @JsonProperty(value = "OF-version")
@@ -61,15 +64,17 @@ public class SpeakerSwitchView implements Serializable {
     @JsonCreator
     public SpeakerSwitchView(
             @JsonProperty("datapath") SwitchId datapath,
-            @JsonProperty("switch-socket") InetSocketAddress switchSocketAddress,
-            @JsonProperty("speaker-socket") InetSocketAddress speakerSocketAddress,
+            @JsonProperty("switch-socket") IpSocketAddress switchSocketAddress,
+            @JsonProperty("speaker-socket") IpSocketAddress speakerSocketAddress,
+            @JsonProperty("hostname") String hostname,
             @JsonProperty("OF-version") String ofVersion,
             @JsonProperty("description") SpeakerSwitchDescription description,
             @JsonProperty("features") Set<SwitchFeature> features,
             @Singular @JsonProperty("ports") List<SpeakerSwitchPortView> ports) {
         this.datapath = datapath;
         this.switchSocketAddress = switchSocketAddress;
         this.speakerSocketAddress = speakerSocketAddress;
+        this.hostname = hostname;
         this.ofVersion = ofVersion;
         this.description = description;
 

diff --git a/...y/base-messaging/src/main/java/org/openkilda/messaging/model/SwitchAvailabilityEntry.java b/...y/base-messaging/src/main/java/org/openkilda/messaging/model/SwitchAvailabilityEntry.java
@@ -15,13 +15,13 @@
 
 package org.openkilda.messaging.model;
 
+import org.openkilda.model.IpSocketAddress;
 import org.openkilda.model.SwitchConnectMode;
 
 import lombok.Builder;
 import lombok.Value;
 
 import java.io.Serializable;
-import java.net.InetSocketAddress;
 import java.time.Instant;
 
 @Value
@@ -35,6 +35,6 @@ public class SwitchAvailabilityEntry implements Serializable {
 
     Instant connectedAt;
 
-    InetSocketAddress switchAddress;
-    InetSocketAddress speakerAddress;
+    IpSocketAddress switchAddress;
+    IpSocketAddress speakerAddress;
 }
diff --git a/.../base-topology/base-messaging/src/test/java/org/openkilda/messaging/model/SwitchTest.java b/.../base-topology/base-messaging/src/test/java/org/openkilda/messaging/model/SwitchTest.java
@@ -17,6 +17,7 @@
 
 import org.openkilda.messaging.StringSerializer;
 import org.openkilda.messaging.Utils;
+import org.openkilda.model.IpSocketAddress;
 import org.openkilda.model.Switch;
 import org.openkilda.model.SwitchFeature;
 import org.openkilda.model.SwitchId;
@@ -26,7 +27,6 @@
 import org.junit.Test;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.util.Collections;
 
 public class SwitchTest {
@@ -52,7 +52,7 @@ public void serializeLoop() throws Exception {
                 .pop("test_pop")
                 .status(SwitchStatus.ACTIVE)
                 .underMaintenance(true)
-                .socketAddress(new InetSocketAddress(1))
+                .socketAddress(new IpSocketAddress("192.168.10.20", 1))
                 .features(Collections.singleton(SwitchFeature.GROUP_PACKET_OUT_CONTROLLER))
                 .build();
         serializer.serialize(origin);

diff --git a/...oodlight-modules/src/main/java/org/openkilda/floodlight/converter/IofSwitchConverter.java b/...oodlight-modules/src/main/java/org/openkilda/floodlight/converter/IofSwitchConverter.java
@@ -50,7 +50,6 @@ public static SwitchInfoData buildSwitchInfoData(IOFSwitch sw, SpeakerSwitchView
                 String.format("%s:%d",
                         address.getHostString(),
                         address.getPort()),
-                address.getHostName(),
                 String.format("%s %s %s",
                         sw.getSwitchDescription().getManufacturerDescription(),
                         sw.getOFFactory().getVersion().toString(),