Skip to content

Commit

Permalink
Merge branch 'release-1.100.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
timofei-durakov committed Jul 26, 2021
2 parents 4a67498 + a3aad3c commit e166fcb
Show file tree
Hide file tree
Showing 64 changed files with 1,146 additions and 437 deletions.
27 changes: 27 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,32 @@
# Changelog

## v1.100.0 (26/07/2021)

### Bug Fixes:
- [#4337](https://github.com/telstra/open-kilda/pull/4337) Replace InetSocketAddress with own IpSocket objects [**floodlight**][**storm-topologies**]
- [#4381](https://github.com/telstra/open-kilda/pull/4381) [Server42] Support long flowid fixes #3720

### Improvements:
- [#4352](https://github.com/telstra/open-kilda/pull/4352) DB cleanup after fix feature toggles
- [#4324](https://github.com/telstra/open-kilda/pull/4324) Rework flow monitoring topology to support parallelism (Issue: [#3968](https://github.com/telstra/open-kilda/issues/3968))
- [#4361](https://github.com/telstra/open-kilda/pull/4361) use sla.check variable in file with properties for func-tests [**tests**]
- [#4370](https://github.com/telstra/open-kilda/pull/4370) refactor MflStatSpec [**tests**]

### Other changes:
- [#4362](https://github.com/telstra/open-kilda/pull/4362) Tests: fix case when traffgen cleanup is not being run [**tests**]
- [#4373](https://github.com/telstra/open-kilda/pull/4373) Update mirror tests to iterate over all available sw models [**tests**]
- [#4378](https://github.com/telstra/open-kilda/pull/4378) [Server42] Fix build dpdk on linux 5.6

For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.99.0...v1.100.0).

### Affected Components:
network, flow-monitor, server42, fl

### Upgrade notes:
OrientDB schema have been changed in this release. You need to apply schema migration. Please follow [migration instructions](https://github.com/telstra/open-kilda/tree/develop/docker/db-migration/migrations).

---

## v1.99.0 (15/07/2021)

### Features:
Expand Down
1 change: 1 addition & 0 deletions confd/templates/atdd-staging/kilda.properties.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,4 @@ flow.ping.interval=5
docker.host=localhost

latency.update.interval = {{ getv "/kilda_latency_update_interval" }}
flow.sla.check.interval.seconds = {{ getv "/kilda_flow_sla_check_interval_seconds" }}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,3 @@ bolts:
parallelism: 1
- id: "TICK_BOLT"
parallelism: 1
- id: "FLOW_CACHE_BOLT"
parallelism: 1
- id: "ISL_CACHE_BOLT"
parallelism: 1
26 changes: 26 additions & 0 deletions docker/db-migration/migrations/009-cleanup-config-class.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
databaseChangeLog:
- changeSet:
id: tag
author: dpoltavets
changes:
- tagDatabase:
tag: 009-cleanup-config-class

- changeSet:
id: cleanup_config_class
author: dpoltavets
changes:
- sql: "DELETE VERTEX config"
- sql: "DROP CLASS config"
- sql: "DROP INDEX kilda_configuration.unique"
- sql: "UPDATE kilda_configuration REMOVE unique"
- sql: "DROP PROPERTY kilda_configuration.unique"
rollback:
- sql: "CREATE CLASS config IF NOT EXISTS EXTENDS V"
- sql: "CREATE PROPERTY config.unique IF NOT EXISTS STRING"
- sql: "CREATE INDEX config.unique UNIQUE_HASH_INDEX"
- sql: "INSERT INTO config FROM (SELECT FROM kilda_feature_toggles)"
- sql: "UPDATE config SET unique='config'"
- sql: "CREATE PROPERTY kilda_configuration.unique IF NOT EXISTS STRING"
- sql: "CREATE INDEX kilda_configuration.unique UNIQUE_HASH_INDEX"
- sql: "UPDATE kilda_configuration SET unique='config'"
3 changes: 3 additions & 0 deletions docker/db-migration/migrations/root.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,6 @@ databaseChangeLog:
- include:
relativeToChangelogFile: true
file: 008-add-kilda-feature-toggles-class.yaml
- include:
relativeToChangelogFile: true
file: 009-cleanup-config-class.yaml
2 changes: 1 addition & 1 deletion src-cpp/server42/Dockerfile.prebuild
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM ubuntu:18.04
FROM ubuntu:20.04

WORKDIR /root

Expand Down
3 changes: 1 addition & 2 deletions src-cpp/server42/docker_install_requirements.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
echo 'deb http://archive.ubuntu.com/ubuntu/ bionic-proposed main' > /etc/apt/sources.list.d/bionic-proposed.list

apt-get update -y && \
apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3.7 python3.7-dev wget linux-headers-$(uname -r) patch sudo iproute2 && \
update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1000 && \
apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3 python3-dev wget linux-headers-$(uname -r) patch sudo iproute2 git && \
mkdir -p bin && \
mkdir -p tools/cmake && \
cd tools/cmake && \
Expand Down
6 changes: 4 additions & 2 deletions src-cpp/server42/external/upstream/dpdk/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
include(ExternalProject)
ExternalProject_Add(dpdk_external
URL
http://fast.dpdk.org/rel/dpdk-19.08.2.tar.xz
https://github.com/DPDK/dpdk/archive/refs/tags/v19.11.tar.gz
URL_HASH
SHA1=e203bbe12aeebb638c1eb320c4abeb423b3e7aec
SHA1=f8dd55689df1baf123942bb308a06141d25698d8
DOWNLOAD_NO_PROGRESS
1
CONFIGURE_COMMAND
${CMAKE_MAKE_PROGRAM} config T=x86_64-native-linuxapp-gcc
COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/dpdk_config.sh <SOURCE_DIR>
COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/kni.patch <SOURCE_DIR>
COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/eal_rte_random.patch <SOURCE_DIR>
COMMAND /bin/sh dpdk_config.sh
BUILD_COMMAND
${CMAKE_MAKE_PROGRAM}
Expand Down
2 changes: 2 additions & 0 deletions src-cpp/server42/external/upstream/dpdk/dpdk_config.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
sed -i "s/CONFIG_RTE_PORT_PCAP=n/CONFIG_RTE_PORT_PCAP=y/g" build/.config
sed -i "s/CONFIG_RTE_LIBRTE_PMD_PCAP=n/CONFIG_RTE_LIBRTE_PMD_PCAP=y/g" build/.config
sed -i "s/CONFIG_RTE_LIBEAL_USE_HPET=n/CONFIG_RTE_LIBEAL_USE_HPET=y/g" build/.config
patch -p0 < kni.patch
patch -p0 < eal_rte_random.patch
11 changes: 11 additions & 0 deletions src-cpp/server42/external/upstream/dpdk/eal_rte_random.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
--- lib/librte_eal/common/rte_random.c 2019-11-29 00:13:49.000000000 +0200
+++ lib/librte_eal/common/rte_random_new.c 2021-07-15 16:26:34.057113279 +0300
@@ -198,7 +198,7 @@
return (uint64_t)rdseed_low | ((uint64_t)rdseed_high << 32);
#endif
/* second fallback: seed using rdtsc */
- return rte_get_timer_cycles();
+ return rte_get_tsc_cycles();
}

RTE_INIT(rte_rand_init)
29 changes: 29 additions & 0 deletions src-cpp/server42/external/upstream/dpdk/kni.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
unchanged:
--- kernel/linux/kni/kni_net.c 2019-11-29 00:13:49.000000000 +0200
+++ kernel/linux/kni/kni_net_new.c 2021-07-06 18:07:17.457994637 +0300
@@ -623,8 +623,13 @@
/*
* Deal with a transmit timeout.
*/
+#ifdef HAVE_TX_TIMEOUT_TXQUEUE
+static void
+kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)
+#else
static void
kni_net_tx_timeout(struct net_device *dev)
+#endif
{
pr_debug("Transmit timeout at %ld, latency %ld\n", jiffies,
jiffies - dev_trans_start(dev));
only in patch2:
unchanged:
--- kernel/linux/kni/compat.h 2019-11-29 00:13:49.000000000 +0200
+++ kernel/linux/kni/compat_new.h 2021-07-06 18:04:44.252843148 +0300
@@ -130,3 +130,7 @@
#if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE
#define HAVE_IOVA_TO_KVA_MAPPING_SUPPORT
#endif
+
+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE
+#define HAVE_TX_TIMEOUT_TXQUEUE
+#endif
3 changes: 1 addition & 2 deletions src-cpp/server42/host_install_requirements.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#!/usr/bin/env bash

sudo apt-get update -y && \
sudo apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3.7 python3.7-dev wget linux-headers-$(uname -r) && \
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1000 && \
sudo apt-get install -y libpcap-dev gcc g++ libnuma-dev make python3-dev wget linux-headers-$(uname -r) && \
mkdir -p tools/cmake && \
cd tools/cmake && \
wget -nc https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.tar.gz && \
Expand Down
12 changes: 7 additions & 5 deletions src-cpp/server42/src/PacketGenerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,15 @@ namespace org::openkilda {
newPacket.addLayer(&newUdpLayer);

FlowPayload payload{};

size_t length = arg.flow_id.copy(payload.flow_id, sizeof(payload.flow_id) - 1);
payload.flow_id[length] = '\0';

payload.direction = arg.direction;
payload.flow_id_length = arg.flow_id.size();
payload.flow_id_offset = sizeof payload;

pcpp::PayloadLayer payloadLayer(reinterpret_cast<uint8_t *>(&payload), sizeof(payload), false);
std::vector<uint8_t> buffer(sizeof payload + arg.flow_id.size());
std::memcpy(buffer.data(), &payload, sizeof payload);
std::memcpy(buffer.data() + payload.flow_id_offset, arg.flow_id.c_str(), arg.flow_id.length());

pcpp::PayloadLayer payloadLayer(buffer.data(), buffer.size(), false);
newPacket.addLayer(&payloadLayer);
newPacket.computeCalculateFields();

Expand Down
3 changes: 2 additions & 1 deletion src-cpp/server42/src/Payload.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ namespace org::openkilda {
struct FlowPayload {
int64_t t0;
int64_t t1;
char flow_id[32];
bool direction;
uint8_t flow_id_offset;
size_t flow_id_length;
};

struct IslPayload {
Expand Down
14 changes: 7 additions & 7 deletions src-cpp/server42/src/Workers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -276,21 +276,22 @@ namespace org::openkilda {

if (likely(eth->getDestMac() == src_mac && udp)) {
auto udpSrcPort = ntohs(udp->getUdpHeader()->portSrc);
if (udpSrcPort == Config::flow_rtt_caught_packet_udp_src_port
&& udp->getLayerPayloadSize() == sizeof(FlowPayload)) {
if (udpSrcPort == Config::flow_rtt_caught_packet_udp_src_port) {
packet_id++;
auto payload = reinterpret_cast<const org::openkilda::FlowPayload *>(udp->getLayerPayload());
auto packet = bucket.add_flowlatencypacket();

BOOST_LOG_TRIVIAL(debug) << "flow_id: " << payload->flow_id << " "
std::string flow_id(reinterpret_cast<const char*>(payload) + payload->flow_id_offset, payload->flow_id_length);

BOOST_LOG_TRIVIAL(debug) << "flow_id: " << flow_id << " "
<< "t0 raw: " << payload->t0 << " "
<< "t1 raw: " << payload->t1 << " "
<< "t0: " << be64toh(payload->t0) << " "
<< "t1: " << be64toh(payload->t1) << " "
<< "packet_id: " << packet_id << " "
<< "direction: " << payload->direction;

packet->set_flow_id(payload->flow_id);
packet->set_flow_id(flow_id);
packet->set_t0(be64toh(payload->t0));

if (payload->t1) {
Expand Down Expand Up @@ -411,13 +412,12 @@ namespace org::openkilda {
auto *udp_layer = parsed_packet.getLayerOfType<pcpp::UdpLayer>();
auto udpDstPort = ntohs(udp_layer->getUdpHeader()->portDst);

if (udpDstPort == Config::flow_rtt_generated_packet_udp_dst_port
&& udp_layer->getLayerPayloadSize() == sizeof(FlowPayload)) {
if (udpDstPort == Config::flow_rtt_generated_packet_udp_dst_port) {
udp_layer->getUdpHeader()->portSrc = htons(Config::flow_rtt_caught_packet_udp_src_port);

auto *payload = reinterpret_cast<org::openkilda::FlowPayload *>(udp_layer->getLayerPayload());

std::string flow_id(payload->flow_id);
std::string flow_id(reinterpret_cast<const char*>(payload) + payload->flow_id_offset, payload->flow_id_length);

if (!fake_duration.count(flow_id)) {
fake_duration[flow_id] = base_latency(rd) * cycles_in_1_ms;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,6 @@ public class SwitchInfoData extends CacheTimeTag {
@JsonProperty("address")
private String address;

@JsonProperty("hostname")
private String hostname;

@JsonProperty("description")
private String description;

Expand All @@ -73,7 +70,7 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state) {

public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address, String hostname,
String description, String controller, boolean underMaintenance) {
this(switchId, state, address, hostname, description, controller, underMaintenance, null);
this(switchId, state, address, description, controller, underMaintenance, null);
}

/**
Expand All @@ -82,7 +79,6 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address,
* @param switchId switch datapath id
* @param state switch state
* @param address switch ip address
* @param hostname switch name
* @param description switch description
* @param controller switch controller
* @param switchView data for ISL/switch discovery
Expand All @@ -92,15 +88,13 @@ public SwitchInfoData(SwitchId switchId, SwitchChangeType state, String address,
public SwitchInfoData(@JsonProperty("switch_id") SwitchId switchId,
@JsonProperty("state") SwitchChangeType state,
@JsonProperty("address") String address,
@JsonProperty("hostname") String hostname,
@JsonProperty("description") String description,
@JsonProperty("controller") String controller,
@JsonProperty("under_maintenance") boolean underMaintenance,
@JsonProperty("switch") SpeakerSwitchView switchView) {
this.switchId = switchId;
this.state = state;
this.address = address;
this.hostname = hostname;
this.description = description;
this.controller = controller;
this.switchView = switchView;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package org.openkilda.messaging.model;

import org.openkilda.model.IpSocketAddress;
import org.openkilda.model.SwitchFeature;
import org.openkilda.model.SwitchId;

Expand All @@ -27,7 +28,6 @@
import lombok.Value;

import java.io.Serializable;
import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
Expand All @@ -39,10 +39,13 @@ public class SpeakerSwitchView implements Serializable {
private SwitchId datapath;

@JsonProperty(value = "switch-socket", required = true)
private InetSocketAddress switchSocketAddress;
private IpSocketAddress switchSocketAddress;

@JsonProperty(value = "speaker-socket", required = true)
private InetSocketAddress speakerSocketAddress;
private IpSocketAddress speakerSocketAddress;

@JsonProperty(value = "hostname")
String hostname;

// TODO: move to enum
@JsonProperty(value = "OF-version")
Expand All @@ -61,15 +64,17 @@ public class SpeakerSwitchView implements Serializable {
@JsonCreator
public SpeakerSwitchView(
@JsonProperty("datapath") SwitchId datapath,
@JsonProperty("switch-socket") InetSocketAddress switchSocketAddress,
@JsonProperty("speaker-socket") InetSocketAddress speakerSocketAddress,
@JsonProperty("switch-socket") IpSocketAddress switchSocketAddress,
@JsonProperty("speaker-socket") IpSocketAddress speakerSocketAddress,
@JsonProperty("hostname") String hostname,
@JsonProperty("OF-version") String ofVersion,
@JsonProperty("description") SpeakerSwitchDescription description,
@JsonProperty("features") Set<SwitchFeature> features,
@Singular @JsonProperty("ports") List<SpeakerSwitchPortView> ports) {
this.datapath = datapath;
this.switchSocketAddress = switchSocketAddress;
this.speakerSocketAddress = speakerSocketAddress;
this.hostname = hostname;
this.ofVersion = ofVersion;
this.description = description;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@

package org.openkilda.messaging.model;

import org.openkilda.model.IpSocketAddress;
import org.openkilda.model.SwitchConnectMode;

import lombok.Builder;
import lombok.Value;

import java.io.Serializable;
import java.net.InetSocketAddress;
import java.time.Instant;

@Value
Expand All @@ -35,6 +35,6 @@ public class SwitchAvailabilityEntry implements Serializable {

Instant connectedAt;

InetSocketAddress switchAddress;
InetSocketAddress speakerAddress;
IpSocketAddress switchAddress;
IpSocketAddress speakerAddress;
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import org.openkilda.messaging.StringSerializer;
import org.openkilda.messaging.Utils;
import org.openkilda.model.IpSocketAddress;
import org.openkilda.model.Switch;
import org.openkilda.model.SwitchFeature;
import org.openkilda.model.SwitchId;
Expand All @@ -26,7 +27,6 @@
import org.junit.Test;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collections;

public class SwitchTest {
Expand All @@ -52,7 +52,7 @@ public void serializeLoop() throws Exception {
.pop("test_pop")
.status(SwitchStatus.ACTIVE)
.underMaintenance(true)
.socketAddress(new InetSocketAddress(1))
.socketAddress(new IpSocketAddress("192.168.10.20", 1))
.features(Collections.singleton(SwitchFeature.GROUP_PACKET_OUT_CONTROLLER))
.build();
serializer.serialize(origin);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ public static SwitchInfoData buildSwitchInfoData(IOFSwitch sw, SpeakerSwitchView
String.format("%s:%d",
address.getHostString(),
address.getPort()),
address.getHostName(),
String.format("%s %s %s",
sw.getSwitchDescription().getManufacturerDescription(),
sw.getOFFactory().getVersion().toString(),
Expand Down
Loading

0 comments on commit e166fcb

Please sign in to comment.