Skip to content

Commit

Permalink
Merge branch 'release-1.103.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
rozdy committed Aug 25, 2021
2 parents ee16400 + 1333a91 commit 7230abf
Show file tree
Hide file tree
Showing 99 changed files with 1,611 additions and 448 deletions.
1 change: 1 addition & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ S42C_MEM_LIMIT=1073741824
S42S_MEM_LIMIT=1073741824
S42SS_MEM_LIMIT=1073741824
S42STS_MEM_LIMIT=1073741824
MYSQL_MEM_LIMIT=256741824
30 changes: 30 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,35 @@
# Changelog

## v1.103.0 (20/08/2021)

### Features:
- [#4426](https://github.com/telstra/open-kilda/pull/4426) Bulk dump of switch properties [**northbound**][**storm-topologies**]
- [#4401](https://github.com/telstra/open-kilda/pull/4401) Add flow affinity groups [**northbound**][**storm-topologies**]

### Bug Fixes:
- [#4434](https://github.com/telstra/open-kilda/pull/4434) Fix TypeError in get_issues_from_pr in changelog-generator.py
- [#4438](https://github.com/telstra/open-kilda/pull/4438) Remove flow monitoring extra logging

### Improvements:
- [#4425](https://github.com/telstra/open-kilda/pull/4425) Improve mysql init script compatibility
- [#4435](https://github.com/telstra/open-kilda/pull/4435) refactor grpc spec [**tests**]
- [#4433](https://github.com/telstra/open-kilda/pull/4433) refactor s42Spec according to #4381 (Issue: [#4381](https://github.com/telstra/open-kilda/issues/4381)) [**tests**]
- [#4441](https://github.com/telstra/open-kilda/pull/4441) add missed "assumeTrue(useMultitable)" in FlowHistorySpec [**tests**]
- [#4412](https://github.com/telstra/open-kilda/pull/4412) refactor qinq/s42 specs according to #4408 (Issue: [#4408](https://github.com/telstra/open-kilda/issues/4408)) [**tests**]

### Other changes:
- [#4439](https://github.com/telstra/open-kilda/pull/4439) Add affinity flow spec [**tests**]
- [#4442](https://github.com/telstra/open-kilda/pull/4442) Fix IterationTags issue [**tests**]
- [#4445](https://github.com/telstra/open-kilda/pull/4445) Fix history calculation and cleanup in AutoRerouteSpec [**tests**]
- [#4446](https://github.com/telstra/open-kilda/pull/4446) Remove switch ids from test names, use hwSwString instead [**tests**]

For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.102.0...v1.103.0).

### Affected Components:
nbworker, orientdb, flow-monitor, reroute, flow-hs, nb

---

## v1.102.0 (13/08/2021)

### Features:
Expand Down
1 change: 1 addition & 0 deletions confd/templates/docker-compose/docker-compose.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ services:
default:
aliases:
- mysql.pendev
mem_limit: ${MYSQL_MEM_LIMIT:-256741824}

db_migration:
container_name: db_migration
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
databaseChangeLog:
- changeSet:
id: tag
author: dpoltavets
changes:
- tagDatabase:
tag: 012-add-diverse-and-affinity-group-id-to-flow-class

- changeSet:
id: add_diverse_and_affinity_group_id_to_flow_class
author: dpoltavets
changes:
- sql: "CREATE PROPERTY flow.diverse_group_id IF NOT EXISTS STRING"
- sql: "CREATE INDEX flow.diverse_group_id NOTUNIQUE_HASH_INDEX"
- sql: "CREATE PROPERTY flow.affinity_group_id IF NOT EXISTS STRING"
- sql: "CREATE INDEX flow.affinity_group_id NOTUNIQUE_HASH_INDEX"
- sql: "CREATE PROPERTY flow_dump.diverse_group_id IF NOT EXISTS STRING"
- sql: "CREATE PROPERTY flow_dump.affinity_group_id IF NOT EXISTS STRING"
rollback:
- sql: "DROP INDEX flow.diverse_group_id"
- sql: "UPDATE flow REMOVE diverse_group_id"
- sql: "DROP PROPERTY flow.diverse_group_id"
- sql: "DROP INDEX flow.affinity_group_id"
- sql: "UPDATE flow REMOVE affinity_group_id"
- sql: "DROP PROPERTY flow.affinity_group_id"
- sql: "UPDATE flow_dump REMOVE diverse_group_id"
- sql: "DROP PROPERTY flow_dump.diverse_group_id"
- sql: "UPDATE flow_dump REMOVE affinity_group_id"
- sql: "DROP PROPERTY flow_dump.affinity_group_id"
4 changes: 4 additions & 0 deletions docker/db-migration/migrations/root.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,7 @@ databaseChangeLog:
- include:
relativeToChangelogFile: true
file: 011-add-lag-classes.yaml
- include:
relativeToChangelogFile: true
file: 012-add-diverse-and-affinity-group-id-to-flow-class.yaml

88 changes: 44 additions & 44 deletions docker/mysql/setup_data/initial-schema.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,58 +2,58 @@

CREATE TABLE flow_event
(
id integer primary key auto_increment,
flow_id text not null,
task_id text not null,
task_id_unique_key char(100) not null,
action text not null,
unstructured text not null,
time_create datetime(3) not null default now(3),
time_modify datetime(3),
event_time datetime(3),
index `ix_event_time` (event_time),
index `ix_flow_id` (flow_id(64)),
unique `ix_task_id` (task_id_unique_key(100))
);
`id` integer primary key auto_increment,
`flow_id` text not null,
`task_id` text not null,
`task_id_unique_key` char(100) not null,
`action` text not null,
`unstructured` text not null,
`time_create` datetime(3) not null default now(3),
`time_modify` datetime(3),
`event_time` datetime(3),
index `ix_event_time` (`event_time`),
index `ix_flow_id` (`flow_id`(64)),
unique `ix_task_id` (`task_id_unique_key`(100))
) engine=innodb;

CREATE TABLE flow_event_action
(
id integer primary key auto_increment,
flow_event_id integer not null,
action text not null,
details text,
time_create datetime(3) not null default now(3),
time_modify datetime(3),
event_time datetime(3),
index `ix_flow_event_id` (flow_event_id),
foreign key `fk_flow_event_action_2_flow_event` (flow_event_id)
REFERENCES flow_event(id)
`id` integer primary key auto_increment,
`flow_event_id` integer not null,
`action` text not null,
`details` text,
`time_create` datetime(3) not null default now(3),
`time_modify` datetime(3),
`event_time` datetime(3),
index `ix_flow_event_id` (`flow_event_id`),
foreign key `fk_flow_event_action_2_flow_event` (`flow_event_id`)
REFERENCES flow_event (`id`)
ON UPDATE CASCADE ON DELETE RESTRICT
);
) engine=innodb;

CREATE TABLE flow_event_dump
(
id integer primary key auto_increment,
flow_event_id integer not null,
kind varchar(64) not null,
unstructured text not null,
time_create datetime(3) not null default now(3),
time_modify datetime(3),
index `ix_flow_event_id` (flow_event_id),
foreign key `fk_flow_event_dump_2_flow_event` (flow_event_id)
REFERENCES flow_event (id)
`id` integer primary key auto_increment,
`flow_event_id` integer not null,
`kind` varchar(64) not null,
`unstructured` text not null,
`time_create` datetime(3) not null default now(3),
`time_modify` datetime(3),
index `ix_flow_event_id` (`flow_event_id`),
foreign key `fk_flow_event_dump_2_flow_event` (`flow_event_id`)
REFERENCES flow_event (`id`)
ON UPDATE CASCADE ON DELETE RESTRICT
);
) engine = innodb;

CREATE TABLE port_event
(
id char(36) primary key, -- uuid
switch_id char(23) not null,
port_number integer not null,
event varchar(64) not null,
unstructured text not null,
time_create datetime(3) not null default now(3),
time_modify datetime(3),
event_time datetime(3),
index `ix_event_time` (event_time)
);
`id` char(36) primary key, -- uuid
`switch_id` char(23) not null,
`port_number` integer not null,
`event` varchar(64) not null,
`unstructured` text not null,
`time_create` datetime(3) not null default now(3),
`time_modify` datetime(3),
`event_time` datetime(3),
index `ix_event_time` (`event_time`)
) engine = innodb;
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
@startuml
title Affinity flows

participant NB
participant FlowHSTopology
participant PathComputer
participant AvailableNetwork
database OrientDB

activate FlowHSTopology
NB -> FlowHSTopology : create affinity flow

FlowHSTopology -> OrientDB : get affinity flow by id
OrientDB --> FlowHSTopology : Flow instance
alt flow affinity group property is not defined
FlowHSTopology -> OrientDB : save affinity flow id as affinity group id
end

FlowHSTopology -> PathComputer : get path with flow affinity group id
activate PathComputer

PathComputer -> AvailableNetwork : getAvailableNetwork\nwith flow affinity group id
activate AvailableNetwork

group Build AvailableNetwork
AvailableNetwork -> OrientDB: get active links
OrientDB--> AvailableNetwork
AvailableNetwork -> AvailableNetwork : create network from retrieved links

group Fill network affinity weights
AvailableNetwork -> OrientDB : get flow segments for main flow in the affinity group
OrientDB--> AvailableNetwork : path segments

AvailableNetwork -> AvailableNetwork: fill affinity weights using\naffinity flow segments
end
AvailableNetwork --> PathComputer : AvailableNetwork
deactivate AvailableNetwork
end

group Find path
PathComputer -> PathComputer : PathFinder.findPathInNetwork
PathComputer --> FlowHSTopology : computed path
deactivate PathComputer
end

alt path found
FlowHSTopology -> OrientDB : create new flow with affinity group id

FlowHSTopology --> NB : flow create operation response

else path not found
FlowHSTopology --> NB: creation error
end

deactivate FlowHSTopology

@enduml
43 changes: 43 additions & 0 deletions docs/design/solutions/pce-affinity-flows/pce-affinity-flows.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Affinity flows

## Goals
The main goal is to be able to search for paths that maximally repeat the path of a certain main flow.

## NB contract changes
Create\Update Flow Request: add optional flow id, to make flow affinity with.

Flow Response: If flow is in an affinity group, flow response will contain an `affinity_with` field
with a main flow id of that group.

## DB changes
Flow relation keeps flow affinity group id as a property.

## Path computation algorithm changes
In PCE flow affinity working similar to diversity groups. During available network computation weight for all edges not used by affinity flow paths is increased by the configurable parameter value. These penalties should make calculated paths to prefer reuse as many links as possible. In other words if we need to compare two possible paths: first step is to compare number of links unused by affinity flow and second step is compare weights by path computation strategy (cost, latency etc).

### Use cases
1. Affinity flow has path 1-2-3. We need to build path from node 1 to 4. There are three available paths: 1-4, 1-2-4, 1-2-3-4. Since they all have only one link unused by main path they will have the same affinity penalty and the path with minimal total weight will be chosen. If we have equal initial weights for all these links we will use path 1-4 because it's total weight is minimal.

![Use case 1](use-case-1.png)

2. Affinity flow has path 1-2-3-4. We need to build path from node 1 to 7. A lot of paths are available but only two of them has one link unused by main path: 1-2-3-7 and 1-2-3-4-7. All other paths will have more links unused by main path and thus will have greater affinity penalty. If we have equal initial weights for all these links we will use path 1-2-3-7 because it's total weight is minimal.

![Use case 2](use-case-2.png)

3. Affinity flow has path 1-2-5. We need to build path from node 3 to 6. Two paths are available 3-2-5-6 and 3-4-7-6. First path reuses one link from main flow so has lesser affinity penalty and should be used.

![Use case 3](use-case-3.png)

## Limitations
- Flow can only belong to one flow affinity group.

- Flow affinity groups are an implementation detail and there is no API to access it directly.

- In the current implementation, affinity and diverse groups can not be specified at the same time for one flow.
([Diverse flows](../pce-diverse-flows/pce-diverse-flows.md))

- In current implementation affinity groups doesn't support protected flows because protected flows implemented using diversity feature.

## Sequence Diagram
![Create affinity flows](pce-affinity-flows-create.png)

Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions docs/design/solutions/pce-diverse-flows/pce-diverse-flows.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ Flow groups is an implementation detail and there is no API to access it directl

System knows nothing about physical network topology, so computed paths not truly diverse in hardware meaning.

In the current implementation, affinity and diverse groups can not be specified at the same time for one flow.
([Affinity flows](../pce-affinity-flows/pce-affinity-flows.md))

## Sequence Diagram
![Ability to create diverse flows](pce-diverse-flows-create.png)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2019 Telstra Open Source
/* Copyright 2021 Telstra Open Source
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -187,6 +187,9 @@ public class FlowDto implements Serializable {
@JsonProperty("diverse_with")
private Set<String> diverseWith;

@JsonProperty("affinity_with")
private String affinityWith;

@JsonProperty("loop_switch_id")
private SwitchId loopSwitchId;

Expand Down Expand Up @@ -234,6 +237,7 @@ public FlowDto() {
* @param pathComputationStrategy path computation strategy
* @param targetPathComputationStrategy target path computation strategy
* @param diverseWith flow ids diverse with
* @param affinityWith flow id affinity with
* @param loopSwitchId loop switch id
* @param mirrorPointStatuses mirror path statuses
* @param forwardLatency forward path latency nanoseconds
Expand Down Expand Up @@ -274,6 +278,7 @@ public FlowDto(@JsonProperty(Utils.FLOW_ID) final String flowId,
@JsonProperty("target_path_computation_strategy")
PathComputationStrategy targetPathComputationStrategy,
@JsonProperty("diverse_with") Set<String> diverseWith,
@JsonProperty("affinity_with") String affinityWith,
@JsonProperty("loop_switch_id") SwitchId loopSwitchId,
@JsonProperty("mirror_point_statuses") List<MirrorPointStatusDto> mirrorPointStatuses,
@JsonProperty("forward_latency") long forwardLatency,
Expand Down Expand Up @@ -310,6 +315,7 @@ public FlowDto(@JsonProperty(Utils.FLOW_ID) final String flowId,
this.pathComputationStrategy = pathComputationStrategy;
this.targetPathComputationStrategy = targetPathComputationStrategy;
this.diverseWith = diverseWith;
this.affinityWith = affinityWith;
this.loopSwitchId = loopSwitchId;
this.mirrorPointStatuses = mirrorPointStatuses;
this.forwardLatency = forwardLatency;
Expand Down Expand Up @@ -355,7 +361,7 @@ public FlowDto(String flowId,
sourceVlan,
destinationVlan, 0, 0,
null, 0, null, null, null, null, null, null, pinned, null, detectConnectedDevices, null, null, null,
null, null, 0, 0);
null, null, null, 0, 0);
}

public FlowDto(FlowPayload input) {
Expand Down Expand Up @@ -390,7 +396,7 @@ public FlowDto(FlowPayload input) {
input.getDestination().getDetectConnectedDevices().isArp()),
input.getPathComputationStrategy() != null ? PathComputationStrategy.valueOf(
input.getPathComputationStrategy().toUpperCase()) : null, null, null,
null, null, 0L, 0L);
null, null, null, 0L, 0L);
}

@JsonIgnore
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2020 Telstra Open Source
/* Copyright 2021 Telstra Open Source
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -39,6 +39,7 @@ public class FlowPatch {
private PathComputationStrategy pathComputationStrategy;
private PathComputationStrategy targetPathComputationStrategy;
private String diverseFlowId;
private String affinityFlowId;
private Boolean pinned;
private Boolean ignoreBandwidth;
private Boolean strictBandwidth;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import org.openkilda.messaging.payload.flow.FlowEncapsulationType;
import org.openkilda.model.MacAddress;
import org.openkilda.model.SwitchId;

import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
Expand All @@ -26,6 +27,9 @@

@Data
public class SwitchPropertiesDto implements Serializable {
@JsonProperty("switch_id")
private SwitchId switchId;

@JsonProperty("supported_transit_encapsulation")
private Set<FlowEncapsulationType> supportedTransitEncapsulation;

Expand Down
Loading

0 comments on commit 7230abf

Please sign in to comment.