Merge branch 'release-1.103.0'

.env.example

@@ -19,3 +19,4 @@ S42C_MEM_LIMIT=1073741824
 S42S_MEM_LIMIT=1073741824
 S42SS_MEM_LIMIT=1073741824
 S42STS_MEM_LIMIT=1073741824
+MYSQL_MEM_LIMIT=256741824

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## v1.103.0 (20/08/2021)
+
+### Features:
+-  [#4426](https://github.com/telstra/open-kilda/pull/4426) Bulk dump of switch properties [**northbound**][**storm-topologies**]
+-  [#4401](https://github.com/telstra/open-kilda/pull/4401) Add flow affinity groups [**northbound**][**storm-topologies**]
+
+### Bug Fixes:
+-  [#4434](https://github.com/telstra/open-kilda/pull/4434) Fix TypeError in get_issues_from_pr in changelog-generator.py 
+-  [#4438](https://github.com/telstra/open-kilda/pull/4438) Remove flow monitoring extra logging 
+
+### Improvements:
+-  [#4425](https://github.com/telstra/open-kilda/pull/4425) Improve mysql init script compatibility 
+-  [#4435](https://github.com/telstra/open-kilda/pull/4435) refactor grpc spec [**tests**]
+-  [#4433](https://github.com/telstra/open-kilda/pull/4433) refactor s42Spec according to #4381 (Issue: [#4381](https://github.com/telstra/open-kilda/issues/4381)) [**tests**]
+-  [#4441](https://github.com/telstra/open-kilda/pull/4441) add missed "assumeTrue(useMultitable)" in FlowHistorySpec [**tests**]
+-  [#4412](https://github.com/telstra/open-kilda/pull/4412) refactor qinq/s42 specs according to #4408 (Issue: [#4408](https://github.com/telstra/open-kilda/issues/4408)) [**tests**]
+
+### Other changes:
+-  [#4439](https://github.com/telstra/open-kilda/pull/4439) Add affinity flow spec [**tests**]
+-  [#4442](https://github.com/telstra/open-kilda/pull/4442) Fix IterationTags issue [**tests**]
+-  [#4445](https://github.com/telstra/open-kilda/pull/4445) Fix history calculation and cleanup in AutoRerouteSpec [**tests**]
+-  [#4446](https://github.com/telstra/open-kilda/pull/4446) Remove switch ids from test names, use hwSwString instead [**tests**]
+
+For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.102.0...v1.103.0).
+
+### Affected Components:
+nbworker, orientdb, flow-monitor, reroute, flow-hs, nb
+
+---
+
 ## v1.102.0 (13/08/2021)
 
 ### Features:

confd/templates/docker-compose/docker-compose.tmpl

@@ -141,6 +141,7 @@ services:
       default:
         aliases:
           - mysql.pendev
+    mem_limit: ${MYSQL_MEM_LIMIT:-256741824}
 
   db_migration:
     container_name: db_migration

docker/db-migration/migrations/012-add-diverse-and-affinity-group-id-to-flow-class.yaml

@@ -0,0 +1,29 @@
+databaseChangeLog:
+  - changeSet:
+      id: tag
+      author: dpoltavets
+      changes:
+        - tagDatabase:
+            tag: 012-add-diverse-and-affinity-group-id-to-flow-class
+
+  - changeSet:
+      id: add_diverse_and_affinity_group_id_to_flow_class
+      author: dpoltavets
+      changes:
+        - sql: "CREATE PROPERTY flow.diverse_group_id IF NOT EXISTS STRING"
+        - sql: "CREATE INDEX flow.diverse_group_id NOTUNIQUE_HASH_INDEX"
+        - sql: "CREATE PROPERTY flow.affinity_group_id IF NOT EXISTS STRING"
+        - sql: "CREATE INDEX flow.affinity_group_id NOTUNIQUE_HASH_INDEX"
+        - sql: "CREATE PROPERTY flow_dump.diverse_group_id IF NOT EXISTS STRING"
+        - sql: "CREATE PROPERTY flow_dump.affinity_group_id IF NOT EXISTS STRING"
+      rollback:
+        - sql: "DROP INDEX flow.diverse_group_id"
+        - sql: "UPDATE flow REMOVE diverse_group_id"
+        - sql: "DROP PROPERTY flow.diverse_group_id"
+        - sql: "DROP INDEX flow.affinity_group_id"
+        - sql: "UPDATE flow REMOVE affinity_group_id"
+        - sql: "DROP PROPERTY flow.affinity_group_id"
+        - sql: "UPDATE flow_dump REMOVE diverse_group_id"
+        - sql: "DROP PROPERTY flow_dump.diverse_group_id"
+        - sql: "UPDATE flow_dump REMOVE affinity_group_id"
+        - sql: "DROP PROPERTY flow_dump.affinity_group_id"

docker/db-migration/migrations/root.yaml

@@ -45,3 +45,7 @@ databaseChangeLog:
   - include:
       relativeToChangelogFile: true
       file: 011-add-lag-classes.yaml
+  - include:
+      relativeToChangelogFile: true
+      file: 012-add-diverse-and-affinity-group-id-to-flow-class.yaml
+

docker/mysql/setup_data/initial-schema.sql

@@ -2,58 +2,58 @@
 
 CREATE TABLE flow_event
 (
-    id           integer primary key auto_increment,
-    flow_id      text     not null,
-    task_id      text     not null,
-    task_id_unique_key char(100) not null,
-    action       text     not null,
-    unstructured text     not null,
-    time_create  datetime(3) not null default now(3),
-    time_modify  datetime(3),
-    event_time   datetime(3),
-    index        `ix_event_time` (event_time),
-    index        `ix_flow_id` (flow_id(64)),
-    unique       `ix_task_id` (task_id_unique_key(100))
-);
+    `id`                 integer primary key auto_increment,
+    `flow_id`            text        not null,
+    `task_id`            text        not null,
+    `task_id_unique_key` char(100)   not null,
+    `action`             text        not null,
+    `unstructured`       text        not null,
+    `time_create`        datetime(3) not null default now(3),
+    `time_modify`        datetime(3),
+    `event_time`         datetime(3),
+    index `ix_event_time` (`event_time`),
+    index `ix_flow_id` (`flow_id`(64)),
+    unique `ix_task_id` (`task_id_unique_key`(100))
+) engine=innodb;
 
 CREATE TABLE flow_event_action
 (
-    id            integer primary key auto_increment,
-    flow_event_id integer  not null,
-    action        text     not null,
-    details       text,
-    time_create   datetime(3) not null default now(3),
-    time_modify   datetime(3),
-    event_time    datetime(3),
-    index         `ix_flow_event_id` (flow_event_id),
-    foreign key `fk_flow_event_action_2_flow_event` (flow_event_id)
-        REFERENCES flow_event(id)
+    `id`            integer primary key auto_increment,
+    `flow_event_id` integer     not null,
+    `action`        text        not null,
+    `details`       text,
+    `time_create`   datetime(3) not null default now(3),
+    `time_modify`   datetime(3),
+    `event_time`    datetime(3),
+    index `ix_flow_event_id` (`flow_event_id`),
+    foreign key `fk_flow_event_action_2_flow_event` (`flow_event_id`)
+        REFERENCES flow_event (`id`)
         ON UPDATE CASCADE ON DELETE RESTRICT
-);
+) engine=innodb;
 
 CREATE TABLE flow_event_dump
 (
-    id            integer primary key auto_increment,
-    flow_event_id integer     not null,
-    kind          varchar(64) not null,
-    unstructured  text        not null,
-    time_create   datetime(3)    not null default now(3),
-    time_modify   datetime(3),
-    index         `ix_flow_event_id` (flow_event_id),
-    foreign key `fk_flow_event_dump_2_flow_event` (flow_event_id)
-        REFERENCES flow_event (id)
+    `id`            integer primary key auto_increment,
+    `flow_event_id` integer     not null,
+    `kind`          varchar(64) not null,
+    `unstructured`  text        not null,
+    `time_create`   datetime(3) not null default now(3),
+    `time_modify`   datetime(3),
+    index `ix_flow_event_id` (`flow_event_id`),
+    foreign key `fk_flow_event_dump_2_flow_event` (`flow_event_id`)
+        REFERENCES flow_event (`id`)
         ON UPDATE CASCADE ON DELETE RESTRICT
-);
+) engine = innodb;
 
 CREATE TABLE port_event
 (
-    id           char(36) primary key, -- uuid
-    switch_id    char(23)    not null,
-    port_number  integer     not null,
-    event        varchar(64) not null,
-    unstructured text        not null,
-    time_create  datetime(3)    not null default now(3),
-    time_modify  datetime(3),
-    event_time   datetime(3),
-    index        `ix_event_time` (event_time)
-);
+    `id`           char(36) primary key, -- uuid
+    `switch_id`    char(23)    not null,
+    `port_number`  integer     not null,
+    `event`        varchar(64) not null,
+    `unstructured` text        not null,
+    `time_create`  datetime(3) not null default now(3),
+    `time_modify`  datetime(3),
+    `event_time`   datetime(3),
+    index `ix_event_time` (`event_time`)
+) engine = innodb;

docs/design/solutions/pce-affinity-flows/pce-affinity-flows-create.puml

@@ -0,0 +1,57 @@
+@startuml
+title Affinity flows
+
+participant NB
+participant FlowHSTopology
+participant PathComputer
+participant AvailableNetwork
+database OrientDB
+
+activate FlowHSTopology
+NB -> FlowHSTopology : create affinity flow
+
+FlowHSTopology -> OrientDB : get affinity flow by id
+OrientDB --> FlowHSTopology : Flow instance
+alt flow affinity group property is not defined
+   FlowHSTopology -> OrientDB : save affinity flow id as affinity group id
+end
+
+FlowHSTopology -> PathComputer : get path with flow affinity group id
+activate PathComputer
+
+PathComputer -> AvailableNetwork : getAvailableNetwork\nwith flow affinity group id
+activate AvailableNetwork
+
+group Build AvailableNetwork
+    AvailableNetwork -> OrientDB: get active links
+    OrientDB--> AvailableNetwork
+    AvailableNetwork -> AvailableNetwork : create network from retrieved links
+
+    group Fill network affinity weights
+        AvailableNetwork -> OrientDB : get flow segments for main flow in the affinity group
+        OrientDB--> AvailableNetwork : path segments
+
+        AvailableNetwork -> AvailableNetwork: fill affinity weights using\naffinity flow segments
+    end
+    AvailableNetwork --> PathComputer : AvailableNetwork
+    deactivate AvailableNetwork
+end
+
+group Find path
+    PathComputer -> PathComputer : PathFinder.findPathInNetwork
+    PathComputer --> FlowHSTopology : computed path
+    deactivate PathComputer
+end
+
+alt path found
+    FlowHSTopology -> OrientDB : create new flow with affinity group id
+
+    FlowHSTopology --> NB : flow create operation response
+
+else path not found
+   FlowHSTopology --> NB: creation error
+end
+
+deactivate FlowHSTopology
+
+@enduml

docs/design/solutions/pce-affinity-flows/pce-affinity-flows.md

@@ -0,0 +1,43 @@
+# Affinity flows
+
+## Goals
+The main goal is to be able to search for paths that maximally repeat the path of a certain main flow.
+
+## NB contract changes
+Create\Update Flow Request: add optional flow id, to make flow affinity with.
+
+Flow Response: If flow is in an affinity group, flow response will contain an `affinity_with` field 
+with a main flow id of that group.
+
+## DB changes
+Flow relation keeps flow affinity group id as a property.
+
+## Path computation algorithm changes
+In PCE flow affinity working similar to diversity groups. During available network computation weight for all edges not used by affinity flow paths is increased by the configurable parameter value. These penalties should make calculated paths to prefer reuse as many links as possible. In other words if we need to compare two possible paths: first step is to compare number of links unused by affinity flow and second step is compare weights by path computation strategy (cost, latency etc).  
+
+### Use cases
+1. Affinity flow has path 1-2-3. We need to build path from node 1 to 4. There are three available paths: 1-4, 1-2-4, 1-2-3-4. Since they all have only one link unused by main path they will have the same affinity penalty and the path with minimal total weight will be chosen. If we have equal initial weights for all these links we will use path 1-4 because it's total weight is minimal.
+
+![Use case 1](use-case-1.png)
+
+2. Affinity flow has path 1-2-3-4. We need to build path from node 1 to 7. A lot of paths are available but only two of them has one link unused by main path: 1-2-3-7 and 1-2-3-4-7. All other paths will have more links unused by main path and thus will have greater affinity penalty. If we have equal initial weights for all these links we will use path 1-2-3-7 because it's total weight is minimal.     
+
+![Use case 2](use-case-2.png)
+
+3. Affinity flow has path 1-2-5. We need to build path from node 3 to 6. Two paths are available 3-2-5-6 and 3-4-7-6. First path reuses one link from main flow so has lesser affinity penalty and should be used.
+
+![Use case 3](use-case-3.png)
+
+## Limitations
+ - Flow can only belong to one flow affinity group.
+
+ - Flow affinity groups are an implementation detail and there is no API to access it directly.
+
+ - In the current implementation, affinity and diverse groups can not be specified at the same time for one flow.
+   ([Diverse flows](../pce-diverse-flows/pce-diverse-flows.md))
+
+ - In current implementation affinity groups doesn't support protected flows because protected flows implemented using diversity feature.
+
+## Sequence Diagram
+![Create affinity flows](pce-affinity-flows-create.png)
+

docs/design/solutions/pce-diverse-flows/pce-diverse-flows.md

@@ -34,6 +34,9 @@ Flow groups is an implementation detail and there is no API to access it directl
 
 System knows nothing about physical network topology, so computed paths not truly diverse in hardware meaning.
 
+In the current implementation, affinity and diverse groups can not be specified at the same time for one flow.
+([Affinity flows](../pce-affinity-flows/pce-affinity-flows.md))
+
 ## Sequence Diagram
 ![Ability to create diverse flows](pce-diverse-flows-create.png)
 

src-java/base-topology/base-messaging/src/main/java/org/openkilda/messaging/model/FlowDto.java

@@ -1,4 +1,4 @@
-/* Copyright 2019 Telstra Open Source
+/* Copyright 2021 Telstra Open Source
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -187,6 +187,9 @@ public class FlowDto implements Serializable {
     @JsonProperty("diverse_with")
     private Set<String> diverseWith;
 
+    @JsonProperty("affinity_with")
+    private String affinityWith;
+
     @JsonProperty("loop_switch_id")
     private SwitchId loopSwitchId;
 
@@ -234,6 +237,7 @@ public FlowDto() {
      * @param pathComputationStrategy   path computation strategy
      * @param targetPathComputationStrategy   target path computation strategy
      * @param diverseWith               flow ids diverse with
+     * @param affinityWith              flow id affinity with
      * @param loopSwitchId              loop switch id
      * @param mirrorPointStatuses       mirror path statuses
      * @param forwardLatency            forward path latency nanoseconds
@@ -274,6 +278,7 @@ public FlowDto(@JsonProperty(Utils.FLOW_ID) final String flowId,
                    @JsonProperty("target_path_computation_strategy")
                                PathComputationStrategy targetPathComputationStrategy,
                    @JsonProperty("diverse_with") Set<String> diverseWith,
+                   @JsonProperty("affinity_with") String affinityWith,
                    @JsonProperty("loop_switch_id") SwitchId loopSwitchId,
                    @JsonProperty("mirror_point_statuses") List<MirrorPointStatusDto> mirrorPointStatuses,
                    @JsonProperty("forward_latency") long forwardLatency,
@@ -310,6 +315,7 @@ public FlowDto(@JsonProperty(Utils.FLOW_ID) final String flowId,
         this.pathComputationStrategy = pathComputationStrategy;
         this.targetPathComputationStrategy = targetPathComputationStrategy;
         this.diverseWith = diverseWith;
+        this.affinityWith = affinityWith;
         this.loopSwitchId = loopSwitchId;
         this.mirrorPointStatuses = mirrorPointStatuses;
         this.forwardLatency = forwardLatency;
@@ -355,7 +361,7 @@ public FlowDto(String flowId,
                 sourceVlan,
                 destinationVlan, 0, 0,
                 null, 0, null, null, null, null, null, null, pinned, null, detectConnectedDevices, null, null, null,
-                null, null, 0, 0);
+                null, null, null, 0, 0);
     }
 
     public FlowDto(FlowPayload input) {
@@ -390,7 +396,7 @@ public FlowDto(FlowPayload input) {
                         input.getDestination().getDetectConnectedDevices().isArp()),
                 input.getPathComputationStrategy() != null ? PathComputationStrategy.valueOf(
                         input.getPathComputationStrategy().toUpperCase()) : null, null, null,
-                null, null, 0L, 0L);
+                null, null, null, 0L, 0L);
     }
 
     @JsonIgnore

src-java/base-topology/base-messaging/src/main/java/org/openkilda/messaging/model/FlowPatch.java

@@ -1,4 +1,4 @@
-/* Copyright 2020 Telstra Open Source
+/* Copyright 2021 Telstra Open Source
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -39,6 +39,7 @@ public class FlowPatch {
     private PathComputationStrategy pathComputationStrategy;
     private PathComputationStrategy targetPathComputationStrategy;
     private String diverseFlowId;
+    private String affinityFlowId;
     private Boolean pinned;
     private Boolean ignoreBandwidth;
     private Boolean strictBandwidth;

src-java/base-topology/base-messaging/src/main/java/org/openkilda/messaging/model/SwitchPropertiesDto.java

@@ -17,6 +17,7 @@
 
 import org.openkilda.messaging.payload.flow.FlowEncapsulationType;
 import org.openkilda.model.MacAddress;
+import org.openkilda.model.SwitchId;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Data;
@@ -26,6 +27,9 @@
 
 @Data
 public class SwitchPropertiesDto implements Serializable {
+    @JsonProperty("switch_id")
+    private SwitchId switchId;
+
     @JsonProperty("supported_transit_encapsulation")
     private Set<FlowEncapsulationType> supportedTransitEncapsulation;