Merge branch 'release-1.121.0'

CHANGELOG.md

@@ -1,7 +1,28 @@
 # Changelog
 
-## v1.120.2 (10/06/2022)
+## v1.121.0 (22/06/2022)
+
+### Bug Fixes:
+-  [#4846](https://github.com/telstra/open-kilda/pull/4846) Fix y-flow sync after flow sync reimplementation [**storm-topologies**]
+-  [#4831](https://github.com/telstra/open-kilda/pull/4831) Fix validation and sync for one-switch y-flow (Issues: [#4824](https://github.com/telstra/open-kilda/issues/4824) [#4825](https://github.com/telstra/open-kilda/issues/4825))
+
+### Improvements:
+-  [#4840](https://github.com/telstra/open-kilda/pull/4840) Return diversity info in get paths response [**northbound**][**storm-topologies**]
+-  [#4847](https://github.com/telstra/open-kilda/pull/4847) Set Stats topology parallelism to 1 for local env (Issue: [#4844](https://github.com/telstra/open-kilda/issues/4844)) [**storm-topologies**]
+
+### Other changes:
+-  [#4817](https://github.com/telstra/open-kilda/pull/4817) True flow sync [**storm-topologies**]
+-  [#4796](https://github.com/telstra/open-kilda/pull/4796) Unmark DRAFT from y-flow API [**northbound**]
+-  [#4830](https://github.com/telstra/open-kilda/pull/4830) Update rule manager docs [**docs**]
 
+For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.120.2...v1.121.0).
+
+### Affected Components:
+nbworker, stats, nb, flow-hs
+
+---
+
+## v1.120.2 (10/06/2022)
 
 ### Bug Fixes:
 -  [#4839](https://github.com/telstra/open-kilda/pull/4839) Fix switch sync to be able to repair misconfigured rules with the same cookie (Issue: [#4838](https://github.com/telstra/open-kilda/issues/4838)) [**floodlight**]
@@ -28,6 +49,8 @@ flow-hs, fl, stats, swmanager, nb, network
 
 OrientDB schema have been changed in this release. You need to apply schema migration. Please follow [migration instructions](https://github.com/telstra/open-kilda/tree/develop/docker/db-migration/migrations).
 
+---
+
 ## v1.120.1 (01/06/2022)
 
 ### Bug Fixes:

confd/vars/main.yaml

@@ -172,7 +172,7 @@ kilda_storm_flow_hs_reroute_hub_count_multiplier: 2
 kilda_storm_flowhs_workers: 1
 kilda_storm_parallelism_workers_count: 1
 kilda_storm_history_parallelism: 2
-kilda_storm_stats_parallelism: 2
+kilda_storm_stats_parallelism: 1 # must be 1 until bug https://github.com/telstra/open-kilda/issues/4844 will be fixed
 
 kilda_storm_spout_parallelism: 2
 

docs/design/hub-and-spoke/crud/path/flow-path-delete-fsm.puml

@@ -0,0 +1,38 @@
+@startuml
+title Flow path delete FSM state diagram
+
+[*] --> DELETE
+
+DELETE: enter / activate next segments chunk
+DELETE: enter / fire "no_more_chunks" if there is no more chunks
+DELETE: enter / emit DELETE request for each segment in active chunk
+DELETE: enter / fire "chunk_complete" if there is no more pending requests
+DELETE: enter / fire "next"
+DELETE --> WAIT_PENDING: next
+DELETE --> REVERT: error / set result_code into SPEAKER_ERROR
+DELETE --> REVERT_WAIT_PENDING: cancel / set result_code into CANCEL
+DELETE --> END: (chunk_complete | no_more_chunks)
+
+WAIT_PENDING --> REVERT: error / set result_code into SPEAKER_ERROR
+WAIT_PENDING --> REVERT_WAIT_PENDING: cancel / set result_code into CANCEL
+WAIT_PENDING --> DELETE: chunk_complete
+WAIT_PENDING: speaker_response / handle speaker response
+WAIT_PENDING: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+WAIT_PENDING: speaker_response / [no more pending request && error responses are present] fire "error"
+
+REVERT_WAIT_PENDING --> REVERT: (error | chunk_complete)
+REVERT_WAIT_PENDING: speaker_response / handle speaker response
+REVERT_WAIT_PENDING: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+REVERT_WAIT_PENDING: speaker_response / [no more pending request && error responses are present] fire "error"
+
+REVERT: enter / [! is revert enabled] fire "chunk_complete"
+REVERT: enter / assemble and activate revert chunk
+REVERT: enter / emit INSTALL request for each segment in active chunk
+REVERT --> END: (chunk_complete | error)
+REVERT: speaker_response / handle speaker response
+REVERT: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+REVERT: speaker_response / [no more pending request && error responses are present] fire "error"
+
+END: enter / [result code is null] set result_code into success
+END --> [*]
+@enduml

docs/design/hub-and-spoke/crud/path/flow-patth-install-fsm.puml

@@ -0,0 +1,41 @@
+@startuml
+title Flow path install FSM state diagram
+
+[*] --> INSTALL
+
+INSTALL: enter / activate next segments chunk
+INSTALL: enter / fire "no_more_chunks" if there is no more chunks
+INSTALL: enter / emit INSTALL request for each segment in active chunk
+INSTALL: enter / fire "chunk_complete" if there is no more pending requests
+INSTALL --> REVERT: error / set result_code into SPEAKER_ERROR
+INSTALL --> REVERT_WAIT_PENDING: cancel / set result_code into CANCEL
+INSTALL --> VERIFY: chunk_complete
+INSTALL --> END: no_more_chunks
+INSTALL: speaker_response / handle speaker response
+INSTALL: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+INSTALL: speaker_response / [no more pending request && error responses are present] fire "error"
+
+VERIFY: enter / emit VERIFY request for each segment in active chunk
+VERIFY --> REVERT: error / set result_code into SPEAKER_ERROR
+VERIFY --> REVERT_WAIT_PENDING: cancel / set result_code into CANCEL
+VERIFY --> INSTALL: chunk_complete
+VERIFY: speaker_response / handle speaker response
+VERIFY: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+VERIFY: speaker_response / [no more pending request && error responses are present] fire "error"
+
+REVERT_WAIT_PENDING --> END: (chunk_complete | error)
+REVERT_WAIT_PENDING: speaker_response / handle speaker response
+REVERT_WAIT_PENDING: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+REVERT_WAIT_PENDING: speaker_response / [no more pending request && error responses are present] fire "error"
+
+REVERT: enter / [! is revert enabled] fire "chunk_complete"
+REVERT: enter / assemble and activate revert chunk
+REVERT: enter / emit DELETE request for each segment in active chunk
+REVERT --> END: (chunk_complete | error)
+REVERT: speaker_response / handle speaker response
+REVERT: speaker_response / [no more pending request && no error responses received] fire "chunk_complete"
+REVERT: speaker_response / [no more pending request && error responses are present] fire "error"
+
+END: enter / [result code is null] set result_code into success
+END --> [*]
+@enduml

docs/design/hub-and-spoke/crud/sync/flow-sync-fsm.puml

@@ -0,0 +1,46 @@
+@startuml
+title Flow sync FSM state diagram
+
+[*] -> SETUP
+SETUP: enter / fire "error" if flow is IN_PROGRESS status
+SETUP: enter / set flow status to IN_PROGRESS
+SETUP: enter / apply postponed flow changes
+SETUP --> SYNC: next
+SETUP --> COMMIT_ERROR: error
+
+SYNC: enter / launch main path install operation
+SYNC: enter / [have protected path] launch protected path install operation
+SYNC --> SYNC_FAIL: sync_fail
+SYNC --> COMMIT_SUCCESS: guard_passed
+SYNC --> CANCEL: error
+SYNC --> CANCEL: timeout / report global timeout
+SYNC: path_operation_response / handle path operation response
+SYNC: path_operation_response / fire "sync_fail" on failed path response
+SYNC: path_operation_response / fire "guard_passed" if no more pending operations
+
+CANCEL: enter / request cancel path operation for all pending operations
+CANCEL --> COMMIT_ERROR: guard_passed
+CANCEL: path_operation_response / handle path operation response
+CANCEL: path_operation_response / fire "guard_passed" if no more pending operations
+CANCEL: error / fire "guard_passed" if no pending operations
+CANCEL: timeout / report global timeout
+
+SYNC_FAIL: enter / fire "guard_passed" if no pending operations
+SYNC_FAIL --> COMMIT_ERROR: guard_passed
+SYNC_FAIL --> CANCEL: error
+SYNC_FAIL --> CANCEL: timeout / report global timeout
+SYNC_FAIL: path_operation_response / handle path operation response
+SYNC_FAIL: path_operation_response / fire "guard_passed" if no more pending operations
+
+COMMIT_SUCCESS: enter / update flow status
+COMMIT_SUCCESS: enter / send success response
+COMMIT_SUCCESS --> FINISHED: next
+COMMIT_SUCCESS --> COMMIT_ERROR: error
+
+COMMIT_ERROR: enter / update flow status
+COMMIT_ERROR: enter / send failed response
+COMMIT_ERROR --> FINISHED_WITH_ERROR: next
+
+FINISHED --> [*]
+FINISHED_WITH_ERROR --> [*]
+@enduml

docs/design/rule-manager/implementation-details.md

@@ -1,13 +1,19 @@
 # Implementation details for RuleManager library
 
 Facade of RuleManager lib is implemented as an interface with two methods:
-1. buildRulesForPaths(SwitchId switchId, DataAdapter adapter)
-2. buildRulesForSwitch(PathId flowPathId, DataAdapter adapter)
+1. buildRulesForSwitch(SwitchId switchId, DataAdapter adapter)
+2. buildRulesForPath(FlowPath flowPath, boolean filterOutUsedSharedRules, DataAdapter adapter)
+3. buildRulesForYFlow(List<FlowPath> flowPaths, DataAdapter adapter)
+4. buildIslServiceRules(SwitchId switchId, int port, DataAdapter adapter)
 
 ![Class diagram](class-diagram.png "class diagram")
 
 Different adapter implementations may query database or read a file under the hood. Each operation should use separate adapter because they may cache data for better performance. All required parameters are provided as constructor params and filled by the calling component.
 
 To generate service rules generators hierarchy (similar to current SwitchManager implementation) is used but the logic is moved from floodlight to RuleManager lib and result is in custom Command format. To generate flow related rules similar way is introduced. Custom Command format is close to OpenFlow format to simplify translation in speaker.
 
-From floodlight perspective 3 new SpeakerCommands added: InstallCommand, VerifyCommand and RemoveCommand. Each command contains only one Command. Commands should be sent to speaker one by one or using a batch grouped by target switch. Floodlight should translate custom Commands into loxigen OpenFlow representation, send it to devices and process results. RuleManager returns collection of Commands with dependencies and the calling component (Flow-HS or SwitchManager) is responsible for encapsulating and sending this commands into floodlight in right order and correct batch options.
+From floodlight perspective 3 new SpeakerCommands added: InstallSpeakerCommandsRequest, ModifySpeakerCommandsRequest and DeleteSpeakerCommandsRequest. Each request contains a collection of Commands of the same type (install/modify/delete). Commands inside collection may have dependepcies on each other. Requests should be sent to speaker one by one. Floodlight should build dependency tree, check it for errors (missing dependency, cycles etc), split collection to execution stages according to dependency, translate custom Commands into loxigen OpenFlow representation, send it to devices and process results. 
+
+RuleManager returns collection of Commands and the calling component (Flow-HS/SwitchManager/Network) is responsible for encapsulating and sending this commands into floodlight in right order and correct batch options.
+
+![Install/delete service isl rules sequence diagram](isl-service-rules.png "Install/delete service isl rules sequence diagram")

docs/design/rule-manager/isl-service-rules.puml

@@ -0,0 +1,55 @@
+@startuml
+title Service isl rules installation
+
+box "Network topology" #LightBlue
+participant NetworkIslService
+participant carrier
+participant IslRuleService
+participant RuleManager
+participant SpeakerRulesWorker
+end box
+control speaker.topic
+control network.speaker.topic
+participant Floodlight
+
+== Install service isl rules ==
+
+NetworkIslService -> carrier : SpeakerRulesIslInstallCommand
+carrier -> IslRuleService : SpeakerRulesIslInstallCommand
+IslRuleService -> RuleManager : create isl service rules
+RuleManager -> IslRuleService : SpeakerData
+activate IslRuleService
+IslRuleService -> IslRuleService : form InstallSpeakerCommandsRequest
+IslRuleService -> carrier : InstallSpeakerCommandsRequest
+deactivate IslRuleService
+carrier -> SpeakerRulesWorker : InstallSpeakerCommandsRequest
+SpeakerRulesWorker -> speaker.topic : InstallSpeakerCommandsRequest
+speaker.topic -> Floodlight : InstallSpeakerCommandsRequest
+Floodlight -> network.speaker.topic : SpeakerCommandResponse
+network.speaker.topic -> SpeakerRulesWorker
+SpeakerRulesWorker -> carrier : SpeakerCommandResponse
+carrier -> IslRuleService : SpeakerCommandResponse
+IslRuleService -> carrier : IslDefaultRuleCreatedCommand
+carrier -> NetworkIslService : IslDefaultRuleCreatedCommand
+
+== Delete service isl rules ==
+
+NetworkIslService -> carrier : SpeakerRulesIslRemoveCommand
+carrier -> IslRuleService : SpeakerRulesIslRemoveCommand
+IslRuleService -> RuleManager : create isl service rules
+RuleManager -> IslRuleService : SpeakerData
+activate IslRuleService
+IslRuleService -> IslRuleService : form DeleteSpeakerCommandsRequest
+IslRuleService -> carrier : DeleteSpeakerCommandsRequest
+deactivate IslRuleService
+carrier -> SpeakerRulesWorker : DeleteSpeakerCommandsRequest
+SpeakerRulesWorker -> speaker.topic : DeleteSpeakerCommandsRequest
+speaker.topic -> Floodlight : DeleteSpeakerCommandsRequest
+Floodlight -> network.speaker.topic : SpeakerCommandResponse
+network.speaker.topic -> SpeakerRulesWorker
+SpeakerRulesWorker -> carrier : SpeakerCommandResponse
+carrier -> IslRuleService : SpeakerCommandResponse
+IslRuleService -> carrier : IslDefaultRuleRemovedommand
+carrier -> NetworkIslService : IslDefaultRuleRemovedCommand
+
+@enduml

src-java/base-topology/base-messaging/src/main/java/org/openkilda/messaging/error/ErrorType.java

@@ -66,6 +66,8 @@ public enum ErrorType {
      */
     ALREADY_EXISTS("Object already exists"),
 
+    BUSY("Operation can't be processed at this moment"),
+
     /**
      * The error message for invalid request data.
      */

src-java/base-topology/base-storm-topology/src/main/java/org/openkilda/wfm/share/history/model/FlowEventData.java

@@ -45,6 +45,7 @@ public enum Event {
         UPDATE("Flow updating"),
         REROUTE("Flow rerouting"),
         DELETE("Flow deleting"),
+        SYNC("Flow paths sync"),
         PATH_SWAP("Flow paths swap"),
         SWAP_ENDPOINTS("Flows swap endpoints"),
         FLOW_LOOP_CREATE("Flow loop creating"),

src-java/base-topology/base-storm-topology/src/main/java/org/openkilda/wfm/share/logger/FlowOperationsDashboardLogger.java

@@ -41,6 +41,8 @@ public class FlowOperationsDashboardLogger extends AbstractDashboardLogger {
     private static final String UPDATE_RESULT_EVENT = "flow_update_result";
     private static final String FLOW_DELETE_EVENT = "flow_delete";
     private static final String DELETE_RESULT_EVENT = "flow_delete_result";
+    private static final String FLOW_SYNC_EVENT = "flow_sync";
+    private static final String SYNC_RESULT_EVENT = "flow_sync_result";
     private static final String PATHS_SWAP_EVENT = "paths_swap";
     private static final String REROUTE_EVENT = "flow_reroute";
     private static final String REROUTE_RESULT_EVENT = "flow_reroute_result";
@@ -305,6 +307,45 @@ public void onFailedFlowDelete(String flowId, String failureReason) {
                 data);
     }
 
+    /**
+     * Log a flow-sync event.
+     */
+    public void onFlowSync(String flowId) {
+        Map<String, String> data = new HashMap<>();
+        data.put(TAG, "flow-sync");
+        data.put(FLOW_ID, flowId);
+        data.put(EVENT_TYPE, FLOW_SYNC_EVENT);
+        invokeLogger(Level.INFO, String.format("Performing flow \"%s\" SYNC", flowId), data);
+    }
+
+    /**
+     * Log a flow-sync-successful event.
+     */
+    public void onSuccessfulFlowSync(String flowId) {
+        Map<String, String> data = new HashMap<>();
+        data.put(TAG, "flow-sync-success");
+        data.put(FLOW_ID, flowId);
+        data.put(EVENT_TYPE, FLOW_SYNC_EVENT);
+        data.put("sync-result", "successful");
+        invokeLogger(Level.INFO, String.format("Flow \"%s\" SYNC success", flowId), data);
+    }
+
+    /**
+     * Log a flow-sync-failed event.
+     */
+    public void onFailedFlowSync(String flowId, int failedPathsCount, int totalPathsCount) {
+        Map<String, String> data = new HashMap<>();
+        data.put(TAG, "flow-sync-failed");
+        data.put(FLOW_ID, flowId);
+        data.put(EVENT_TYPE, FLOW_SYNC_EVENT);
+        data.put("sync-result", "failed");
+        invokeLogger(
+                Level.INFO, String.format(
+                        "Flow \"%s\" SYNC failed - %d of %d path have failed to sync",
+                        flowId, failedPathsCount, totalPathsCount),
+                data);
+    }
+
     /**
      * Log a flow-endpoint-swap event.
      */

src-java/base-topology/base-storm-topology/src/main/java/org/openkilda/wfm/share/mappers/FlowPathMapper.java

@@ -117,6 +117,9 @@ public List<PathNodePayload> mapToPathNodes(NetworkEndpoint ingress, List<PathSe
                             + "%s and %s have different switches", ingress.getSwitchId(), right));
                 }
                 resultList.add(new PathNodePayload(ingress.getSwitchId(), ingress.getPortNumber(), right.getSrcPort()));
+            } else {
+                // The case of the same dest port, but different src port.
+                resultList.add(new PathNodePayload(right.getSrcSwitchId(), null, right.getSrcPort()));
             }
             while (rightIter.hasNext()) {
                 left = leftIter.next();
@@ -134,6 +137,9 @@ public List<PathNodePayload> mapToPathNodes(NetworkEndpoint ingress, List<PathSe
                             + "%s and %s have different switches", egress.getSwitchId(), right));
                 }
                 resultList.add(new PathNodePayload(egress.getSwitchId(), right.getDestPort(), egress.getPortNumber()));
+            } else {
+                // The case of the same src port, but different dest port.
+                resultList.add(new PathNodePayload(right.getDestSwitchId(), right.getDestPort(), null));
             }
         }