Merge branch 'release-1.150.0'

CHANGELOG.md

@@ -1,4 +1,27 @@
 # Changelog
+## v1.150.0 (19/12/2023)
+
+### Features:
+-  [#4931](https://github.com/telstra/open-kilda/pull/4931) Tool for dump/restore OpenTSDB data
+
+### Bug Fixes:
+-  [#5513](https://github.com/telstra/open-kilda/pull/5513) #5420: [TEST] Ignore failing WB5164 mirror check (Issue: [#5420](https://github.com/telstra/open-kilda/issues/5420)) [**tests**]
+-  [#5501](https://github.com/telstra/open-kilda/pull/5501) Adjust ordering of HA-flow history actions. (Issue: [#5366](https://github.com/telstra/open-kilda/issues/5366))
+
+### Improvements:
+-  [#5507](https://github.com/telstra/open-kilda/pull/5507) [TEST]: #5504: Storm: Updating network topology manipulation (Issue: [#5504](https://github.com/telstra/open-kilda/issues/5504)) [**tests**]
+-  [#5508](https://github.com/telstra/open-kilda/pull/5508) #5390: [TEST] Fixed tests after changes in history API pt.2 (Issues: [#5390](https://github.com/telstra/open-kilda/issues/5390) [#5390](https://github.com/telstra/open-kilda/issues/5390)) [**tests**]
+-  [#5491](https://github.com/telstra/open-kilda/pull/5491) Remove the deprecated authentication plugin for the MySQL container. (Issue: [#5460](https://github.com/telstra/open-kilda/issues/5460)) [**configuration**]
+-  [#5493](https://github.com/telstra/open-kilda/pull/5493) [TEST] Refactoring the way to choose switch pairs for test [**tests**]
+
+
+For the complete list of changes, check out [the commit log](https://github.com/telstra/open-kilda/compare/v1.149.0...v1.150.0).
+
+### Affected Components:
+history, otsdb
+
+---
+
 ## v1.149.0 (07/12/2023)
 
 ### Bug Fixes:

confd/templates/docker-compose/docker-compose.tmpl

@@ -127,7 +127,7 @@ services:
   mysql_db:
     hostname: mysql.pendev
     image: mysql:8.0.34
-    command: --default-authentication-plugin=mysql_native_password
+    command: --default-authentication-plugin=caching_sha2_password
     volumes:
       - ./docker/mysql/setup_data:/docker-entrypoint-initdb.d
       - sql_data:/var/lib/mysql

src-java/kilda-persistence-hibernate/src/main/java/org/openkilda/persistence/hibernate/entities/history/HibernateHaFlowEvent.java

@@ -70,8 +70,14 @@ public class HibernateHaFlowEvent extends EntityBase implements HaFlowEventData
     @Column(name = "details")
     private String details;
 
+    /**
+     * The ordering relies on grouping messages in topologies: history messages for the same correlation ID arrive to
+     * the same worker and processed sequentially within a single operation (see fieldsGrouping in topology builders).
+     * However, timestamps are not generated by DB, so assumptions about the relation between timestamp and
+     * id cannot be made.
+     */
     @OneToMany(mappedBy = "haFlowEvent", cascade = CascadeType.ALL)
-    @OrderBy("timestamp")
+    @OrderBy("timestamp, id")
     @LazyCollection(LazyCollectionOption.FALSE)
     private List<HibernateHaFlowEventAction> eventActions = new ArrayList<>();
 

src-java/kilda-persistence-hibernate/src/main/java/org/openkilda/persistence/hibernate/repositories/HibernateHistoryHaFlowEventRepository.java

@@ -62,7 +62,7 @@ public List<HaFlowEvent> findByHaFlowIdAndTimeFrame(String haFlowId,
                 () -> fetch(haFlowId, timeFrom, timeTo, maxCount).stream()
                         .map(HaFlowEvent::new)
                         .collect(Collectors.toList()));
-        // fetch does ordering [1,2,3,4,5] and limit to maxCount (let's say top 3) [1,2,3]
+        // fetch does the ordering [1,2,3,4,5] and limit to maxCount (let's say top 3) [1,2,3]
         // then we reverse the collection [3,2,1].
         // This is different from having the opposite ordering in the query: order [5,4,3,2,1] and top 3: [5,4,3]
         Collections.reverse(results);

src-java/kilda-persistence-tinkerpop/src/main/java/org/openkilda/persistence/ferma/frames/HaFlowEventFrame.java

@@ -90,7 +90,8 @@ public List<HaFlowEventAction> getEventActions() {
                         .hasLabel(HaFlowEventActionFrame.FRAME_LABEL)
                         .has(HaFlowEventActionFrame.TASK_ID_PROPERTY, getTaskId()))
                 .toListExplicit(HaFlowEventActionFrame.class).stream()
-                .sorted(Comparator.comparing(HaFlowEventActionFrame::getTimestamp))
+                .sorted(Comparator.comparing(HaFlowEventActionFrame::getTimestamp)
+                        .thenComparing(x -> x.getId()))
                 .map(HaFlowEventAction::new)
                 .collect(Collectors.toList());
     }

src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/DockerHelper.groovy

@@ -59,4 +59,12 @@ class DockerHelper {
     private String getNetworkName() {
         dockerClient.listNetworks()*.name().find { it.contains('_default') && it.contains('kilda') }
     }
+
+    String execute(String containerId, String [] command) {
+        def execCreation = dockerClient.execCreate(containerId, command,
+                DockerClient.ExecCreateParam.attachStdout(), DockerClient.ExecCreateParam.attachStderr())
+        def output = dockerClient.execStart(execCreation.id())
+        def execOutput = output.readFully()
+        return execOutput
+    }
 }

src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/TopologyHelper.groovy

@@ -41,62 +41,14 @@ class TopologyHelper {
     @Autowired
     FloodlightsHelper flHelper
 
-    /**
-     * Get a switch pair of random switches.
-     *
-     * @param forceDifferent whether to exclude the picked src switch when looking for dst switch
-     * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
-     */
-    @Deprecated
-    Tuple2<Switch, Switch> getRandomSwitchPair(boolean forceDifferent = true) {
-        def randomSwitch = { List<Switch> switches ->
-            switches[new Random().nextInt(switches.size())]
-        }
-        def src = randomSwitch(topology.activeSwitches)
-        def dst = randomSwitch(forceDifferent ? topology.activeSwitches - src : topology.activeSwitches)
-        return new Tuple2(src, dst)
+    List<SwitchPair> getAllSwitchPairs(boolean includeReverse = true) {
+        return getSwitchPairs(includeReverse)
     }
 
-    SwitchPairs getAllSwitchPairs(boolean includeReverse = false) {
-        return new SwitchPairs(getSwitchPairs(includeReverse))
-    }
-
-    /**
-     * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
-     */
-    @Deprecated
-    SwitchPair getSingleSwitchPair() {
-        return SwitchPair.singleSwitchInstance(topology.activeSwitches.first())
-    }
-
-    /**
-     * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
-     */
-    @Deprecated
     List<SwitchPair> getAllSingleSwitchPairs() {
         return topology.activeSwitches.collect { SwitchPair.singleSwitchInstance(it) }
     }
 
-    /**
-     * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
-     */
-    @Deprecated
-    SwitchPair getNeighboringSwitchPair() {
-        getSwitchPairs().find {
-            it.paths.min { it.size() }?.size() == 2
-        }
-    }
-
-    /**
-     * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
-     */
-    @Deprecated
-    SwitchPair getNotNeighboringSwitchPair() {
-        getSwitchPairs().find {
-            it.paths.min { it.size() }?.size() > 2
-        }
-    }
-
     /**
      * @deprecated Use new mechanism from org.openkilda.functionaltests.helpers.model.SwitchPairs class
      */
@@ -208,9 +160,9 @@ class TopologyHelper {
     }
 
     SwitchTriplet findSwitchTripletWithSharedEpInTheMiddleOfTheChain() {
-        def pairSharedEpAndEp1 = getAllSwitchPairs().neighbouring().random()
+        def pairSharedEpAndEp1 = new SwitchPairs(getAllSwitchPairs()).neighbouring().random()
         //shared endpoint should be in the middle of the switches chain to deploy ha-flow without shared path
-        def pairEp1AndEp2 = getAllSwitchPairs().neighbouring().excludePairs([pairSharedEpAndEp1]).includeSwitch(pairSharedEpAndEp1.src).random()
+        def pairEp1AndEp2 = new SwitchPairs(getAllSwitchPairs()).neighbouring().excludePairs([pairSharedEpAndEp1]).includeSwitch(pairSharedEpAndEp1.src).random()
         Switch thirdSwitch = pairSharedEpAndEp1.src == pairEp1AndEp2.dst ? pairEp1AndEp2.src : pairEp1AndEp2.dst
         return switchTriplets.find {
                     it.shared.dpId == pairSharedEpAndEp1.src.dpId

src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/WfmManipulator.groovy

@@ -1,5 +1,8 @@
 package org.openkilda.functionaltests.helpers
 
+import java.util.regex.Pattern
+
+import static org.openkilda.functionaltests.helpers.model.ContainerName.STORM
 import static org.openkilda.functionaltests.helpers.model.ContainerName.WFM
 
 import com.spotify.docker.client.DockerClient
@@ -39,14 +42,27 @@ class WfmManipulator {
         }
     }
 
+    String getStormActualNetworkTopology() {
+        String stormUIContainerId = dockerHelper."get container by name"(STORM).id()
+        String[] topologiesList = ["sh", "-c", "PATH=\${PATH}:/opt/storm/bin; storm list | grep network"]
+        String commandOutput = dockerHelper.execute(stormUIContainerId, topologiesList)
+        Pattern pattern = ~/network\w*/
+        assert pattern.matcher(commandOutput).find(), "Something went wrong, network topology name has not been retrieved: \n $commandOutput"
+        //in the blue/green mode, all topologies have a name format: topologyName_mode (mode: blue/green, ex.: network_blue)
+        // to deploy/kill topology use the format topologyName-mode (ex. network-blue)
+        return pattern.matcher(commandOutput).findAll().first().toString().replace("_", "-")
+    }
+
     def killTopology(String topologyName) {
         log.warn "Killing wfm $topologyName topology"
         manipulateTopology("kill", topologyName)
+        log.info "WFM $topologyName topology has been deleted"
     }
 
     def deployTopology(String topologyName) {
         log.warn "Deploying wfm $topologyName topology"
         manipulateTopology("deploy", topologyName)
+        log.info("WFM $topologyName topology has been deployed")
     }
 
     private def manipulateTopology(String action, String topologyName) {

src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/model/ContainerName.groovy

@@ -3,7 +3,8 @@ package org.openkilda.functionaltests.helpers.model
 enum ContainerName {
     GRPC("grpc-speaker"),
     GRPC_STUB("grpc-stub"),
-    WFM("wfm")
+    WFM("wfm"),
+    STORM("storm-ui")
 
     private final String id;
 

src-java/testing/functional-tests/src/main/groovy/org/openkilda/functionaltests/helpers/model/SwitchPairs.groovy

@@ -1,54 +1,85 @@
 package org.openkilda.functionaltests.helpers.model
 
+import org.openkilda.functionaltests.helpers.SwitchHelper
+import org.openkilda.functionaltests.helpers.TopologyHelper
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.context.annotation.Scope
+import org.springframework.stereotype.Component
+
 import static org.junit.jupiter.api.Assumptions.assumeFalse
 
 import org.openkilda.testing.model.topology.TopologyDefinition.Switch
 
+import static org.openkilda.model.SwitchFeature.NOVIFLOW_COPY_FIELD
+import static org.springframework.beans.factory.config.ConfigurableBeanFactory.SCOPE_PROTOTYPE
+
 /**
  * Class which simplifies search for corresponding switch pair. Just chain existing methods to combine requirements
- * Usage: topologyHelper.getAllSwitchPairs()
+ * Usage: switchPairs.all()
  * .nonNeighouring()
  * .withAllTraffgerns()
  * .random()
  * Also it eliminates need to verify if no switch can be found (skips test immediately)
  */
+
+@Component
+@Scope(SCOPE_PROTOTYPE)
 class SwitchPairs {
-    private List<SwitchPair> switchPairs
+    List<SwitchPair> switchPairs
+    @Autowired
+    SwitchHelper switchHelper
+    @Autowired
+    TopologyHelper topologyHelper
 
     SwitchPairs(List<SwitchPair> switchPairs) {
         this.switchPairs = switchPairs
     }
 
+    SwitchPairs all(Boolean includeReverse = true) {
+        switchPairs = topologyHelper.getAllSwitchPairs(includeReverse)
+        return this
+    }
+
+    SwitchPairs singleSwitch() {
+        switchPairs = topologyHelper.getAllSingleSwitchPairs()
+        return this
+    }
+
     SwitchPairs withAtLeastNNonOverlappingPaths(int nonOverlappingPaths) {
-        return new SwitchPairs(switchPairs.findAll {
-            it.paths.unique(false) { a, b -> a.intersect(b) == [] ? 1 : 0 }.size() >= nonOverlappingPaths})
+        switchPairs = switchPairs.findAll {
+            it.paths.unique(false) { a, b -> a.intersect(b) == [] ? 1 : 0 }.size() >= nonOverlappingPaths
+        }
+        return this
     }
 
     SwitchPairs withShortestPathShorterThanOthers() {
-        return new SwitchPairs(switchPairs.findAll {it.getPaths()[0].size() != it.getPaths()[1].size()})
+        switchPairs = switchPairs.findAll { it.getPaths()[0].size() != it.getPaths()[1].size() }
+        return this
     }
 
     SwitchPairs nonNeighbouring() {
-        return new SwitchPairs(switchPairs.findAll { it.paths.min { it.size() }?.size() > 2})
+        switchPairs = switchPairs.findAll { it.paths.min { it.size() }?.size() > 2 }
+        return this
     }
+
     SwitchPairs neighbouring() {
-        return new SwitchPairs(switchPairs.findAll { it.paths.min { it.size() }?.size() == 2})
+        switchPairs = switchPairs.findAll { it.paths.min { it.size() }?.size() == 2 }
+        return this
     }
 
     SwitchPairs excludePairs(List<SwitchPair> excludePairs) {
-        return new SwitchPairs(switchPairs.findAll { !excludePairs.contains(it) })
+        switchPairs = switchPairs.findAll { !excludePairs.contains(it) }
+        return this
     }
 
     SwitchPairs sortedByShortestPathLengthAscending() {
-        return new SwitchPairs(switchPairs.sort {it.paths.min { it.size() }?.size() > 2})
-    }
-
-    SwitchPairs sortedBySmallestPathsAmount() {
-        return new SwitchPairs(switchPairs.sort{it.paths.size()})
+        switchPairs = switchPairs.sort { it.paths.min { it.size() }?.size() > 2 }
+        return this
     }
 
     SwitchPair random() {
-        return new SwitchPairs(switchPairs.shuffled()).first()
+        switchPairs = switchPairs.shuffled()
+        return this.first()
     }
 
     SwitchPair first() {
@@ -57,14 +88,52 @@ class SwitchPairs {
     }
 
     SwitchPairs includeSwitch(Switch sw) {
-        return new SwitchPairs(switchPairs.findAll { it.src == sw || it.dst == sw})
+        switchPairs = switchPairs.findAll { it.src == sw || it.dst == sw }
+        return this
     }
 
     SwitchPairs excludeSwitches(List<Switch> switchesList) {
-        return new SwitchPairs(switchPairs.findAll { !(it.src in switchesList) || !(it.dst in switchesList)})
+        switchPairs = switchPairs.findAll { !(it.src in switchesList) || !(it.dst in switchesList) }
+        return this
     }
 
     List<Switch> collectSwitches() {
         switchPairs.collectMany { return [it.src, it.dst] }.unique()
     }
+
+    SwitchPairs withAtLeastNTraffgensOnSource(int traffgensConnectedToSource) {
+        switchPairs = switchPairs.findAll { it.getSrc().getTraffGens().size() >= traffgensConnectedToSource }
+        return this
+    }
+
+    SwitchPairs withBothSwitchesVxLanEnabled() {
+        switchPairs = switchPairs.findAll { [it.src, it.dst].every { sw -> switchHelper.isVxlanEnabled(sw.dpId) } }
+        return this
+    }
+
+    SwitchPairs withIslRttSupport() {
+        this.assertAllSwitchPairsAreNeighbouring()
+        switchPairs = switchPairs.findAll { [it.src, it.dst].every { it.features.contains(NOVIFLOW_COPY_FIELD) } }
+        return this
+    }
+
+    SwitchPairs withExactlyNIslsBetweenSwitches(int expectedIslsBetweenSwitches) {
+        this.assertAllSwitchPairsAreNeighbouring()
+        switchPairs = switchPairs.findAll { it.paths.findAll { it.size() == 2 }.size() == expectedIslsBetweenSwitches }
+        return this
+    }
+
+    SwitchPairs withMoreThanNIslsBetweenSwitches(int expectedMinimalIslsBetweenSwitches) {
+        this.assertAllSwitchPairsAreNeighbouring()
+        switchPairs = switchPairs.findAll {
+                    it.paths.findAll { it.size() == 2 }
+                            .size() > expectedMinimalIslsBetweenSwitches
+                }
+        return this
+    }
+
+    private void assertAllSwitchPairsAreNeighbouring() {
+        assert switchPairs.size() == this.neighbouring().getSwitchPairs().size(),
+                "This method is applicable only to the neighbouring switch pairs"
+    }
 }

src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/BaseSpecification.groovy

@@ -1,5 +1,7 @@
 package org.openkilda.functionaltests
 
+import org.openkilda.functionaltests.helpers.model.SwitchPairs
+
 import static groovyx.gpars.GParsPool.withPool
 import static org.junit.jupiter.api.Assumptions.assumeTrue
 
@@ -69,6 +71,8 @@ class BaseSpecification extends Specification {
     StatsHelper statsHelper
     @Autowired @Shared
     LabService labService
+    @Autowired @Shared
+    SwitchPairs switchPairs
 
     @Value('${spring.profiles.active}') @Shared
     String profile

src-java/testing/functional-tests/src/test/groovy/org/openkilda/functionaltests/spec/configuration/ConfigurationSpec.groovy

@@ -35,12 +35,13 @@ class ConfigurationSpec extends HealthCheckSpecification {
     @Shared
     FlowEncapsulationType defaultEncapsulationType = FlowEncapsulationType.TRANSIT_VLAN
 
+
     def "System takes into account default flow encapsulation type while creating a flow"() {
         when: "Create a flow without encapsulation type"
-        def switchPair = topologyHelper.getAllNeighboringSwitchPairs().find { swP ->
-            [swP.src, swP.dst].every { sw -> switchHelper.isVxlanEnabled(sw.dpId) }
-        }
-        assumeTrue(switchPair != null, "Unable to find required switch pair in topology")
+        def switchPair = switchPairs.all()
+                .neighbouring()
+                .withBothSwitchesVxLanEnabled()
+                .random()
         def flow1 = flowHelperV2.randomFlow(switchPair)
         flow1.encapsulationType = null
         flowHelperV2.addFlow(flow1)