From 66de8b502482a984583442bf007ae22608f79f1a Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 17:56:59 +0200 Subject: [PATCH 01/11] extend input and persistence stuff for k8s --- src/main/avro/ExplorvizProtocol.avdl | 5 +++++ src/main/java/net/explorviz/span/hash/HashHelper.java | 7 +++++-- .../java/net/explorviz/span/kafka/TopologyProducer.java | 4 ++++ .../net/explorviz/span/persistence/PersistenceSpan.java | 6 +++++- .../span/persistence/PersistenceSpanProcessor.java | 4 +++- .../net/explorviz/span/persistence/SpanConverter.java | 9 +++++++-- src/main/resources/init_script.cql | 4 ++++ 7 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/main/avro/ExplorvizProtocol.avdl b/src/main/avro/ExplorvizProtocol.avdl index 4626924..5f4a1d6 100644 --- a/src/main/avro/ExplorvizProtocol.avdl +++ b/src/main/avro/ExplorvizProtocol.avdl @@ -36,5 +36,10 @@ protocol ExplorvizProtocol { string appInstanceId; string appLanguage; string hashCode = "default-hashcode"; + + string k8sPodName = ""; + string k8sNodeName = ""; + string k8sNamespace = ""; + string k8sDeploymentName = ""; } } diff --git a/src/main/java/net/explorviz/span/hash/HashHelper.java b/src/main/java/net/explorviz/span/hash/HashHelper.java index 77a07f1..5bde56e 100644 --- a/src/main/java/net/explorviz/span/hash/HashHelper.java +++ b/src/main/java/net/explorviz/span/hash/HashHelper.java @@ -14,14 +14,17 @@ private HashHelper() { } public static String calculateSpanHash(final UUID landscapeToken, final String nodeIpAddress, - final String applicationName, final int applicationInstance, final String methodFqn) { + final String applicationName, final int applicationInstance, final String methodFqn, + final String k8sPodName, final String k8sNodeName, final String k8sNamespace, + final String k8sDeploymentName) { final HighwayHash hash = new HighwayHash(HIGHWAY_HASH_KEY); // TODO: Fill with IPv6 address bits (Convert IPv4 to IPv4-in-IPv6 representation) hash.update(landscapeToken.getMostSignificantBits(), landscapeToken.getLeastSignificantBits(), applicationInstance, 0L); - final String builder = applicationName + ';' + nodeIpAddress + ';' + methodFqn; + final String builder = applicationName + ';' + nodeIpAddress + ';' + methodFqn + + ';' + k8sPodName + ';' + k8sNodeName + ';' + k8sNamespace + ';' + k8sDeploymentName; final byte[] bytes = builder.getBytes(StandardCharsets.UTF_8); int position = 0; for (; bytes.length - position >= 32; position += 32) { diff --git a/src/main/java/net/explorviz/span/kafka/TopologyProducer.java b/src/main/java/net/explorviz/span/kafka/TopologyProducer.java index aef2160..f10d283 100644 --- a/src/main/java/net/explorviz/span/kafka/TopologyProducer.java +++ b/src/main/java/net/explorviz/span/kafka/TopologyProducer.java @@ -48,6 +48,10 @@ public Topology buildTopology() { final KStream spanStream = builder.stream(this.inTopic, Consumed.with(Serdes.String(), this.spanSerde)); + spanStream.foreach((token, span) -> { + final var a = 1; + }); + spanStream.foreach((token, span) -> this.lastReceivedSpans.incrementAndGet()); if (LOGGER.isTraceEnabled()) { diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpan.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpan.java index 305fe1c..5e3ee79 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpan.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpan.java @@ -14,7 +14,11 @@ public record PersistenceSpan( String applicationLanguage, int applicationInstance, String methodFqn, - String methodHash + String methodHash, + String k8sPodName, + String k8sNodeName, + String k8sNamespace, + String k8sDeploymentName ) { public static final UUID DEFAULT_UUID = UUID.fromString("7cd8a9a7-b840-4735-9ef0-2dbbfa01c039"); diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java index 6e35a36..6c8a334 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java @@ -59,7 +59,8 @@ public PersistenceSpanProcessor(final QuarkusCqlSession session) { + "VALUES (?, ?, ?, ?, ?)"); this.insertSpanStructureStatement = session.prepare("INSERT INTO span_structure " + "(landscape_token, method_hash, node_ip_address, application_name, application_language, " - + "application_instance, method_fqn, time_seen) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + + "application_instance, method_fqn, time_seen, k8s_pod_name, k8s_node_name, k8s_namespace, k8s_deployment_name) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "USING TIMESTAMP ?"); this.updateSpanBucketCounter = session.prepare("UPDATE span_count_per_time_bucket_and_token " + "SET span_count = span_count + 1 " @@ -105,6 +106,7 @@ private void insertSpanStructure(final PersistenceSpan span) { insertSpanStructureStatement.bind(span.landscapeToken(), span.methodHash(), span.nodeIpAddress(), span.applicationName(), span.applicationLanguage(), span.applicationInstance(), span.methodFqn(), span.startTime(), + span.k8sPodName(), span.k8sNodeName(), span.k8sNamespace(), span.k8sDeploymentName(), Instant.now().toEpochMilli()); session.executeAsync(stmtStructure).whenComplete((result, failure) -> { diff --git a/src/main/java/net/explorviz/span/persistence/SpanConverter.java b/src/main/java/net/explorviz/span/persistence/SpanConverter.java index 00e924e..421c67d 100644 --- a/src/main/java/net/explorviz/span/persistence/SpanConverter.java +++ b/src/main/java/net/explorviz/span/persistence/SpanConverter.java @@ -25,14 +25,19 @@ public PersistenceSpan apply(final Span span) { final int applicationInstance = Integer.parseInt(span.getAppInstanceId()); final String applicationLanguage = span.getAppLanguage(); final String methodFqn = span.getFullyQualifiedOperationName(); + final String k8sPodName = span.getK8sPodName(); + final String k8sNodeName = span.getK8sNodeName(); + final String k8sNamespace = span.getK8sNamespace(); + final String k8sDeploymentName = span.getK8sDeploymentName(); final String methodHashCode = HashHelper.calculateSpanHash(landscapeToken, nodeIpAddress, applicationName, - applicationInstance, methodFqn); + applicationInstance, methodFqn, k8sPodName, k8sNodeName, k8sNamespace, + k8sDeploymentName); return new PersistenceSpan(landscapeToken, span.getSpanId(), span.getParentSpanId(), span.getTraceId(), startTime, endTime, nodeIpAddress, applicationName, applicationLanguage, applicationInstance, methodFqn, - methodHashCode); + methodHashCode, k8sPodName, k8sNodeName, k8sNamespace, k8sDeploymentName); } } diff --git a/src/main/resources/init_script.cql b/src/main/resources/init_script.cql index e22e8f0..fc3e7c6 100644 --- a/src/main/resources/init_script.cql +++ b/src/main/resources/init_script.cql @@ -75,5 +75,9 @@ CREATE TABLE IF NOT EXISTS explorviz.span_structure application_instance int, method_fqn text, time_seen bigint, + k8s_pod_name text, + k8s_node_name text, + k8s_namespace text, + k8s_deployment_name text, PRIMARY KEY ((landscape_token), method_hash) ); From e887db4f340d0dafba0594c5db659acd549df492 Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 19:00:00 +0200 Subject: [PATCH 02/11] add k8s stuff to landscape assembly --- .../span/landscape/K8sDeployment.java | 9 ++ .../span/landscape/K8sNamespace.java | 9 ++ .../net/explorviz/span/landscape/K8sNode.java | 9 ++ .../net/explorviz/span/landscape/K8sPod.java | 9 ++ .../explorviz/span/landscape/Landscape.java | 3 +- .../impl/DefaultLandscapeAssembler.java | 87 +++++++++++++++++-- .../landscape/loader/LandscapeRecord.java | 12 ++- 7 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 src/main/java/net/explorviz/span/landscape/K8sDeployment.java create mode 100644 src/main/java/net/explorviz/span/landscape/K8sNamespace.java create mode 100644 src/main/java/net/explorviz/span/landscape/K8sNode.java create mode 100644 src/main/java/net/explorviz/span/landscape/K8sPod.java diff --git a/src/main/java/net/explorviz/span/landscape/K8sDeployment.java b/src/main/java/net/explorviz/span/landscape/K8sDeployment.java new file mode 100644 index 0000000..e440f4b --- /dev/null +++ b/src/main/java/net/explorviz/span/landscape/K8sDeployment.java @@ -0,0 +1,9 @@ +package net.explorviz.span.landscape; + +import java.util.List; + +public record K8sDeployment( + String name, + List k8sPods +) { +} diff --git a/src/main/java/net/explorviz/span/landscape/K8sNamespace.java b/src/main/java/net/explorviz/span/landscape/K8sNamespace.java new file mode 100644 index 0000000..9a673d8 --- /dev/null +++ b/src/main/java/net/explorviz/span/landscape/K8sNamespace.java @@ -0,0 +1,9 @@ +package net.explorviz.span.landscape; + +import java.util.List; + +public record K8sNamespace( + String name, + List k8sDeployments +) { +} diff --git a/src/main/java/net/explorviz/span/landscape/K8sNode.java b/src/main/java/net/explorviz/span/landscape/K8sNode.java new file mode 100644 index 0000000..3eb5159 --- /dev/null +++ b/src/main/java/net/explorviz/span/landscape/K8sNode.java @@ -0,0 +1,9 @@ +package net.explorviz.span.landscape; + +import java.util.List; + +public record K8sNode( + String name, + List k8sNamespaces +) { +} diff --git a/src/main/java/net/explorviz/span/landscape/K8sPod.java b/src/main/java/net/explorviz/span/landscape/K8sPod.java new file mode 100644 index 0000000..f3710c2 --- /dev/null +++ b/src/main/java/net/explorviz/span/landscape/K8sPod.java @@ -0,0 +1,9 @@ +package net.explorviz.span.landscape; + +import java.util.List; + +public record K8sPod( + String name, + List applications +) { +} diff --git a/src/main/java/net/explorviz/span/landscape/Landscape.java b/src/main/java/net/explorviz/span/landscape/Landscape.java index 7cefeba..9051212 100644 --- a/src/main/java/net/explorviz/span/landscape/Landscape.java +++ b/src/main/java/net/explorviz/span/landscape/Landscape.java @@ -5,7 +5,8 @@ public record Landscape( UUID landscapeToken, - List nodes + List nodes, + List k8sNodes ) { } diff --git a/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java b/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java index 82a85c3..5e900ec 100644 --- a/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java +++ b/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java @@ -1,13 +1,15 @@ package net.explorviz.span.landscape.assembler.impl; import jakarta.enterprise.context.ApplicationScoped; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Optional; -import java.util.UUID; + +import java.util.*; + import net.explorviz.span.landscape.Application; import net.explorviz.span.landscape.Class; +import net.explorviz.span.landscape.K8sDeployment; +import net.explorviz.span.landscape.K8sNamespace; +import net.explorviz.span.landscape.K8sNode; +import net.explorviz.span.landscape.K8sPod; import net.explorviz.span.landscape.Landscape; import net.explorviz.span.landscape.Method; import net.explorviz.span.landscape.Node; @@ -27,7 +29,7 @@ public Landscape assembleFromRecords(final Collection records) records.stream().findFirst().orElseThrow(NoRecordsException::new).landscapeToken(); // Create empty landscape and insert all records - final Landscape landscape = new Landscape(token, new ArrayList<>()); + final Landscape landscape = new Landscape(token, new ArrayList<>(), new ArrayList<>()); this.insertAll(landscape, records); return landscape; @@ -43,8 +45,11 @@ public void insertAll(final Landscape landscape, final Collection k8sConstructs = getK8sConstructsForRecord(record, landscape); + final Node node = k8sConstructs.isEmpty() ? getNodeForRecord(landscape, record) : null; + final Application app = + k8sConstructs.isEmpty() ? getApplicationForRecord(record, node) : k8sConstructs.get() + .app(); final String[] packages = getPackagesForRecord(record, app); final Package leafPkg = PackageHelper.fromPath(app, packages); final Class cls = getClassForRecord(record, leafPkg); @@ -72,6 +77,72 @@ private Node getNodeForRecord(final Landscape landscape, final LandscapeRecord r return node; } + private record K8sConstructs( + K8sPod pod, + K8sNode node, + K8sNamespace namespace, + K8sDeployment deployment, + Application app + ) { + } + + private Optional getK8sConstructsForRecord(final LandscapeRecord record, + final Landscape landscape) { + final var podName = record.k8sPodName(); + final var nodeName = record.k8sNodeName(); + final var namespaceName = record.k8sNamespace(); + final var deploymentName = record.k8sDeploymentName(); + + if (record.k8sPodName() == null || record.k8sPodName().isEmpty()) { + return Optional.empty(); + } + + var node = landscape.k8sNodes().stream().filter(n -> Objects.equals(n.name(), nodeName)) + .findFirst().orElse(null); + if (node == null) { + node = new K8sNode(nodeName, new ArrayList<>()); + landscape.k8sNodes().add(node); + } + + var namespace = + node.k8sNamespaces().stream().filter(n -> Objects.equals(n.name(), namespaceName)) + .findFirst().orElse(null); + if (namespace == null) { + namespace = new K8sNamespace(namespaceName, new ArrayList<>()); + node.k8sNamespaces().add(namespace); + } + + var deployment = namespace.k8sDeployments().stream() + .filter(d -> Objects.equals(d.name(), deploymentName)).findFirst().orElse(null); + if (deployment == null) { + deployment = new K8sDeployment(deploymentName, new ArrayList<>()); + namespace.k8sDeployments().add(deployment); + } + + var pod = deployment.k8sPods().stream().filter(p -> Objects.equals(p.name(), podName)) + .findFirst().orElse(null); + if (pod == null) { + pod = new K8sPod(podName, new ArrayList<>()); + deployment.k8sPods().add(pod); + } + + + final String applicationName = record.applicationName(); + final int applicationInstance = record.applicationInstance(); + final String applicationLanguage = record.applicationLanguage(); + var app = pod.applications().stream() + .filter( + a -> Objects.equals(a.name(), applicationName) && a.instance() == applicationInstance) + .findFirst().orElse(null); + if (app == null) { + app = new Application(applicationName, applicationLanguage, applicationInstance, + new ArrayList<>()); + pod.applications().add(app); + } + + return Optional.of(new K8sConstructs(pod, node, namespace, deployment, app)); + } + private Application getApplicationForRecord(final LandscapeRecord record, final Node node) { final Application app; diff --git a/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java b/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java index d645b19..998a678 100644 --- a/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java +++ b/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java @@ -15,6 +15,10 @@ public record LandscapeRecord( String packageName, String className, String methodName, + String k8sPodName, + String k8sNodeName, + String k8sNamespace, + String k8sDeploymentName, long timeSeen ) { @@ -27,6 +31,11 @@ public static LandscapeRecord fromRow(final Row row) { final int applicationInstance = row.getInt("application_instance"); final String methodFqn = row.getString("method_fqn"); final long timeSeen = row.getLong("time_seen"); + final String k8sPodName = row.getString("k8s_pod_name"); + final String k8sNodeName = row.getString("k8s_node_name"); + final String k8sNamespace = row.getString("k8s_namespace"); + final String k8sDeploymentName = row.getString("k8s_deployment_name"); + // TODO: Error handling /* @@ -41,6 +50,7 @@ public static LandscapeRecord fromRow(final Row row) { final String methodName = operationFqnSplit[operationFqnSplit.length - 1]; return new LandscapeRecord(landscapeToken, methodHash, nodeIpAddress, applicationName, - applicationLanguage, applicationInstance, packageName, className, methodName, timeSeen); + applicationLanguage, applicationInstance, packageName, className, methodName, k8sPodName, + k8sNodeName, k8sNamespace, k8sDeploymentName, timeSeen); } } From 240330daf8a9ec9e4e3a7706ba8b08cbb7babf29 Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 19:00:17 +0200 Subject: [PATCH 03/11] remove dbg code --- src/main/java/net/explorviz/span/kafka/TopologyProducer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/net/explorviz/span/kafka/TopologyProducer.java b/src/main/java/net/explorviz/span/kafka/TopologyProducer.java index f10d283..aef2160 100644 --- a/src/main/java/net/explorviz/span/kafka/TopologyProducer.java +++ b/src/main/java/net/explorviz/span/kafka/TopologyProducer.java @@ -48,10 +48,6 @@ public Topology buildTopology() { final KStream spanStream = builder.stream(this.inTopic, Consumed.with(Serdes.String(), this.spanSerde)); - spanStream.foreach((token, span) -> { - final var a = 1; - }); - spanStream.foreach((token, span) -> this.lastReceivedSpans.incrementAndGet()); if (LOGGER.isTraceEnabled()) { From eded215517585baad46a108d081069583ff0fab2 Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 19:08:43 +0200 Subject: [PATCH 04/11] fix damn code style --- .../assembler/impl/DefaultLandscapeAssembler.java | 9 ++++++--- .../span/persistence/PersistenceSpanProcessor.java | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java b/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java index 5e900ec..7a6693c 100644 --- a/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java +++ b/src/main/java/net/explorviz/span/landscape/assembler/impl/DefaultLandscapeAssembler.java @@ -1,9 +1,12 @@ package net.explorviz.span.landscape.assembler.impl; import jakarta.enterprise.context.ApplicationScoped; - -import java.util.*; - +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; import net.explorviz.span.landscape.Application; import net.explorviz.span.landscape.Class; import net.explorviz.span.landscape.K8sDeployment; diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java index 6c8a334..4d8a90e 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java @@ -59,7 +59,8 @@ public PersistenceSpanProcessor(final QuarkusCqlSession session) { + "VALUES (?, ?, ?, ?, ?)"); this.insertSpanStructureStatement = session.prepare("INSERT INTO span_structure " + "(landscape_token, method_hash, node_ip_address, application_name, application_language, " - + "application_instance, method_fqn, time_seen, k8s_pod_name, k8s_node_name, k8s_namespace, k8s_deployment_name) " + + "application_instance, method_fqn, time_seen, " + + "k8s_pod_name, k8s_node_name, k8s_namespace, k8s_deployment_name) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "USING TIMESTAMP ?"); this.updateSpanBucketCounter = session.prepare("UPDATE span_count_per_time_bucket_and_token " From 31a1dc27c89c4e95cfcd2ad6bf932c79c518b088 Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 19:33:21 +0200 Subject: [PATCH 05/11] fix misformed log --- .../net/explorviz/span/persistence/PersistenceSpanProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java index 4d8a90e..4303687 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java @@ -179,7 +179,6 @@ private void insertTrace(final PersistenceSpan span) { lastSavedTraces.incrementAndGet(); LOGGER.atTrace().addArgument(span::landscapeToken).addArgument(span::traceId) .addArgument(tenSecondBucket) - .addArgument(span::traceId) .log("Saved new trace with token={}, trace_id={}, and ten second epoch bucket={}"); } else { lastFailures.incrementAndGet(); From 2e0d653e818e1b114014701b8277efd0514aca83 Mon Sep 17 00:00:00 2001 From: kirill Date: Sun, 23 Jun 2024 19:53:19 +0200 Subject: [PATCH 06/11] fix test compilation, why cant i run them?? am i stupid? i wish they were failing, but they dont even run. i hope it works in ci. --- .../span/api/LandscapeResourceIt.java | 24 ++++++++++++------- .../span/api/TimestampResourceIt.java | 11 +++++---- .../span/persistence/TraceLoaderIt.java | 6 ++--- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/integrationTest/java/net/explorviz/span/api/LandscapeResourceIt.java b/src/integrationTest/java/net/explorviz/span/api/LandscapeResourceIt.java index 92e8ad1..96781df 100644 --- a/src/integrationTest/java/net/explorviz/span/api/LandscapeResourceIt.java +++ b/src/integrationTest/java/net/explorviz/span/api/LandscapeResourceIt.java @@ -38,22 +38,26 @@ void testLoadAllStructureSpans() { final PersistenceSpan differentTokenSpan = new PersistenceSpan( UUID.randomUUID(), "123L", "", "1L", startEarly, - endEarly, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", "847"); + endEarly, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final String duplicateMethodName = "myMethodName()"; final String otherMethodName = "myOtherMethodName()"; final PersistenceSpan firstOccurenceSpan = new PersistenceSpan(uuidExpected, "123L", "", "1L", startEarly, endEarly, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + duplicateMethodName, "847"); + "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan secondOccurenceSpan = new PersistenceSpan(uuidExpected, "789L", "", "3L", startLate, endLate, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + duplicateMethodName, "847"); + "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan otherSpan = new PersistenceSpan(uuidExpected, "456L", "0L", "", startExpected, endExpected, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + otherMethodName, "321"); + "net.explorviz.Class." + otherMethodName, "321", + "iamnotapod", "iamnotanode", "iamnotanamespace", "iamnotadeployment"); spanProcessor.accept(differentTokenSpan); spanProcessor.accept(firstOccurenceSpan); @@ -86,22 +90,26 @@ void testLoadStructureSpansByTimeRange() { final PersistenceSpan differentTokenSpan = new PersistenceSpan( UUID.randomUUID(), "123L", "", "1L", startEarly, - endEarly, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", "847"); + endEarly, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment" ); final String duplicateMethodName = "myMethodName()"; final String otherMethodName = "myOtherMethodName()"; final PersistenceSpan firstOccurenceSpan = new PersistenceSpan(uuidExpected, "123L", "", "1L", startEarly, endEarly, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + duplicateMethodName, "847"); + "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan secondOccurenceSpan = new PersistenceSpan(uuidExpected, "789L", "", "3L", startLate, endLate, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + duplicateMethodName, "847"); + "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan otherSpan = new PersistenceSpan(uuidExpected, "456L", "", "2L", startExpected, endExpected, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class." + otherMethodName, "321"); + "net.explorviz.Class." + otherMethodName, "321", + "iamnotapod", "iamnotanode", "iamnotanamespace", "iamnotadeployment"); spanProcessor.accept(differentTokenSpan); spanProcessor.accept(firstOccurenceSpan); diff --git a/src/integrationTest/java/net/explorviz/span/api/TimestampResourceIt.java b/src/integrationTest/java/net/explorviz/span/api/TimestampResourceIt.java index d26ffe7..6e23e08 100644 --- a/src/integrationTest/java/net/explorviz/span/api/TimestampResourceIt.java +++ b/src/integrationTest/java/net/explorviz/span/api/TimestampResourceIt.java @@ -40,23 +40,26 @@ void testLoadAllTimestampsForToken() { final PersistenceSpan differentTokenSpan = new PersistenceSpan(UUID.randomUUID(), "123L", "", "1L", startEarly, endEarly, "nodeIp", "app-name", "java", 0, - "net.explorviz.Class.myMethod()", "847"); + "net.explorviz.Class.myMethod()", "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final String duplicateMethodName = "myMethodName()"; final String otherMethodName = "myOtherMethodName()"; final PersistenceSpan firstOccurenceSpan = new PersistenceSpan(uuidExpected, "123L", "", "1L", startEarly, endEarly, - "nodeIp", "app-name", "java", 0, "net.explorviz.Class." + duplicateMethodName, "847"); + "nodeIp", "app-name", "java", 0, "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan secondOccurenceSpan = new PersistenceSpan(uuidExpected, "789L", "", "3L", startLate, endLate, - "nodeIp", "app-name", "java", 0, "net.explorviz.Class." + duplicateMethodName, "847"); + "nodeIp", "app-name", "java", 0, "net.explorviz.Class." + duplicateMethodName, "847", + "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan otherSpan = new PersistenceSpan(uuidExpected, "456L", "0L", "", startExpected, endExpected, "nodeIp", "app-name", "java", 0, "net.explorviz.Class." + otherMethodName, - "321"); + "321", "iamnotapod", "iamnotanode", "iamnotanamespace", "iamnotadeployment"); spanProcessor.accept(differentTokenSpan); spanProcessor.accept(firstOccurenceSpan); diff --git a/src/integrationTest/java/net/explorviz/span/persistence/TraceLoaderIt.java b/src/integrationTest/java/net/explorviz/span/persistence/TraceLoaderIt.java index 5922cf9..6bc6126 100644 --- a/src/integrationTest/java/net/explorviz/span/persistence/TraceLoaderIt.java +++ b/src/integrationTest/java/net/explorviz/span/persistence/TraceLoaderIt.java @@ -44,17 +44,17 @@ void testLoadTracesByTimeRange() { final PersistenceSpan earlySpan = new PersistenceSpan(landscapeToken, "123L", "", "1L", startEarly, endEarly, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", - "847"); + "847", "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan expectedSpan = new PersistenceSpan(landscapeToken, "456L", "", "2L", startExpected, endExpected, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", - "847"); + "847", "iamapod", "iamanode", "iamanamespace", "iamadeployment"); final PersistenceSpan lateSpan = new PersistenceSpan(landscapeToken, "789L", "", "3L", startLate, endLate, "nodeIp", "app-name", "java", 0, "net.explorviz.Class.myMethod()", - "847"); + "847", "iamapod", "iamanode", "iamanamespace", "iamadeployment"); spanProcessor.accept(earlySpan); spanProcessor.accept(expectedSpan); From 8d3070f76252e36beb2ac8bae555ee69a2468176 Mon Sep 17 00:00:00 2001 From: kirill Date: Thu, 22 Aug 2024 14:16:54 +0200 Subject: [PATCH 07/11] dont include k8s metadata in hash when not provided --- src/main/java/net/explorviz/span/hash/HashHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/explorviz/span/hash/HashHelper.java b/src/main/java/net/explorviz/span/hash/HashHelper.java index 5bde56e..ef27641 100644 --- a/src/main/java/net/explorviz/span/hash/HashHelper.java +++ b/src/main/java/net/explorviz/span/hash/HashHelper.java @@ -23,8 +23,10 @@ public static String calculateSpanHash(final UUID landscapeToken, final String n hash.update(landscapeToken.getMostSignificantBits(), landscapeToken.getLeastSignificantBits(), applicationInstance, 0L); - final String builder = applicationName + ';' + nodeIpAddress + ';' + methodFqn - + ';' + k8sPodName + ';' + k8sNodeName + ';' + k8sNamespace + ';' + k8sDeploymentName; + String builder = applicationName + ';' + nodeIpAddress + ';' + methodFqn; + if(k8sPodName != null && !k8sPodName.isEmpty()) { + builder += ';' + k8sPodName + ';' + k8sNodeName + ';' + k8sNamespace + ';' + k8sDeploymentName; + } final byte[] bytes = builder.getBytes(StandardCharsets.UTF_8); int position = 0; for (; bytes.length - position >= 32; position += 32) { From fcf752f034c3b4711fa5170ca11070872a894a86 Mon Sep 17 00:00:00 2001 From: kirill Date: Thu, 5 Sep 2024 16:41:25 +0200 Subject: [PATCH 08/11] fix formatting --- src/main/java/net/explorviz/span/hash/HashHelper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/explorviz/span/hash/HashHelper.java b/src/main/java/net/explorviz/span/hash/HashHelper.java index ef27641..8279141 100644 --- a/src/main/java/net/explorviz/span/hash/HashHelper.java +++ b/src/main/java/net/explorviz/span/hash/HashHelper.java @@ -24,8 +24,9 @@ public static String calculateSpanHash(final UUID landscapeToken, final String n applicationInstance, 0L); String builder = applicationName + ';' + nodeIpAddress + ';' + methodFqn; - if(k8sPodName != null && !k8sPodName.isEmpty()) { - builder += ';' + k8sPodName + ';' + k8sNodeName + ';' + k8sNamespace + ';' + k8sDeploymentName; + if (k8sPodName != null && !k8sPodName.isEmpty()) { + builder += + ';' + k8sPodName + ';' + k8sNodeName + ';' + k8sNamespace + ';' + k8sDeploymentName; } final byte[] bytes = builder.getBytes(StandardCharsets.UTF_8); int position = 0; From 7ab29f48dc90948434edf20b3cc742540b1aaa2f Mon Sep 17 00:00:00 2001 From: kirill Date: Thu, 12 Sep 2024 17:47:01 +0200 Subject: [PATCH 09/11] fix bug kinda --- .../span/persistence/PersistenceSpanProcessor.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java index 4303687..311257a 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java @@ -71,12 +71,13 @@ public PersistenceSpanProcessor(final QuarkusCqlSession session) { @Override public void accept(final PersistenceSpan span) { - final Set knownHashes = knownHashesByLandscape.computeIfAbsent(span.landscapeToken(), - uuid -> ConcurrentHashMap.newKeySet()); - - if (knownHashes.add(span.methodHash())) { + // condition was removed, since it was for some unknown reason causing issues. + // However, with cassandras primary key the whole table basically functions as a set, so no worries. +// final Set knownHashes = knownHashesByLandscape.computeIfAbsent(span.landscapeToken(), +// uuid -> ConcurrentHashMap.newKeySet()); +// if (knownHashes.add(span.methodHash())) { insertSpanStructure(span); - } +// } // TODO: We should probably only insert spans // after corresponding span_structure has been inserted? From 3d06819610b19b254306af2c32fb55c9dffc80e6 Mon Sep 17 00:00:00 2001 From: kirill Date: Sat, 14 Sep 2024 18:53:49 +0200 Subject: [PATCH 10/11] unfix bug. there was no bug. i don't know. --- .../persistence/PersistenceSpanProcessor.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java index 311257a..de0f2b6 100644 --- a/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java +++ b/src/main/java/net/explorviz/span/persistence/PersistenceSpanProcessor.java @@ -70,14 +70,11 @@ public PersistenceSpanProcessor(final QuarkusCqlSession session) { @Override public void accept(final PersistenceSpan span) { - - // condition was removed, since it was for some unknown reason causing issues. - // However, with cassandras primary key the whole table basically functions as a set, so no worries. -// final Set knownHashes = knownHashesByLandscape.computeIfAbsent(span.landscapeToken(), -// uuid -> ConcurrentHashMap.newKeySet()); -// if (knownHashes.add(span.methodHash())) { + final Set knownHashes = knownHashesByLandscape.computeIfAbsent(span.landscapeToken(), + uuid -> ConcurrentHashMap.newKeySet()); + if (knownHashes.add(span.methodHash())) { insertSpanStructure(span); -// } + } // TODO: We should probably only insert spans // after corresponding span_structure has been inserted? @@ -158,7 +155,7 @@ private void insertSpanDynamic(final PersistenceSpan span) { .log("Saved new dynamic span with method_hash={}, method_fqn={}, trace_id={}"); } else { lastFailures.incrementAndGet(); - //LOGGER.error("Could not persist trace by time", failure); + LOGGER.error("Could not persist trace by time", failure); } }); /*session.executeAsync(stmtByHash).exceptionally(failure -> { @@ -183,7 +180,7 @@ private void insertTrace(final PersistenceSpan span) { .log("Saved new trace with token={}, trace_id={}, and ten second epoch bucket={}"); } else { lastFailures.incrementAndGet(); - //LOGGER.error("Could not persist trace by time", failure); + LOGGER.error("Could not persist trace by time", failure); } }); } From 85cd4de350b23d77ad4d94f666ecac22713e118a Mon Sep 17 00:00:00 2001 From: Malte Hansen Date: Tue, 3 Dec 2024 14:12:42 +0100 Subject: [PATCH 11/11] Fix code formatting --- .../net/explorviz/span/landscape/loader/LandscapeRecord.java | 3 ++- .../java/net/explorviz/span/persistence/SpanConverter.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java b/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java index 2a4a658..cbabc90 100644 --- a/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java +++ b/src/main/java/net/explorviz/span/landscape/loader/LandscapeRecord.java @@ -45,7 +45,8 @@ public static LandscapeRecord fromRow(final Row row) { */ final String[] operationFqnSplit = methodFqn.split("\\."); - final String packageName = String.join(".", Arrays.copyOf(operationFqnSplit, operationFqnSplit.length - 2)); + final String packageName = String.join(".", Arrays.copyOf(operationFqnSplit, + operationFqnSplit.length - 2)); final String className = operationFqnSplit[operationFqnSplit.length - 2]; final String methodName = operationFqnSplit[operationFqnSplit.length - 1]; diff --git a/src/main/java/net/explorviz/span/persistence/SpanConverter.java b/src/main/java/net/explorviz/span/persistence/SpanConverter.java index aaecee5..65769b7 100644 --- a/src/main/java/net/explorviz/span/persistence/SpanConverter.java +++ b/src/main/java/net/explorviz/span/persistence/SpanConverter.java @@ -32,8 +32,8 @@ public PersistenceSpan apply(final Span span) { final String k8sNamespace = span.getK8sNamespace(); final String k8sDeploymentName = span.getK8sDeploymentName(); - final String methodHashCode = HashHelper.calculateSpanHash(landscapeToken, nodeIpAddress, applicationName, - applicationInstance, methodFqn, k8sPodName, k8sNodeName, k8sNamespace, + final String methodHashCode = HashHelper.calculateSpanHash(landscapeToken, nodeIpAddress, + applicationName, applicationInstance, methodFqn, k8sPodName, k8sNodeName, k8sNamespace, k8sDeploymentName); return new PersistenceSpan(landscapeToken, gitCommitChecksum, span.getSpanId(),