Merge branch '2.x' of github.com:opensearch-project/observability int…

…o backport/backport-1572-to-2.x Signed-off-by: Derek Ho <[email protected]>
opensearch-project · Jul 31, 2023 · 359f515 · 359f515
2 parents 5a7dded + 7b4b7d8
commit 359f515
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 162 deletions.
diff --git a/build.gradle b/build.gradle
@@ -10,7 +10,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "2.8.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.10.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
@@ -21,7 +21,7 @@ buildscript {
             opensearch_build += "-SNAPSHOT"
         }
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = System.getProperty("kotlin.version", "1.8.21")
+        kotlin_version = System.getProperty("kotlin.version", "1.6.0")
         jackson_version = "2.14.1"
     }
 
@@ -191,7 +191,6 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9"
     implementation "${group}:common-utils:${common_utils_version}"
-    implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     implementation 'org.json:json:20230227'
     implementation group: 'com.github.wnameless.json', name: 'json-flattener', version: '0.15.1'
     // json-base, jackson-databind, jackson-annotations are only used by json-flattener.

diff --git a/release-notes/opensearch-observability.release-notes-2.8.0.0.md b/release-notes/opensearch-observability.release-notes-2.8.0.0.md
@@ -0,0 +1,12 @@
+## Version 2.8.0.0 Release Notes
+
+Compatible with OpenSearch 2.8.0
+
+### Bug Fixes
+* fix guava jar hell issue ([#1536](https://github.com/opensearch-project/observability/pull/1536))
+
+### Infrastructure
+* Update Gradle Wrapper to 7.6.1 ([#1512](https://github.com/opensearch-project/observability/pull/1512))
+
+### Maintenance
+* Increment version to 2.8.0-SNAPSHOT ([#1505](https://github.com/opensearch-project/observability/pull/1505))
diff --git a/release-notes/opensearch-observability.release-notes-2.9.0.0.md b/release-notes/opensearch-observability.release-notes-2.9.0.0.md
@@ -0,0 +1,6 @@
+## Version 2.9.0.0 Release Notes
+
+Compatible with OpenSearch 2.9.0
+
+### Refactoring
+- Add class for loading mapping templates in bulk ([#1550](https://github.com/opensearch-project/observability/pull/1550))
diff --git a/src/main/kotlin/org/opensearch/observability/ObservabilityPlugin.kt b/src/main/kotlin/org/opensearch/observability/ObservabilityPlugin.kt
@@ -24,8 +24,7 @@ import org.opensearch.observability.action.DeleteObservabilityObjectAction
 import org.opensearch.observability.action.GetObservabilityObjectAction
 import org.opensearch.observability.action.UpdateObservabilityObjectAction
 import org.opensearch.observability.index.ObservabilityIndex
-import org.opensearch.observability.index.ObservabilityMetricsIndex
-import org.opensearch.observability.index.ObservabilityTracesIndex
+import org.opensearch.observability.index.ObservabilityIntegrationsIndex
 import org.opensearch.observability.resthandler.ObservabilityRestHandler
 import org.opensearch.observability.resthandler.ObservabilityStatsRestHandler
 import org.opensearch.observability.settings.PluginSettings
@@ -79,15 +78,13 @@ class ObservabilityPlugin : Plugin(), ActionPlugin, ClusterPlugin {
     ): Collection<Any> {
         PluginSettings.addSettingsUpdateConsumer(clusterService)
         ObservabilityIndex.initialize(client, clusterService)
-        ObservabilityMetricsIndex.initialize(client, clusterService)
-        ObservabilityTracesIndex.initialize(client, clusterService)
+        ObservabilityIntegrationsIndex.initialize(client, clusterService)
         return emptyList()
     }
 
     override fun onNodeStarted() {
         ObservabilityIndex.afterStart()
-        ObservabilityTracesIndex.afterStart()
-        ObservabilityMetricsIndex.afterStart()
+        ObservabilityIntegrationsIndex.afterStart()
     }
 
     /**

diff --git a/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt b/src/main/kotlin/org/opensearch/observability/action/UpdateObservabilityObjectRequest.kt
@@ -22,6 +22,8 @@ import org.opensearch.observability.model.BaseObjectData
 import org.opensearch.observability.model.ObservabilityObjectDataProperties
 import org.opensearch.observability.model.ObservabilityObjectType
 import org.opensearch.observability.model.RestTag.OBJECT_ID_FIELD
+import org.opensearch.observability.util.fieldIfNotNull
+import org.opensearch.observability.util.logger
 import java.io.IOException
 
 /**

diff --git a/...bility/index/ObservabilityMetricsIndex.kt → ...y/index/ObservabilityIntegrationsIndex.kt b/...bility/index/ObservabilityMetricsIndex.kt → ...y/index/ObservabilityIntegrationsIndex.kt
@@ -24,11 +24,8 @@ import java.util.*
 /**
  * Class for doing OpenSearch Metrics schema mapping & default index init operation
  */
-internal object ObservabilityMetricsIndex : LifecycleListener() {
-    private val log by logger(ObservabilityMetricsIndex::class.java)
-    private const val METRICS_MAPPING_TEMPLATE_NAME = "ss4o_metric_template"
-    private const val METRICS_MAPPING_TEMPLATE_FILE = "metrics-mapping-template.json"
-    private const val METRIC_PATTERN_NAME = "ss4o_metrics-*-*"
+internal object ObservabilityIntegrationsIndex : LifecycleListener() {
+    private val log by logger(ObservabilityIntegrationsIndex::class.java)
 
     private lateinit var client: Client
     private lateinit var clusterService: ClusterService
@@ -38,7 +35,7 @@ internal object ObservabilityMetricsIndex : LifecycleListener() {
      * @param client The OpenSearch client
      * @param clusterService The OpenSearch cluster service
      */
-    fun initialize(client: Client, clusterService: ClusterService): ObservabilityMetricsIndex {
+    fun initialize(client: Client, clusterService: ClusterService): ObservabilityIntegrationsIndex {
         this.client = SecureIndexClient(client)
         this.clusterService = clusterService
         return this
@@ -48,47 +45,59 @@ internal object ObservabilityMetricsIndex : LifecycleListener() {
      * once lifecycle indicate start has occurred - instantiating the mapping template
      */
     override fun afterStart() {
-        // create default mapping
-        createMappingTemplate()
+        // create default mappings
+        createMappingTemplates()
+    }
+
+    private fun getTemplateNames(): List<String> {
+        // TODO classloader doesn't support directory scanning by default, need to write manual traversal later
+        // Hardcoding template list for now, as a hotfix you can add names from resources/templates/{name}-mapping-template.json here
+        return listOf("metrics", "traces")
+    }
+
+    private fun createMappingTemplates() {
+        for (name in getTemplateNames()) {
+            createMappingTemplate("ss4o_${name}_template", "ss4o_$name-*-*", "templates/$name-mapping-template.json")
+        }
     }
 
     /**
      * Create the pre-defined mapping template
      */
     @Suppress("TooGenericExceptionCaught", "MagicNumber")
-    private fun createMappingTemplate() {
-        log.info("$LOG_PREFIX:createMappingTemplate $METRICS_MAPPING_TEMPLATE_NAME API called")
-        if (!isTemplateExists(METRICS_MAPPING_TEMPLATE_NAME)) {
-            val classLoader = ObservabilityMetricsIndex::class.java.classLoader
-            val indexMappingSource = classLoader.getResource(METRICS_MAPPING_TEMPLATE_FILE)?.readText()!!
+    private fun createMappingTemplate(templateName: String, patternName: String, path: String) {
+        log.info("$LOG_PREFIX:createMappingTemplate $templateName API called")
+        if (!isTemplateExists(templateName)) {
+            val classLoader = ObservabilityIntegrationsIndex::class.java.classLoader
+            val indexMappingSource = classLoader.getResource(path)?.readText()!!
             val settings = Settings.builder()
                 .put("index.number_of_shards", 3)
                 .put("index.auto_expand_replicas", "0-2")
                 .build()
             val template = Template(settings, CompressedXContent(indexMappingSource), null)
-            val request = PutComposableIndexTemplateAction.Request(METRICS_MAPPING_TEMPLATE_NAME)
+            val request = PutComposableIndexTemplateAction.Request(templateName)
                 .indexTemplate(
                     ComposableIndexTemplate(
-                        listOf(METRIC_PATTERN_NAME),
+                        listOf(patternName),
                         template,
                         Collections.emptyList(),
                         1,
                         1,
-                        Collections.singletonMap("description", "Observability Metrics Mapping Template") as Map<String, Any>?,
+                        Collections.singletonMap("description", "Observability $templateName Mapping Template") as Map<String, Any>?,
                         ComposableIndexTemplate.DataStreamTemplate()
                     )
                 )
             try {
                 val validationException = request.validateIndexTemplate(null)
                 if (validationException != null && !validationException.validationErrors().isEmpty()) {
-                    error("$LOG_PREFIX:Index Template $METRICS_MAPPING_TEMPLATE_NAME validation errors ${validationException.message}")
+                    error("$LOG_PREFIX:Index Template $templateName validation errors ${validationException.message}")
                 }
                 val actionFuture = client.admin().indices().execute(PutComposableIndexTemplateAction.INSTANCE, request)
                 val response = actionFuture.actionGet(PluginSettings.operationTimeoutMs)
                 if (response.isAcknowledged) {
-                    log.info("$LOG_PREFIX:Mapping Template $METRICS_MAPPING_TEMPLATE_NAME creation Acknowledged")
+                    log.info("$LOG_PREFIX:Mapping Template $templateName creation Acknowledged")
                 } else {
-                    error("$LOG_PREFIX:Mapping Template $METRICS_MAPPING_TEMPLATE_NAME creation not Acknowledged")
+                    error("$LOG_PREFIX:Mapping Template $templateName creation not Acknowledged")
                 }
             } catch (exception: ResourceAlreadyExistsException) {
                 log.warn("message: ${exception.message}")

diff --git a/src/main/kotlin/org/opensearch/observability/index/ObservabilityTracesIndex.kt b/src/main/kotlin/org/opensearch/observability/index/ObservabilityTracesIndex.kt
diff --git a/...n/resources/metrics-mapping-template.json → ...s/templates/metrics-mapping-template.json b/...n/resources/metrics-mapping-template.json → ...s/templates/metrics-mapping-template.json
diff --git a/...in/resources/traces-mapping-template.json → ...es/templates/traces-mapping-template.json b/...in/resources/traces-mapping-template.json → ...es/templates/traces-mapping-template.json
diff --git a/src/test/kotlin/org/opensearch/observability/rest/AssemblyValidationIT.kt b/src/test/kotlin/org/opensearch/observability/rest/AssemblyValidationIT.kt
@@ -12,31 +12,33 @@ import org.opensearch.rest.RestStatus
 
 class AssemblyValidationIT : PluginRestTestCase() {
     companion object {
-        private const val TRACES_MAPPING_TEMPLATE_NAME = "ss4o_trace_template"
-        private const val METRICS_MAPPING_TEMPLATE_NAME = "ss4o_metric_template"
+        private const val TRACES_MAPPING_TEMPLATE_NAME = "ss4o_traces_template"
+        private const val METRICS_MAPPING_TEMPLATE_NAME = "ss4o_metrics_template"
     }
 
-    fun `test observability traces template and was created`() {
-        // verify traces mapping template was created successfully as part of the plugin initialization
-        Thread.sleep(1000)
+    fun `test observability metrics template was created`() {
+        // verify metrics mapping template was created successfully as part of the plugin initialization
         var response = executeRequest(
             RestRequest.Method.GET.name,
-            "/_index_template/$TRACES_MAPPING_TEMPLATE_NAME",
-            "",
+            "/_index_template/$METRICS_MAPPING_TEMPLATE_NAME",
+            "{}",
             RestStatus.OK.status
         )
-        Thread.sleep(1000)
         Assert.assertNotNull(response.get("index_templates"))
         Assert.assertNotNull(!response.get("index_templates").asJsonArray.isEmpty)
 
-        // verify metrics mapping template was created successfully as part of the plugin initialization
+        // verify traces mapping template was created successfully as part of the plugin initialization
+        /*
+         * TODO I'm not sure why, but when this test is moved to its own test function, it fails.
+         * The new function passes if run alone, but not as part as a suite.
+         * Exponential backoff to very long sleep intervals doesn't work either.
+         */
         response = executeRequest(
             RestRequest.Method.GET.name,
-            "/_index_template/$METRICS_MAPPING_TEMPLATE_NAME",
-            "",
+            "/_index_template/$TRACES_MAPPING_TEMPLATE_NAME",
+            "{}",
             RestStatus.OK.status
         )
-        Thread.sleep(1000)
         Assert.assertNotNull(response.get("index_templates"))
         Assert.assertNotNull(!response.get("index_templates").asJsonArray.isEmpty)
     }