update oss v3.1.1

rocket-bi-server/Dockerfile

@@ -7,10 +7,16 @@ ENV MODE development
 ENV HTTP_PORT 8080
 ENV THRIFT_PORT 8084
 
+RUN curl -O https://repo.yandex.ru/clickhouse/tgz/stable/clickhouse-common-static-21.12.3.32.tgz
+RUN curl -O https://repo.yandex.ru/clickhouse/tgz/stable/clickhouse-client-21.12.3.32.tgz
+RUN tar -xzvf clickhouse-common-static-21.12.3.32.tgz
+RUN clickhouse-common-static-21.12.3.32/install/doinst.sh
+RUN tar -xzvf clickhouse-client-21.12.3.32.tgz
+RUN clickhouse-client-21.12.3.32/install/doinst.sh
+
 #install zip
 RUN apt-get update -qq
 RUN apt-get install -y zip
-RUN apt-get install -y clickhouse-client
 
 # install python3 for data-cook
 RUN apt-get install -y python3 python3-pip

rocket-bi-server/conf/development.conf

@@ -634,7 +634,7 @@ google {
   gg_client_id = ""
   gg_client_secret = ""
   server_encoded_url = "https://accounts.google.com/o/oauth2/token"
-  redirect_uri = "https%3A%2F%2Fhello.datainsider.co"
+  redirect_uri = "https%3A%2F%2Fhello.rocket.bi"
   read_timeout_ms = 300000
   connection_timeout_ms = 300000
   batch_size = 100000
@@ -810,8 +810,10 @@ client_pool {
   expire_time_ms = 300000 // 5 minutes
 }
 
-slack_notification {
-  webhook_url = ""
-}
-
 hidden_db_name_patterns = ["^(?:org\\d+_)?(?:preview_etl|etl)_\\d+$"]
+
+mailchimp {
+  dc = ""
+  api_key = ""
+  list_id = ""
+}

rocket-bi-server/conf/local.conf

@@ -634,7 +634,7 @@ google {
   gg_client_id = ""
   gg_client_secret = ""
   server_encoded_url = "https://accounts.google.com/o/oauth2/token"
-  redirect_uri = "https%3A%2F%2Fhello.datainsider.co"
+  redirect_uri = "https%3A%2F%2Fhello.rocket.bi"
   read_timeout_ms = 300000
   connection_timeout_ms = 300000
   batch_size = 100000
@@ -805,13 +805,21 @@ postgres_engine {
   max_query_rows = 10000
 }
 
+redshift_engine {
+  conn_timeout_ms = 30000
+  client_pool_size = 10
+  max_query_rows = 10000
+}
+
 client_pool {
   size = 100
   expire_time_ms = 300000 // 5 minutes
 }
 
-slack_notification {
-  webhook_url = ""
-}
-
 hidden_db_name_patterns = ["^(?:org\\d+_)?(?:preview_etl|etl)_\\d+$"]
+
+mailchimp {
+  dc = ""
+  api_key = ""
+  list_id = ""
+}

rocket-bi-server/conf/production.conf

@@ -634,7 +634,7 @@ google {
   gg_client_id = ""
   gg_client_secret = ""
   server_encoded_url = "https://accounts.google.com/o/oauth2/token"
-  redirect_uri = "https%3A%2F%2Fhello.datainsider.co"
+  redirect_uri = "https%3A%2F%2Fhello.rocket.bi"
   read_timeout_ms = 300000
   connection_timeout_ms = 300000
   batch_size = 100000
@@ -805,13 +805,21 @@ postgres_engine {
   max_query_rows = 10000
 }
 
+redshift_engine {
+  conn_timeout_ms = 30000
+  client_pool_size = 10
+  max_query_rows = 10000
+}
+
 client_pool {
   size = 100
   expire_time_ms = 300000 // 5 minutes
 }
 
-slack_notification {
-  webhook_url = ""
-}
-
 hidden_db_name_patterns = ["^(?:org\\d+_)?(?:preview_etl|etl)_\\d+$"]
+
+mailchimp {
+  dc = ""
+  api_key = ""
+  list_id = ""
+}

rocket-bi-server/pom.xml

@@ -386,6 +386,13 @@
             <version>v4-rev612-1.25.0</version>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/com.google.apis/google-api-services-searchconsole -->
+        <dependency>
+            <groupId>com.google.apis</groupId>
+            <artifactId>google-api-services-searchconsole</artifactId>
+            <version>v1-rev20230701-2.0.0</version>
+        </dependency>
+
         <!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42-no-awssdk -->
         <dependency>
             <groupId>com.amazon.redshift</groupId>

rocket-bi-server/src/main/scala/co/datainsider/bi/Server.scala

@@ -4,7 +4,6 @@ import co.datainsider.bi.controller.http._
 import co.datainsider.bi.controller.http.filter._
 import co.datainsider.bi.module.{BIServiceModule, TestContainerModule, TestModule}
 import co.datainsider.bi.repository.SchemaManager
-import co.datainsider.bi.service.BoostScheduleService
 import co.datainsider.bi.util.ZConfig
 import co.datainsider.caas.admin.controller.http.{
   AdminUserController,
@@ -157,9 +156,6 @@ class Server extends HttpServer {
       case false => throw DbExecuteError("invalid database schema")
     }
     Await.result(preparedSchema)
-
-    val boostScheduleService = injector.instance[BoostScheduleService]
-    boostScheduleService.start()
   }
 
   private def warmupSchemaService(): Unit = {

rocket-bi-server/src/main/scala/co/datainsider/bi/client/BigQueryClient.scala

@@ -30,7 +30,7 @@ class BigQueryClient(val bigquery: BigQuery, maxQueryRows: Int = 10000, defaultT
       converter(tableResult)
     } catch {
       case e: Throwable =>
-        throw InternalError(s"execute query error, message: ${e.getMessage}", e)
+        throw InternalError(s"execute query error, message: ${e.getMessage}, sql: $sql", e)
     }
   }
 

rocket-bi-server/src/main/scala/co/datainsider/bi/client/MailChimpClient.scala

@@ -0,0 +1,57 @@
+package co.datainsider.bi.client
+
+import co.datainsider.bi.util.{Serializer, StringUtils, ZConfig}
+import com.fasterxml.jackson.databind.JsonNode
+import datainsider.client.exception.InternalError
+import scalaj.http.{Http, HttpResponse}
+
+object MailChimpClient {
+  private val dc: String = ZConfig.getString("mailchimp.dc")
+  private val apiKey: String = ZConfig.getString("mailchimp.api_key")
+  private val listId: String = ZConfig.getString("mailchimp.list_id")
+
+  def apply(): MailChimpClient = new MailChimpClient(dc, apiKey, listId)
+}
+
+class MailChimpClient(dc: String, apiKey: String, listId: String) {
+  private val baseUrl = s"https://$dc.api.mailchimp.com/3.0"
+
+  def ping(): String = {
+    val resp: HttpResponse[String] =
+      Http(s"$baseUrl/ping").method("GET").auth("key", apiKey).asString
+
+    extract[String](resp)(resp => resp)
+  }
+
+  def addMember(email: String, firstName: String, lastName: String): Boolean = {
+    val memberData =
+      s"""
+        |{
+        |  "email_address": "$email",
+        |  "status": "subscribed",
+        |  "merge_fields": {
+        |    "FNAME": "$firstName",
+        |    "LNAME": "$lastName"
+        |  }
+        |}
+        |""".stripMargin
+
+    val resp: HttpResponse[String] =
+      Http(s"$baseUrl/lists/$listId/members/${StringUtils.md5(email)}")
+        .method("PUT")
+        .auth("key", apiKey)
+        .put(memberData)
+        .asString
+
+    extract[Boolean](resp)(resp => {
+      val jsonNode = Serializer.fromJson[JsonNode](resp)
+      jsonNode.get("id").asText() != null
+    })
+  }
+
+  private def extract[T](resp: HttpResponse[String])(converter: String => T): T = {
+    if (resp.isSuccess) {
+      converter(resp.body)
+    } else throw InternalError(s"call to MailChimp fail with code: ${resp.code}, message: ${resp.body}")
+  }
+}

rocket-bi-server/src/main/scala/co/datainsider/bi/controller/http/HealthController.scala

@@ -1,7 +1,6 @@
 package co.datainsider.bi.controller.http
 
 import co.datainsider.bi.domain.request.ContactUsRequest
-import co.datainsider.bi.util.SlackUtils
 import co.datainsider.bi.util.profiler.Profiler
 import co.datainsider.caas.user_profile.controller.http.filter.parser.UserContext.UserContextSyntax
 import co.datainsider.caas.user_profile.util.JsonParser
@@ -39,8 +38,7 @@ class HealthController @Inject() (licenseClientService: LicenseClientService) ex
   post("/contact_us") { request: ContactUsRequest =>
     {
       val message = s"New customer's contact submission:\n ${JsonParser.toJson(request)}"
-      SlackUtils.send(message = message)
-      response.ok
+      licenseClientService.notify(message = message)
     }
   }
 

rocket-bi-server/src/main/scala/co/datainsider/bi/domain/chart/ChartSetting.scala

@@ -564,8 +564,10 @@ case class TabControlChartSetting(
   override def toTableColumns: Array[TableColumn] = Array(getLabelColumn, getValueColumn)
 
   override def toQuery: Query = {
+    val func1 = getLabelColumn.function.asInstanceOf[FieldRelatedFunction].customCopy(Some("val1"))
+    val func2 = getLabelColumn.function.asInstanceOf[FieldRelatedFunction].customCopy(Some("val2"))
     val builder = new ObjectQueryBuilder
-    builder.addFunctions(Array(getLabelColumn.function, getValueColumn.function))
+    builder.addFunctions(Array(func1, func2))
     builder.addCondition(Or(filters))
     builder.addOrders(sorts)
     builder.addViews(sqlViews)

rocket-bi-server/src/main/scala/co/datainsider/bi/domain/query/Condition.scala

@@ -2,6 +2,7 @@ package co.datainsider.bi.domain.query
 
 import com.fasterxml.jackson.annotation.JsonSubTypes.Type
 import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo}
+import com.fasterxml.jackson.databind.JsonNode
 
 @JsonTypeInfo(
   use = JsonTypeInfo.Id.NAME,
@@ -360,7 +361,8 @@ case class NotEqualField(
 case class DynamicCondition(
     dynamicWidgetId: Long,
     baseCondition: Condition,
-    finalCondition: Option[Condition] = None
+    finalCondition: Option[Condition] = None,
+    extraData: Option[JsonNode] = None
 ) extends ControlCondition
 
 case class AlwaysTrue() extends ControlCondition

rocket-bi-server/src/main/scala/co/datainsider/bi/domain/query/Field.scala

@@ -34,6 +34,8 @@ abstract class Field {
   @JsonIgnore
   def fullFieldName: String = s"$aliasViewName.$fieldName"
 
+  def fullFieldNameWithEscape: String = s"$aliasViewName.`$fieldName`"
+
   @JsonIgnoreProperties
   @JsonIgnore
   def normalizedFieldName: String = s"${aliasViewName.replace('.', '_')}_$fieldName"
@@ -95,6 +97,8 @@ case class ExpressionField(
 
   override val fullFieldName: String = fieldName
 
+  override val fullFieldNameWithEscape: String = fieldName
+
   val aliasViewName: String = tblAliasName.getOrElse("tbl_" + StringUtils.shortMd5(s"$dbName.$tblName"))
 
   override def customCopy(aliasViewName: String, fieldName: String, fieldType: String): Field = {
@@ -114,6 +118,8 @@ case class CalculatedField(
 
   override val fullFieldName: String = fieldName
 
+  override val fullFieldNameWithEscape: String = fieldName
+
   val aliasViewName: String = tblAliasName.getOrElse("tbl_" + StringUtils.shortMd5(s"$dbName.$tblName"))
 
   override def customCopy(aliasViewName: String, fieldName: String, fieldType: String): Field = {