merge code v1.4.17

bi-service/build_docker_image.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 cd libs/ && ./install.sh && cd ..
 
-mvn clean package && docker build --no-cache -t datainsiderco/bi-service:oss .
+mvn clean package && docker build --no-cache -t datainsiderco/bi-service:main .

bi-service/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   bi_service:
-    image: datainsiderco/bi-service:oss
+    image: datainsiderco/bi-service:main
     container_name: rocket-bi-service
     restart: always
     environment:
@@ -14,8 +14,6 @@ services:
       CLICKHOUSE_USERNAME: default
       CLICKHOUSE_PASSWORD: ""
       CLICKHOUSE_CLUSTER_NAME: ""
-    ports:
-      - "8080:8080"
     volumes:
       - ./conf:/app/conf
       - ./logs:/app/logs

bi-service/pom.xml

@@ -198,7 +198,7 @@
         <dependency>
             <groupId>datainsider</groupId>
             <artifactId>clients</artifactId>
-            <version>2.2.29</version>
+            <version>2.2.31</version>
         </dependency>
 
         <dependency>

bi-service/src/main/scala/co/datainsider/bi/Server.scala

@@ -5,8 +5,7 @@ import co.datainsider.bi.controller.http.filter.{
   CORSFilter,
   CaseClassExceptionMapping,
   CommonExceptionMapping,
-  JsonParseExceptionMapping,
-  LicenceFilter
+  JsonParseExceptionMapping
 }
 import co.datainsider.bi.controller.thrift.TBIServiceController
 import co.datainsider.bi.module.{AccessFilterModule, BIServiceModule, TestModule}
@@ -24,8 +23,9 @@ import com.twitter.finatra.thrift.ThriftServer
 import com.twitter.finatra.thrift.routing.ThriftRouter
 import com.twitter.util.Await
 import datainsider.client.exception.DbExecuteError
-import datainsider.client.filter.{LoggedInUserParser, MustLoggedInFilter}
+import datainsider.client.filter.{LicenceFilter, LoggedInUserParser, MustLoggedInFilter}
 import datainsider.client.module.{CaasClientModule, MockCaasClientModule, MockSchemaClientModule, SchemaClientModule}
+import datainsider.licence.module.LicenceClientModule
 
 /**
   * Created by SangDang on 9/8/

bi-service/src/main/scala/co/datainsider/bi/controller/http/DashboardController.scala

@@ -5,10 +5,9 @@ import co.datainsider.bi.domain.Directory.{MyData, Shared}
 import co.datainsider.bi.domain.Ids.DirectoryId
 import co.datainsider.bi.domain.query.event.{ActionType, ResourceType}
 import co.datainsider.bi.domain.request._
-import co.datainsider.bi.service.{DashboardFieldService, DashboardService, DirectoryService, RecentDirectoryService}
-import co.datainsider.share.domain.response.PageResult
+import co.datainsider.bi.service.{DashboardFieldService, DashboardService, DirectoryService}
 import com.google.inject.Inject
-import com.twitter.finagle.http.{Request, Response}
+import com.twitter.finagle.http.Request
 import com.twitter.finatra.http.Controller
 import com.twitter.util.Future
 import datainsider.authorization.filters.{DashboardAccessFilters, WidgetAccessFilters}

bi-service/src/main/scala/co/datainsider/bi/controller/http/filter/ExceptionMapping.scala

@@ -9,15 +9,18 @@ import com.twitter.finatra.http.response.ResponseBuilder
 import com.twitter.finatra.jackson.caseclass.exceptions.CaseClassMappingException
 import com.twitter.inject.Logging
 import datainsider.client.exception.{DIErrorReason, DIException}
+
 import javax.inject.Inject
+import scala.util.Try
+
 /**
- * @author anhlt
- */
+  * @author anhlt
+  */
 @Singleton
 class CommonExceptionMapping @Inject() (
-                                         response: ResponseBuilder
-                                       ) extends ExceptionMapper[Throwable]
-  with Logging {
+    response: ResponseBuilder
+) extends ExceptionMapper[Throwable]
+    with Logging {
   override def toResponse(request: Request, ex: Throwable): Response = {
     logError(ex)
     val error = ex match {
@@ -26,7 +29,7 @@ class CommonExceptionMapping @Inject() (
           e.getStatus.code,
           e.reason,
           e.getMessage,
-          None
+          Try(e.getCause.getMessage).toOption
         )
       case _ =>
         ApiError(
@@ -42,11 +45,12 @@ class CommonExceptionMapping @Inject() (
     error(s"${ex.getClass.getName}: ${ex.getMessage}", ex)
   }
 }
+
 @Singleton
 class CaseClassExceptionMapping @Inject() (
-                                            response: ResponseBuilder
-                                          ) extends ExceptionMapper[CaseClassMappingException]
-  with Logging {
+    response: ResponseBuilder
+) extends ExceptionMapper[CaseClassMappingException]
+    with Logging {
   override def toResponse(request: Request, throwable: CaseClassMappingException): Response = {
     error("", throwable)
     response.badRequest.json(
@@ -60,9 +64,9 @@ class CaseClassExceptionMapping @Inject() (
 }
 @Singleton
 class JsonParseExceptionMapping @Inject() (
-                                            response: ResponseBuilder
-                                          ) extends ExceptionMapper[JsonParseException]
-  with Logging {
+    response: ResponseBuilder
+) extends ExceptionMapper[JsonParseException]
+    with Logging {
   override def toResponse(request: Request, ex: JsonParseException): Response = {
     error(s"JsonParseExceptionMapping: ${ex.getMessage}", ex)
     response.badRequest.json(
@@ -73,4 +77,4 @@ class JsonParseExceptionMapping @Inject() (
       )
     )
   }
-}
+}

bi-service/src/main/scala/co/datainsider/bi/controller/http/filter/LicenceServerFilter.scala

@@ -0,0 +1,55 @@
+package co.datainsider.bi.controller.http.filter
+
+import com.twitter.finagle.http.{Request, Response}
+import com.twitter.finagle.{Service, SimpleFilter}
+import com.twitter.inject.Logging
+import com.twitter.util.Future
+import datainsider.client.filter.UserContext.UserContextSyntax
+import datainsider.client.util.ZConfig
+import datainsider.licence.exception.LicenceExpiredError
+import datainsider.licence.service.LicenceClientService
+
+import java.sql.Date
+import java.text.SimpleDateFormat
+import java.time.Duration
+import javax.inject.Inject
+
+class LicenceServerFilter @Inject() (licenceClientService: LicenceClientService)
+    extends SimpleFilter[Request, Response]
+    with Logging {
+  val dateFormat = new SimpleDateFormat("dd MM yyyy HH:mm:ss Z")
+  override def apply(request: Request, service: Service[Request, Response]): Future[Response] = {
+    try {
+      //  val licenceKey = request.currentOrganization.get.licenceKey.get
+      val timeOfMonthMillis: Long = Duration.ofDays(30).toMillis
+      val timeOfDayMillis: Long = Duration.ofDays(1).toMillis
+
+      require(request.currentOrganization.get.licenceKey.isDefined)
+
+      for {
+        licence <- licenceClientService.get(request.currentOrganization.get.licenceKey.get)
+        response <- service(request)
+      } yield {
+        val terminatedTimeMillis = licence.expiredAt + timeOfMonthMillis
+        val terminatedDate = new Date(terminatedTimeMillis)
+        if (terminatedTimeMillis < System.currentTimeMillis()) {
+          throw LicenceExpiredError(s"Your license was terminated ${dateFormat.format(terminatedDate)}")
+        }
+
+        if (licence.expiredAt < System.currentTimeMillis()) {
+          val withinDays: Long = (terminatedTimeMillis - System.currentTimeMillis()) / timeOfDayMillis
+          response.headerMap.add(
+            ZConfig.getString("licence.field_name"),
+            s"Your license is expired. Please make sure to charge for your license before ${dateFormat
+              .format(terminatedDate)}/ within ${withinDays} days. Otherwise, your service will be terminated."
+          )
+        }
+        response
+      }
+    } catch {
+      case e: Throwable =>
+        logger.error(s"LicenceFilter::apply ${e.getMessage}", e)
+        service(request)
+    }
+  }
+}

bi-service/src/main/scala/co/datainsider/bi/domain/query/Query.scala

@@ -48,7 +48,7 @@ case class SqlQuery(
       TableExpressionUtils.findFromClause(query).map(resp => TableView(resp._1, resp._2))
     } catch {
       case e: Throwable =>
-        error(s"find dbName.tblName failed with exception: ${e}")
+        error(s"allQueryViews::find dbName.tblName failed with exception", e)
         Seq.empty
     }
   }

bi-service/src/main/scala/co/datainsider/bi/domain/query/SqlBuilder.scala

@@ -184,26 +184,26 @@ class SqlBuilder(sqlParser: SqlParser) {
     */
   private def buildFromClause(): String = {
     if (joinFields.nonEmpty) {
-      var fromClause = getSqlView(joinFields(0).leftViewName) // init
+      var curView = getSqlView(joinFields(0).leftViewName) // init
       val knownViews = mutable.Set[String](joinFields(0).leftViewName)
 
       joinFields.foreach(join => {
         var notChanged = true
 
         if (!knownViews.contains(join.leftViewName)) {
-          fromClause += s"\n  ${join.joinType.toString} join ${getSqlView(join.leftViewName)} on ${join.conditionStr}"
+          curView += s"\n  ${join.joinType.toString} join ${getSqlView(join.leftViewName)} on ${join.conditionStr}"
           knownViews += join.leftViewName
           notChanged = false
         } else if (!knownViews.contains(join.rightViewName)) {
-          fromClause += s"\n  ${join.joinType.toString} join ${getSqlView(join.rightViewName)} on ${join.conditionStr}"
+          curView += s"\n  ${join.joinType.toString} join ${getSqlView(join.rightViewName)} on ${join.conditionStr}"
           knownViews += join.rightViewName
           notChanged = false
         }
 
         if (notChanged) throw BadRequestError("invalid join clause")
       })
 
-      fromClause
+      curView
 
     } else {
       if (fromFields.size > 1) {

bi-service/src/main/scala/co/datainsider/bi/engine/clickhouse/ClickhouseEngine.scala

@@ -60,7 +60,7 @@ class ClickhouseEngine @Inject() (@Named("clickhouse") client: JdbcClient) exten
           DataTable(colNames, colTypes, rows.toArray)
         })
       } catch {
-        case e: SQLException => throw DbExecuteError(e.getMessage)
+        case e: SQLException => throw DbExecuteError(s"execute sql '$sql' failed with exception: $e")
         case e: Throwable    => throw InternalError(s"engine.execute failed with exception: $e")
       }
     }

bi-service/src/main/scala/co/datainsider/bi/engine/clickhouse/ClickhouseParser.scala

@@ -409,7 +409,7 @@ object ClickhouseParser extends SqlParser {
         case None    => toValueStr(value, fieldType)
       }
     } catch {
-      case e: Throwable => throw UnsupportedError(s"fail to parse $value to sql")
+      case e: Throwable => throw UnsupportedError(s"fail to parse $value to sql", e)
     }
   }
 

bi-service/src/main/scala/co/datainsider/bi/module/CustomResponseWriter.scala

@@ -9,16 +9,16 @@ import com.twitter.finatra.jackson.ScalaObjectMapper
 import com.twitter.inject.Logging
 
 case class ApiError(
-                     code: Int,
-                     reason: String,
-                     message: String,
-                     data: Option[Any] = None
-                   )
+    code: Int,
+    reason: String,
+    message: String,
+    cause: Option[String] = None
+)
 
-class CustomResponseWriterImpl @Inject()(
-                                          mapper: ScalaObjectMapper
-                                        ) extends DefaultMessageBodyWriter
-  with Logging {
+class CustomResponseWriterImpl @Inject() (
+    mapper: ScalaObjectMapper
+) extends DefaultMessageBodyWriter
+    with Logging {
   override def write(obj: Any): WriterResponse = {
     obj match {
       case ex: Throwable =>
@@ -44,4 +44,4 @@ class CustomResponseWriterImpl @Inject()(
         )
     }
   }
-}
+}

bi-service/src/main/scala/co/datainsider/bi/service/QueryService.scala

@@ -107,7 +107,7 @@ class QueryServiceImpl @Inject() (
         baseQuery,
         userProfile.map(_.username),
         limit,
-        request.filterRequests,
+        if (request.querySetting.isInstanceOf[FilterSetting]) Array.empty else request.filterRequests,
         relationshipInfo,
         rlsPolicies,
         Some(QueryContext(tableExpressions))

bi-service/src/main/scala/co/datainsider/bi/util/LicenceUtils.scala

@@ -28,7 +28,7 @@ object LicenceUtils extends Logging {
       val keyFactory = KeyFactory.getInstance("RSA")
       keyFactory.generatePublic(keySpec)
     } catch {
-      case e: Throwable => throw new Exception("invalid public key")
+      case e: Throwable => throw new Exception("invalid public key", e)
     }
   }
 

bi-service/src/test/scala/co/datainsider/bi/controller/DashboardControllerTest.scala

@@ -25,6 +25,8 @@ class DashboardControllerTest extends FeatureTest with BeforeAndAfterAll {
       from = 1,
       size = 100
     )
+    println(Serializer.toJson(request))
+
     val response = server.httpPost(
       s"$apiPath/list_drill_through",
       postBody = Serializer.toJson(request),

caas-service/build_docker_image.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 cd libs && ./install.sh && cd ../
 
-mvn clean package -DskipTests && docker build --no-cache -t registry.gitlab.com/datainsider/user-profile:oss .
+mvn clean package -DskipTests && docker build --no-cache -t datainsiderco/caas-service:main .

caas-service/conf/development.conf

@@ -41,6 +41,7 @@ caas {
   org_email {
     google = "^(.+)@datainsider.co"
   }
+  default_organization_id = 0
 
 }
 

caas-service/conf/local.conf

@@ -41,7 +41,7 @@ caas {
   org_email {
     google = "^(.+)@datainsider.co"
   }
-
+  default_organization_id = 0
 }
 
 db {

caas-service/conf/production.conf

@@ -41,6 +41,7 @@ caas {
   org_email {
     google = "^(.+)@datainsider.co"
   }
+  default_organization_id = 0
 
 }
 

caas-service/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.5'
 services:
   user-profile:
-    image: registry.gitlab.com/datainsider/user-profile:local
+    image: datainsiderco/caas-service:main
     container_name: rocket-user-profile
     restart: always
     environment:

caas-service/pom.xml

@@ -20,7 +20,7 @@
         <dependency>
             <groupId>datainsider</groupId>
             <artifactId>clients</artifactId>
-            <version>2.2.29</version>
+            <version>2.2.31</version>
 
             <exclusions>
                 <exclusion>