Skip to content

Commit d9bc335

Browse files
turboFeiturboFei
committed
comments
1 parent f493a2a commit d9bc335

File tree

7 files changed

+18
-26
lines changed

7 files changed

+18
-26
lines changed

‎docs/configuration/settings.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
387387
| kyuubi.metadata.request.async.retry.queue.size | 65536 | The maximum queue size for buffering metadata requests in memory when the external metadata storage is down. Requests will be dropped if the queue exceeds. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 |
388388
| kyuubi.metadata.request.async.retry.threads | 10 | Number of threads in the metadata request async retry manager thread pool. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 |
389389
| kyuubi.metadata.request.retry.interval | PT5S | The interval to check and trigger the metadata request retry tasks. | duration | 1.6.0 |
390-
| kyuubi.metadata.search.window | <undefined> | The time window to search the metadata from metadata store. | duration | 1.10.1 |
390+
| kyuubi.metadata.search.window | <undefined> | The time window to restrict user queries to metadata within a specific period. For example, if the window is set to P7D, only metadata from the past 7 days can be queried. If not set, it allows searching all metadata information in the metadata store. | duration | 1.10.1 |
391391
| kyuubi.metadata.store.class | org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore | Fully qualified class name for server metadata store. | string | 1.6.0 |
392392
| kyuubi.metadata.store.jdbc.database.schema.init | true | Whether to init the JDBC metadata store database schema. | boolean | 1.6.0 |
393393
| kyuubi.metadata.store.jdbc.database.type | SQLITE | The database type for server jdbc metadata store.<ul> <li>SQLITE: SQLite3, JDBC driver `org.sqlite.JDBC`.</li> <li>MYSQL: MySQL, JDBC driver `com.mysql.cj.jdbc.Driver` (fallback `com.mysql.jdbc.Driver`).</li> <li>POSTGRESQL: PostgreSQL, JDBC driver `org.postgresql.Driver`.</li> <li>CUSTOM: User-defined database type, need to specify corresponding JDBC driver.</li> Note that: The JDBC datasource is powered by HiKariCP, for datasource properties, please specify them with the prefix: kyuubi.metadata.store.jdbc.datasource. For example, kyuubi.metadata.store.jdbc.datasource.connectionTimeout=10000. | string | 1.6.0 |

‎kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

+3-1
Original file line numberDiff line numberDiff line change
@@ -2025,7 +2025,9 @@ object KyuubiConf {
20252025

20262026
val METADATA_SEARCH_WINDOW: OptionalConfigEntry[Long] =
20272027
buildConf("kyuubi.metadata.search.window")
2028-
.doc("The time window to search the metadata from metadata store.")
2028+
.doc("The time window to restrict user queries to metadata within a specific period. " +
2029+
"For example, if the window is set to P7D, only metadata from the past 7 days can be " +
2030+
"queried. If not set, it allows searching all metadata information in the metadata store.")
20292031
.version("1.10.1")
20302032
.timeConf
20312033
.checkValue(_ > 0, "must be positive number")

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala

+1-8
Original file line numberDiff line numberDiff line change
@@ -431,14 +431,7 @@ private[v1] class BatchesResource extends ApiRequestContext with Logging {
431431
requestName = batchName,
432432
createTime = createTimeFilter,
433433
endTime = endTime)
434-
val batches =
435-
sessionManager.getBatchesFromMetadataStore(
436-
filter,
437-
from,
438-
size,
439-
desc,
440-
// order by `create_time` rather then `key_id` if `create_time` is specified
441-
orderByKeyId = createTimeFilter == 0)
434+
val batches = sessionManager.getBatchesFromMetadataStore(filter, from, size, desc)
442435
new GetBatchesResponse(from, batches.size, batches.asJava)
443436
}
444437

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataManager.scala

+4-5
Original file line numberDiff line numberDiff line change
@@ -138,15 +138,14 @@ class MetadataManager extends AbstractService("MetadataManager") {
138138
filter: MetadataFilter,
139139
from: Int,
140140
size: Int,
141-
desc: Boolean = false,
142-
orderByKeyId: Boolean = true): Seq[Batch] = {
141+
desc: Boolean = false): Seq[Batch] = {
143142
withMetadataRequestMetrics(_metadataStore.getMetadataList(
144143
filter,
145144
from,
146145
size,
147-
desc,
148-
orderByKeyId)).map(
149-
buildBatch)
146+
// if create_file field is set, order by create_time, which is faster, otherwise by key_id
147+
orderBy = if (filter.createTime > 0) Some("create_time") else Some("key_id"),
148+
direction = if (desc) "DESC" else "ASC")).map(buildBatch)
150149
}
151150

152151
def countBatch(

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/MetadataStore.scala

+4-4
Original file line numberDiff line numberDiff line change
@@ -58,16 +58,16 @@ trait MetadataStore extends Closeable {
5858
* @param from the metadata offset.
5959
* @param size the size to get.
6060
* @param desc the order of metadata list.
61-
* @param orderByKeyId the result order by auto increment key_id, which is stable but might slow.
62-
* If false, order by create_time.
61+
* @param orderBy the order by column, default is the primary key, `key_id`.
62+
* @param direction the order direction, default is `ASC`.
6363
* @return selected metadata list.
6464
*/
6565
def getMetadataList(
6666
filter: MetadataFilter,
6767
from: Int,
6868
size: Int,
69-
desc: Boolean = false,
70-
orderByKeyId: Boolean = true): Seq[Metadata]
69+
orderBy: Option[String] = Some("key_id"),
70+
direction: String = "ASC"): Seq[Metadata]
7171

7272
/**
7373
* Count the metadata list with filter conditions.

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala

+3-4
Original file line numberDiff line numberDiff line change
@@ -257,16 +257,15 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging {
257257
filter: MetadataFilter,
258258
from: Int,
259259
size: Int,
260-
desc: Boolean = false,
261-
orderByKeyId: Boolean = true): Seq[Metadata] = {
260+
orderBy: Option[String] = Some("key_id"),
261+
direction: String = "ASC"): Seq[Metadata] = {
262262
val queryBuilder = new StringBuilder
263263
val params = ListBuffer[Any]()
264264
queryBuilder.append("SELECT ")
265265
queryBuilder.append(METADATA_COLUMNS)
266266
queryBuilder.append(s" FROM $METADATA_TABLE")
267267
queryBuilder.append(s" ${assembleWhereClause(filter, params)}")
268-
queryBuilder.append(" ORDER BY ").append(if (orderByKeyId) "key_id " else "create_time ")
269-
queryBuilder.append(if (desc) "DESC " else "ASC ")
268+
orderBy.foreach(o => queryBuilder.append(s" ORDER BY $o $direction "))
270269
queryBuilder.append(dialect.limitClause(size, from))
271270
val query = queryBuilder.toString
272271
JdbcUtils.withConnection { connection =>

‎kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala

+2-3
Original file line numberDiff line numberDiff line change
@@ -280,9 +280,8 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {
280280
filter: MetadataFilter,
281281
from: Int,
282282
size: Int,
283-
desc: Boolean = false,
284-
orderByKeyId: Boolean = true): Seq[Batch] = {
285-
metadataManager.map(_.getBatches(filter, from, size, desc, orderByKeyId)).getOrElse(Seq.empty)
283+
desc: Boolean = false): Seq[Batch] = {
284+
metadataManager.map(_.getBatches(filter, from, size, desc)).getOrElse(Seq.empty)
286285
}
287286

288287
def getBatchMetadata(batchId: String): Option[Metadata] = {

0 commit comments

Comments
 (0)