Skip to content

Commit 95a8de6

Browse files
turboFeiturboFei
committed
in memory session
1 parent 510a30b commit 95a8de6

File tree

4 files changed

+16
-23
lines changed

4 files changed

+16
-23
lines changed

‎docs/configuration/settings.md

-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
8383
| kyuubi.batch.application.starvation.timeout | PT3M | Threshold above which to warn batch application may be starved. | duration | 1.7.0 |
8484
| kyuubi.batch.conf.ignore.list || A comma-separated list of ignored keys for batch conf. If the batch conf contains any of them, the key and the corresponding value will be removed silently during batch job submission. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering. You can also pre-define some config for batch job submission with the prefix: kyuubi.batchConf.[batchType]. For example, you can pre-define `spark.master` for the Spark batch job with key `kyuubi.batchConf.spark.spark.master`. | set | 1.6.0 |
8585
| kyuubi.batch.extra.resource.file.max.size | 0 | The maximum size in bytes of each uploaded extra resource file when creating batch. 0 or negative value means no limit. | long | 1.10.0 |
86-
| kyuubi.batch.pending.check.window | PT24H | The time window to check the batch pending max elapse time from metadata store. | duration | 1.10.1 |
8786
| kyuubi.batch.resource.file.max.size | 0 | The maximum size in bytes of the uploaded resource file when creating batch. 0 or negative value means no limit. | long | 1.10.0 |
8887
| kyuubi.batch.search.window | <undefined> | The time window to search the batch from metadata store. | duration | 1.10.1 |
8988
| kyuubi.batch.session.idle.timeout | PT6H | Batch session idle timeout, it will be closed when it's not accessed for this duration | duration | 1.6.2 |

‎kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

-8
Original file line numberDiff line numberDiff line change
@@ -1930,14 +1930,6 @@ object KyuubiConf {
19301930
.stringConf
19311931
.createWithDefault("1")
19321932

1933-
val BATCH_PENDING_CHECK_WINDOW: ConfigEntry[Long] =
1934-
buildConf("kyuubi.batch.pending.check.window")
1935-
.doc("The time window to check the batch pending max elapse time from metadata store.")
1936-
.version("1.10.1")
1937-
.timeConf
1938-
.checkValue(_ > 0, "must be positive number")
1939-
.createWithDefault(Duration.ofDays(1).toMillis)
1940-
19411933
val BATCH_SEARCH_WINDOW: OptionalConfigEntry[Long] =
19421934
buildConf("kyuubi.batch.search.window")
19431935
.doc("The time window to search the batch from metadata store.")

‎kyuubi-server/src/main/scala/org/apache/kyuubi/operation/BatchJobSubmission.scala

+8
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,14 @@ class BatchJobSubmission(
424424
Utils.deleteDirectoryRecursively(session.resourceUploadFolderPath.toFile)
425425
}
426426
}
427+
428+
def getPendingElapsedTime: Long = {
429+
if (state == OperationState.PENDING) {
430+
System.currentTimeMillis() - createTime
431+
} else {
432+
0L
433+
}
434+
}
427435
}
428436

429437
object BatchJobSubmission {

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiRestFrontendService.scala

+8-14
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,14 @@ import org.eclipse.jetty.servlet.{ErrorPageErrorHandler, FilterHolder}
3131
import org.apache.kyuubi.{KyuubiException, Utils}
3232
import org.apache.kyuubi.config.KyuubiConf
3333
import org.apache.kyuubi.config.KyuubiConf._
34-
import org.apache.kyuubi.metrics.MetricsConstants
34+
import org.apache.kyuubi.metrics.MetricsConstants.OPERATION_BATCH_PENDING_MAX_ELAPSE
3535
import org.apache.kyuubi.metrics.MetricsSystem
36-
import org.apache.kyuubi.operation.OperationState
3736
import org.apache.kyuubi.server.api.v1.ApiRootResource
3837
import org.apache.kyuubi.server.http.authentication.{AuthenticationFilter, KyuubiHttpAuthenticationFactory}
39-
import org.apache.kyuubi.server.metadata.api.MetadataFilter
4038
import org.apache.kyuubi.server.ui.{JettyServer, JettyUtils}
4139
import org.apache.kyuubi.service.{AbstractFrontendService, Serverable, Service, ServiceUtils}
4240
import org.apache.kyuubi.service.authentication.{AuthTypes, AuthUtils}
43-
import org.apache.kyuubi.session.{KyuubiSessionManager, SessionHandle}
41+
import org.apache.kyuubi.session.{KyuubiBatchSession, KyuubiSessionManager, SessionHandle}
4442
import org.apache.kyuubi.util.{JavaUtils, ThreadUtils}
4543
import org.apache.kyuubi.util.ThreadUtils.scheduleTolerableRunnableWithFixedDelay
4644

@@ -207,12 +205,11 @@ class KyuubiRestFrontendService(override val serverable: Serverable)
207205
}
208206

209207
private def getBatchPendingMaxElapse(): Long = {
210-
val filter = MetadataFilter(
211-
state = OperationState.PENDING.toString,
212-
kyuubiInstance = connectionUrl,
213-
createTime = System.currentTimeMillis() - conf.get(BATCH_PENDING_CHECK_WINDOW))
214-
sessionManager.getBatchesFromMetadataStore(filter, 0, 1, desc = false, orderByKeyId = false)
215-
.headOption.map { batch => System.currentTimeMillis() - batch.getCreateTime }.getOrElse(0L)
208+
val batchPendingElapseTimes = sessionManager.allSessions().map {
209+
case session: KyuubiBatchSession => session.batchJobSubmissionOp.getPendingElapsedTime
210+
case _ => 0L
211+
}
212+
if (batchPendingElapseTimes.isEmpty) 0L else batchPendingElapseTimes.max
216213
}
217214

218215
def waitForServerStarted(): Unit = {
@@ -234,10 +231,7 @@ class KyuubiRestFrontendService(override val serverable: Serverable)
234231
startBatchChecker()
235232
recoverBatchSessions()
236233
MetricsSystem.tracing { ms =>
237-
ms.registerGauge(
238-
MetricsConstants.OPERATION_BATCH_PENDING_MAX_ELAPSE,
239-
getBatchPendingMaxElapse,
240-
0)
234+
ms.registerGauge(OPERATION_BATCH_PENDING_MAX_ELAPSE, getBatchPendingMaxElapse, 0)
241235
}
242236
} catch {
243237
case e: Exception => throw new KyuubiException(s"Cannot start $getName", e)

0 commit comments

Comments
 (0)