Skip to content
This repository has been archived by the owner on Oct 31, 2018. It is now read-only.

Commit

Permalink
Added instrumentation for compute monotask time.
Browse files Browse the repository at this point in the history
This commit adds a field to TaskMetrics to record the total time used
by ComputeMonotasks for each macrotask.

Conflicts:
	core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
  • Loading branch information
kayousterhout authored and Christopher Canel committed Apr 22, 2015
1 parent cd3607f commit 4bb99a9
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,14 @@ class TaskMetrics extends Serializable {
def executorRunTime = _executorRunTime
private[spark] def setExecutorRunTime(value: Long) = _executorRunTime = value

/**
* Total time consumed by compute monotasks for this macrotask. May be larger than executorRunTime
* if multiple compute monotasks ran simultaneously.
*/
private var _computationNanos: Long = _
def computationNanos: Long = _computationNanos
private[spark] def setComputationNanos(value: Long) = _computationNanos = value

/**
* The number of bytes this task transmitted back to the driver as the TaskResult
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ private[spark] abstract class ComputeMonotask(context: TaskContextImpl)

/** Runs the execute method and handles common exceptions thrown by ComputeMonotasks. */
def executeAndHandleExceptions() {
val startTimeNanos = System.nanoTime()
try {
Accumulators.registeredAccumulables.set(context.accumulators)
TaskContext.setTaskContext(context)
Expand Down Expand Up @@ -73,6 +74,8 @@ private[spark] abstract class ComputeMonotask(context: TaskContextImpl)
val closureSerializer = context.env.closureSerializer.newInstance()
context.localDagScheduler.handleTaskFailure(this, closureSerializer.serialize(reason))
}
} finally {
context.taskMetrics.setComputationNanos(System.nanoTime() - startTimeNanos)
}
}
}
3 changes: 3 additions & 0 deletions core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ private[spark] object JsonProtocol {
("Host Name" -> taskMetrics.hostname) ~
("Executor Deserialize Time" -> taskMetrics.executorDeserializeTime) ~
("Executor Run Time" -> taskMetrics.executorRunTime) ~
("Computation Nanos" -> taskMetrics.computationNanos) ~
("Result Size" -> taskMetrics.resultSize) ~
("JVM GC Time" -> taskMetrics.jvmGCTime) ~
("Result Serialization Time" -> taskMetrics.resultSerializationTime) ~
Expand Down Expand Up @@ -721,6 +722,8 @@ private[spark] object JsonProtocol {
metrics.setHostname((json \ "Host Name").extract[String])
metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Long])
metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Long])
metrics.setComputationNanos(
Utils.jsonOption(json \ "Computation Nanos").map(_.extract[Long]).getOrElse(0L))
metrics.setResultSize((json \ "Result Size").extract[Long])
metrics.setJvmGCTime((json \ "JVM GC Time").extract[Long])
metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Long])
Expand Down

0 comments on commit 4bb99a9

Please sign in to comment.