Skip to content
This repository has been archived by the owner on Oct 31, 2018. It is now read-only.

Commit

Permalink
Added instrumentation for compute monotask time.
Browse files Browse the repository at this point in the history
This commit adds a field to TaskMetrics to record the total time used
by ComputeMonotasks for each macrotask.
  • Loading branch information
kayousterhout committed Mar 18, 2015
1 parent d026a19 commit dcafb0e
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ class TaskMetrics extends Serializable {
*/
var executorRunTime: Long = _

/**
* Total time consumed by compute monotasks for this macrotask. May be larger than executorRunTime
* if multiple compute monotasks ran simultaneously.
*/
var computationNanos: Long = _

/**
* The number of bytes this task transmitted back to the driver as the TaskResult
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ private[spark] abstract class ComputeMonotask(context: TaskContext)

/** Runs the execute method and handles common exceptions thrown by ComputeMonotasks. */
def executeAndHandleExceptions() {
val startTimeNanos = System.nanoTime()
try {
Accumulators.registeredAccumulables.set(context.accumulators)
val result = execute()
Expand Down Expand Up @@ -66,6 +67,8 @@ private[spark] abstract class ComputeMonotask(context: TaskContext)
val closureSerializer = context.env.closureSerializer.newInstance()
context.localDagScheduler.handleTaskFailure(this, closureSerializer.serialize(reason))
}
} finally {
context.taskMetrics.computationNanos = System.nanoTime() - startTimeNanos
}
}
}
3 changes: 3 additions & 0 deletions core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ private[spark] object JsonProtocol {
("Host Name" -> taskMetrics.hostname) ~
("Executor Deserialize Time" -> taskMetrics.executorDeserializeTime) ~
("Executor Run Time" -> taskMetrics.executorRunTime) ~
("Computation Nanos" -> taskMetrics.computationNanos) ~
("Result Size" -> taskMetrics.resultSize) ~
("JVM GC Time" -> taskMetrics.jvmGCTime) ~
("Result Serialization Time" -> taskMetrics.resultSerializationTime) ~
Expand Down Expand Up @@ -619,6 +620,8 @@ private[spark] object JsonProtocol {
metrics.hostname = (json \ "Host Name").extract[String]
metrics.executorDeserializeTime = (json \ "Executor Deserialize Time").extract[Long]
metrics.executorRunTime = (json \ "Executor Run Time").extract[Long]
metrics.computationNanos =
Utils.jsonOption(json \ "Computation Nanos").map(_.extract[Long]).getOrElse(0L)
metrics.resultSize = (json \ "Result Size").extract[Long]
metrics.jvmGCTime = (json \ "JVM GC Time").extract[Long]
metrics.resultSerializationTime = (json \ "Result Serialization Time").extract[Long]
Expand Down

0 comments on commit dcafb0e

Please sign in to comment.