benchmark: Avoid spawning a goroutine per unary call (#8591)

The benchmark client is presently spawning a new goroutine per unary
call and blocking on its completion. Since the spawning goroutine is
blocked, it is more efficient to do the work in the spawning goroutine
itself. This change has the following effect on the [benchmark
performance](https://grafana-dot-grpc-testing.appspot.com/):
1. Unary 8-core: 184k QPS to 233k QPS (+26%)
2. Unary 30-core: 403k QPS to 624k QPS (+54%)

## Tested
* Ran the benchmark on the same GKE cluster to repro the results from
the dashboard.
* Created a docker image with the changes in this PR. Re-ran the
benchmark with the new image.

RELEASE NOTES: N/A

Author: arjan-bal

benchmark/worker/benchmark_client.go

@@ -262,7 +262,7 @@ func (bc *benchmarkClient) unaryLoop(conns []*grpc.ClientConn, rpcCountPerConn i
 	for ic, conn := range conns {
 		client := testgrpc.NewBenchmarkServiceClient(conn)
 		// For each connection, create rpcCountPerConn goroutines to do rpc.
-		for j := 0; j < rpcCountPerConn; j++ {
+		for j := range rpcCountPerConn {
 			// Create histogram for each goroutine.
 			idx := ic*rpcCountPerConn + j
 			bc.lockingHistograms[idx].histogram = stats.NewHistogram(bc.histogramOptions)
@@ -273,29 +273,18 @@ func (bc *benchmarkClient) unaryLoop(conns []*grpc.ClientConn, rpcCountPerConn i
 				// The worker client needs to wait for some time after client is created,
 				// before starting benchmark.
 				if poissonLambda == nil { // Closed loop.
-					done := make(chan bool)
 					for {
-						go func() {
-							start := time.Now()
-							if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil {
-								select {
-								case <-bc.stop:
-								case done <- false:
-								}
-								return
-							}
-							elapse := time.Since(start)
-							bc.lockingHistograms[idx].add(int64(elapse))
-							select {
-							case <-bc.stop:
-							case done <- true:
-							}
-						}()
 						select {
 						case <-bc.stop:
 							return
-						case <-done:
+						default:
+						}
+						start := time.Now()
+						if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil {
+							continue
 						}
+						elapse := time.Since(start)
+						bc.lockingHistograms[idx].add(int64(elapse))
 					}
 				} else { // Open loop.
 					timeBetweenRPCs := time.Duration((rand.ExpFloat64() / *poissonLambda) * float64(time.Second))