Merge pull request #475 from tpepper/master

more test race / deadlock hunting

ciao-scheduler/scheduler_ssntp_test.go

@@ -387,7 +387,7 @@ func stopServer() error {
 	netAgentCh := netAgent.AddEventChan(ssntp.NodeDisconnected)
 	agentCh := agent.AddEventChan(ssntp.NodeDisconnected)
 
-	server.ssntp.Stop()
+	go server.ssntp.Stop()
 
 	_, err := controller.GetEventChanResult(controllerCh, ssntp.NodeDisconnected)
 	if err != nil {

testutil/agent.go

@@ -19,6 +19,7 @@ package testutil
 import (
 	"errors"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -485,7 +486,7 @@ func (client *SsntpTestClient) CommandNotify(command ssntp.Command, frame *ssntp
 		result = client.handleDelete(payload)
 
 	default:
-		fmt.Printf("client %s unhandled command %s\n", client.Role.String(), command.String())
+		fmt.Fprintf(os.Stderr, "client %s unhandled command %s\n", client.Role.String(), command.String())
 	}
 
 	client.SendResultAndDelCmdChan(command, result)
@@ -511,7 +512,7 @@ func (client *SsntpTestClient) EventNotify(event ssntp.Event, frame *ssntp.Frame
 			result.Err = err
 		}
 	default:
-		fmt.Printf("client %s unhandled event: %s\n", client.Role.String(), event.String())
+		fmt.Fprintf(os.Stderr, "client %s unhandled event: %s\n", client.Role.String(), event.String())
 	}
 
 	client.SendResultAndDelEventChan(event, result)
@@ -695,7 +696,7 @@ func (client *SsntpTestClient) sendStartFailure(instanceUUID string, reason payl
 
 	_, err = client.Ssntp.SendError(ssntp.StartFailure, y)
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 	}
 }
 
@@ -712,7 +713,7 @@ func (client *SsntpTestClient) sendStopFailure(instanceUUID string, reason paylo
 
 	_, err = client.Ssntp.SendError(ssntp.StopFailure, y)
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 	}
 }
 
@@ -729,7 +730,7 @@ func (client *SsntpTestClient) sendRestartFailure(instanceUUID string, reason pa
 
 	_, err = client.Ssntp.SendError(ssntp.RestartFailure, y)
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 	}
 }
 
@@ -746,6 +747,6 @@ func (client *SsntpTestClient) sendDeleteFailure(instanceUUID string, reason pay
 
 	_, err = client.Ssntp.SendError(ssntp.DeleteFailure, y)
 	if err != nil {
-		fmt.Println(err)
+		fmt.Fprintln(os.Stderr, err)
 	}
 }

testutil/client_server_test.go

@@ -115,7 +115,7 @@ func TestStartFailure(t *testing.T) {
 
 	serverErrorCh := server.AddErrorChan(ssntp.StartFailure)
 	controllerErrorCh := controller.AddErrorChan(ssntp.StartFailure)
-	fmt.Printf("Expecting server and controller to note: \"%s\"\n", ssntp.StartFailure)
+	fmt.Fprintf(os.Stderr, "Expecting server and controller to note: \"%s\"\n", ssntp.StartFailure)
 
 	agent.StartFail = true
 	agent.StartFailReason = payloads.FullCloud
@@ -251,7 +251,7 @@ func TestStopFailure(t *testing.T) {
 
 	serverErrorCh := server.AddErrorChan(ssntp.StopFailure)
 	controllerErrorCh := controller.AddErrorChan(ssntp.StopFailure)
-	fmt.Printf("Expecting server and controller to note: \"%s\"\n", ssntp.StopFailure)
+	fmt.Fprintf(os.Stderr, "Expecting server and controller to note: \"%s\"\n", ssntp.StopFailure)
 
 	agent.StopFail = true
 	agent.StopFailReason = payloads.StopNoInstance
@@ -306,7 +306,7 @@ func TestRestartFailure(t *testing.T) {
 
 	serverErrorCh := server.AddErrorChan(ssntp.RestartFailure)
 	controllerErrorCh := controller.AddErrorChan(ssntp.RestartFailure)
-	fmt.Printf("Expecting server and controller to note: \"%s\"\n", ssntp.RestartFailure)
+	fmt.Fprintf(os.Stderr, "Expecting server and controller to note: \"%s\"\n", ssntp.RestartFailure)
 
 	agent.RestartFail = true
 	agent.RestartFailReason = payloads.RestartNoInstance
@@ -346,7 +346,7 @@ func doDelete(fail bool) error {
 	if fail == true {
 		serverErrorCh = server.AddErrorChan(ssntp.DeleteFailure)
 		controllerErrorCh = controller.AddErrorChan(ssntp.DeleteFailure)
-		fmt.Printf("Expecting server and controller to note: \"%s\"\n", ssntp.DeleteFailure)
+		fmt.Fprintf(os.Stderr, "Expecting server and controller to note: \"%s\"\n", ssntp.DeleteFailure)
 
 		agent.DeleteFail = true
 		agent.DeleteFailReason = payloads.DeleteNoInstance
@@ -511,15 +511,15 @@ func restartServer() error {
 }
 
 func TestReconnects(t *testing.T) {
-	fmt.Println("stopping server")
+	fmt.Fprintln(os.Stderr, "stopping server")
 	err := stopServer()
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	time.Sleep(1 * time.Second)
 
-	fmt.Println("restarting server")
+	fmt.Fprintln(os.Stderr, "restarting server")
 	err = restartServer()
 	if err != nil {
 		t.Fatal(err)

testutil/controller.go

@@ -18,6 +18,7 @@ package testutil
 import (
 	"errors"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -275,7 +276,7 @@ func (ctl *SsntpTestController) CommandNotify(command ssntp.Command, frame *ssnt
 		}
 
 	default:
-		fmt.Printf("controller unhandled command: %s\n", command.String())
+		fmt.Fprintf(os.Stderr, "controller unhandled command: %s\n", command.String())
 	}
 
 	ctl.SendResultAndDelCmdChan(command, result)
@@ -319,7 +320,7 @@ func (ctl *SsntpTestController) EventNotify(event ssntp.Event, frame *ssntp.Fram
 			result.Err = err
 		}
 	default:
-		fmt.Printf("controller unhandled event: %s\n", event.String())
+		fmt.Fprintf(os.Stderr, "controller unhandled event: %s\n", event.String())
 	}
 
 	ctl.SendResultAndDelEventChan(event, result)
@@ -343,7 +344,7 @@ func (ctl *SsntpTestController) ErrorNotify(error ssntp.Error, frame *ssntp.Fram
 	case ssntp.InvalidConfiguration:
 	*/
 	default:
-		fmt.Printf("controller unhandled error %s\n", error.String())
+		fmt.Fprintf(os.Stderr, "controller unhandled error %s\n", error.String())
 	}
 
 	ctl.SendResultAndDelErrorChan(error, result)

testutil/server.go

@@ -19,6 +19,7 @@ package testutil
 import (
 	"fmt"
 	"math/rand"
+	"os"
 	"sync"
 	"time"
 
@@ -304,9 +305,9 @@ func (server *SsntpTestServer) StatusNotify(uuid string, status ssntp.Status, fr
 
 	switch status {
 	case ssntp.READY:
-		fmt.Printf("server received READY from node %s\n", uuid)
+		fmt.Fprintf(os.Stderr, "server received READY from node %s\n", uuid)
 	default:
-		fmt.Printf("server unhandled status frame from node %s\n", uuid)
+		fmt.Fprintf(os.Stderr, "server unhandled status frame from node %s\n", uuid)
 	}
 
 	server.SendResultAndDelStatusChan(status, result)
@@ -391,7 +392,7 @@ func (server *SsntpTestServer) CommandNotify(uuid string, command ssntp.Command,
 		result.Err = err
 
 	default:
-		fmt.Printf("server unhandled command %s\n", command.String())
+		fmt.Fprintf(os.Stderr, "server unhandled command %s\n", command.String())
 	}
 
 	server.SendResultAndDelCmdChan(command, result)
@@ -427,7 +428,7 @@ func (server *SsntpTestServer) EventNotify(uuid string, event ssntp.Event, frame
 	case ssntp.PublicIPAssigned:
 		// forwards from CNCI Controller(s) via server.EventForward()
 	default:
-		fmt.Printf("server unhandled event %s\n", event.String())
+		fmt.Fprintf(os.Stderr, "server unhandled event %s\n", event.String())
 	}
 
 	server.SendResultAndDelEventChan(event, result)
@@ -474,7 +475,7 @@ func (server *SsntpTestServer) EventForward(uuid string, event ssntp.Event, fram
 	}
 
 	if err != nil {
-		fmt.Println("server error parsing event yaml for forwarding")
+		fmt.Fprintf(os.Stderr, "server error parsing event yaml for forwarding")
 		result.Err = err
 	}
 
@@ -515,7 +516,7 @@ func (server *SsntpTestServer) ErrorNotify(uuid string, error ssntp.Error, frame
 		fallthrough
 
 	default:
-		fmt.Printf("server unhandled error %s\n", error.String())
+		fmt.Fprintf(os.Stderr, "server unhandled error %s\n", error.String())
 	}
 
 	server.SendResultAndDelErrorChan(error, result)