Skip to content
This repository has been archived by the owner on Jul 16, 2020. It is now read-only.

Commit

Permalink
Merge pull request #483 from tpepper/master
Browse files Browse the repository at this point in the history 
testutil: more race/deadlock hunting
  • Loading branch information
@markdryan
markdryan authored Aug 19, 2016
2 parents b3a57f6 + 97711a5 commit 7281399
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 42 deletions.
46 changes: 26 additions & 20 deletions testutil/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func (client *SsntpTestClient) GetCmdChanResult(c *chan Result, cmd ssntp.Comman
if result.Err != nil {
err = fmt.Errorf("Client error sending %s command: %s\n", cmd, result.Err)
}
case <-time.After(5 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for client %s command result\n", cmd)
}

Expand Down Expand Up @@ -159,7 +159,7 @@ func (client *SsntpTestClient) GetEventChanResult(c *chan Result, evt ssntp.Even
if result.Err != nil {
err = fmt.Errorf("Client error sending %s event: %s\n", evt, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for client %s event result\n", evt)
}

Expand Down Expand Up @@ -198,7 +198,7 @@ func (client *SsntpTestClient) GetErrorChanResult(c *chan Result, error ssntp.Er
if result.Err != nil {
err = fmt.Errorf("Client error sending %s error: %s\n", error, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for client %s error result\n", error)
}

Expand Down Expand Up @@ -237,7 +237,7 @@ func (client *SsntpTestClient) GetStatusChanResult(c *chan Result, status ssntp.
if result.Err != nil {
err = fmt.Errorf("Client error sending %s status: %s\n", status, result.Err)
}
case <-time.After(5 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for client %s status result\n", status)
}

Expand Down Expand Up @@ -310,14 +310,14 @@ func closeClientChans(client *SsntpTestClient) {
func (client *SsntpTestClient) ConnectNotify() {
var result Result

client.SendResultAndDelEventChan(ssntp.NodeConnected, result)
go client.SendResultAndDelEventChan(ssntp.NodeConnected, result)
}

// DisconnectNotify implements the SSNTP client ConnectNotify callback for SsntpTestClient
func (client *SsntpTestClient) DisconnectNotify() {
var result Result

client.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
go client.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
}

// StatusNotify implements the SSNTP client StatusNotify callback for SsntpTestClient
Expand Down Expand Up @@ -345,7 +345,7 @@ func (client *SsntpTestClient) handleStart(payload []byte) Result {
if client.StartFail == true {
result.Err = errors.New(client.StartFailReason.String())
client.sendStartFailure(cmd.Start.InstanceUUID, client.StartFailReason)
client.SendResultAndDelErrorChan(ssntp.StartFailure, result)
go client.SendResultAndDelErrorChan(ssntp.StartFailure, result)
return result
}

Expand Down Expand Up @@ -376,7 +376,7 @@ func (client *SsntpTestClient) handleStop(payload []byte) Result {
if client.StopFail == true {
result.Err = errors.New(client.StopFailReason.String())
client.sendStopFailure(cmd.Stop.InstanceUUID, client.StopFailReason)
client.SendResultAndDelErrorChan(ssntp.StopFailure, result)
go client.SendResultAndDelErrorChan(ssntp.StopFailure, result)
return result
}

Expand Down Expand Up @@ -405,7 +405,7 @@ func (client *SsntpTestClient) handleRestart(payload []byte) Result {
if client.RestartFail == true {
result.Err = errors.New(client.RestartFailReason.String())
client.sendRestartFailure(cmd.Restart.InstanceUUID, client.RestartFailReason)
client.SendResultAndDelErrorChan(ssntp.RestartFailure, result)
go client.SendResultAndDelErrorChan(ssntp.RestartFailure, result)
return result
}

Expand Down Expand Up @@ -434,7 +434,7 @@ func (client *SsntpTestClient) handleDelete(payload []byte) Result {
if client.DeleteFail == true {
result.Err = errors.New(client.DeleteFailReason.String())
client.sendDeleteFailure(cmd.Delete.InstanceUUID, client.DeleteFailReason)
client.SendResultAndDelErrorChan(ssntp.DeleteFailure, result)
go client.SendResultAndDelErrorChan(ssntp.DeleteFailure, result)
return result
}

Expand Down Expand Up @@ -489,14 +489,20 @@ func (client *SsntpTestClient) CommandNotify(command ssntp.Command, frame *ssntp
fmt.Fprintf(os.Stderr, "client %s unhandled command %s\n", client.Role.String(), command.String())
}

client.SendResultAndDelCmdChan(command, result)
go client.SendResultAndDelCmdChan(command, result)
}

// EventNotify is an SSNTP callback stub for SsntpTestClient
func (client *SsntpTestClient) EventNotify(event ssntp.Event, frame *ssntp.Frame) {
var result Result

switch event {
case ssntp.NodeConnected:
//handled by ConnectNotify()
return
case ssntp.NodeDisconnected:
//handled by DisconnectNotify()
return
case ssntp.TenantAdded:
var tenantAddedEvent payloads.EventTenantAdded

Expand All @@ -515,7 +521,7 @@ func (client *SsntpTestClient) EventNotify(event ssntp.Event, frame *ssntp.Frame
fmt.Fprintf(os.Stderr, "client %s unhandled event: %s\n", client.Role.String(), event.String())
}

client.SendResultAndDelEventChan(event, result)
go client.SendResultAndDelEventChan(event, result)
}

// ErrorNotify is an SSNTP callback stub for SsntpTestClient
Expand All @@ -538,7 +544,7 @@ func (client *SsntpTestClient) SendStatsCmd() {
}
}

client.SendResultAndDelCmdChan(ssntp.STATS, result)
go client.SendResultAndDelCmdChan(ssntp.STATS, result)
}

// SendStatus pushes an ssntp status frame from the SsntpTestClient with
Expand All @@ -558,7 +564,7 @@ func (client *SsntpTestClient) SendStatus(memTotal int, memAvail int) {
}
}

client.SendResultAndDelCmdChan(ssntp.STATS, result)
go client.SendResultAndDelCmdChan(ssntp.STATS, result)
}

// SendTrace allows an SsntpTestClient to push an ssntp.TraceReport event frame
Expand Down Expand Up @@ -590,7 +596,7 @@ func (client *SsntpTestClient) SendTrace() {
}
}

client.SendResultAndDelEventChan(ssntp.TraceReport, result)
go client.SendResultAndDelEventChan(ssntp.TraceReport, result)
}

// SendDeleteEvent allows an SsntpTestClient to push an ssntp.InstanceDeleted event frame
Expand All @@ -615,7 +621,7 @@ func (client *SsntpTestClient) SendDeleteEvent(uuid string) {
}
}

client.SendResultAndDelEventChan(ssntp.InstanceDeleted, result)
go client.SendResultAndDelEventChan(ssntp.InstanceDeleted, result)
}

// SendTenantAddedEvent allows an SsntpTestClient to push an ssntp.TenantAdded event frame
Expand All @@ -627,7 +633,7 @@ func (client *SsntpTestClient) SendTenantAddedEvent() {
result.Err = err
}

client.SendResultAndDelEventChan(ssntp.TenantAdded, result)
go client.SendResultAndDelEventChan(ssntp.TenantAdded, result)
}

// SendTenantRemovedEvent allows an SsntpTestClient to push an ssntp.TenantRemoved event frame
Expand All @@ -639,7 +645,7 @@ func (client *SsntpTestClient) SendTenantRemovedEvent() {
result.Err = err
}

client.SendResultAndDelEventChan(ssntp.TenantRemoved, result)
go client.SendResultAndDelEventChan(ssntp.TenantRemoved, result)
}

// SendPublicIPAssignedEvent allows an SsntpTestClient to push an ssntp.PublicIPAssigned event frame
Expand All @@ -651,7 +657,7 @@ func (client *SsntpTestClient) SendPublicIPAssignedEvent() {
result.Err = err
}

client.SendResultAndDelEventChan(ssntp.PublicIPAssigned, result)
go client.SendResultAndDelEventChan(ssntp.PublicIPAssigned, result)
}

// SendConcentratorAddedEvent allows an SsntpTestClient to push an ssntp.ConcentratorInstanceAdded event frame
Expand Down Expand Up @@ -680,7 +686,7 @@ func (client *SsntpTestClient) SendConcentratorAddedEvent(instanceUUID string, t
}
}

client.SendResultAndDelEventChan(ssntp.ConcentratorInstanceAdded, result)
go client.SendResultAndDelEventChan(ssntp.ConcentratorInstanceAdded, result)
}

func (client *SsntpTestClient) sendStartFailure(instanceUUID string, reason payloads.StartFailureReason) {
Expand Down
2 changes: 1 addition & 1 deletion testutil/client_server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ func stopServer() error {
netAgentCh := netAgent.AddEventChan(ssntp.NodeDisconnected)
agentCh := agent.AddEventChan(ssntp.NodeDisconnected)

server.Shutdown()
go server.Shutdown()

_, err := controller.GetEventChanResult(controllerCh, ssntp.NodeDisconnected)
if err != nil {
Expand Down
24 changes: 14 additions & 10 deletions testutil/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func (ctl *SsntpTestController) GetCmdChanResult(c *chan Result, cmd ssntp.Comma
if result.Err != nil {
err = fmt.Errorf("Controller error sending %s command: %s\n", cmd, result.Err)
}
case <-time.After(5 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for controller %s command result\n", cmd)
}

Expand Down Expand Up @@ -137,7 +137,7 @@ func (ctl *SsntpTestController) GetEventChanResult(c *chan Result, evt ssntp.Eve
if result.Err != nil {
err = fmt.Errorf("Controller error sending %s event: %s\n", evt, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for controller %s event result\n", evt)
}

Expand Down Expand Up @@ -176,7 +176,7 @@ func (ctl *SsntpTestController) GetErrorChanResult(c *chan Result, error ssntp.E
if result.Err != nil {
err = fmt.Errorf("Controller error sending %s error: %s\n", error, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for controller %s error result\n", error)
}

Expand Down Expand Up @@ -238,14 +238,14 @@ func closeControllerChans(ctl *SsntpTestController) {
func (ctl *SsntpTestController) ConnectNotify() {
var result Result

ctl.SendResultAndDelEventChan(ssntp.NodeConnected, result)
go ctl.SendResultAndDelEventChan(ssntp.NodeConnected, result)
}

// DisconnectNotify implements the SSNTP client DisconnectNotify callback for SsntpTestController
func (ctl *SsntpTestController) DisconnectNotify() {
var result Result

ctl.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
go ctl.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
}

// StatusNotify implements the SSNTP client StatusNotify callback for SsntpTestController
Expand Down Expand Up @@ -279,18 +279,22 @@ func (ctl *SsntpTestController) CommandNotify(command ssntp.Command, frame *ssnt
fmt.Fprintf(os.Stderr, "controller unhandled command: %s\n", command.String())
}

ctl.SendResultAndDelCmdChan(command, result)
go ctl.SendResultAndDelCmdChan(command, result)
}

// EventNotify implements the SSNTP client EventNotify callback for SsntpTestController
func (ctl *SsntpTestController) EventNotify(event ssntp.Event, frame *ssntp.Frame) {
var result Result

switch event {
// case ssntp.NodeConnected: handled by ConnectNotify()
// case ssntp.NodeDisconnected: handled by DisconnectNotify()
// case ssntp.TenantAdded: does not reach controller
// case ssntp.TenantRemoved: does not reach controller
case ssntp.NodeConnected:
//handled by ConnectNotify()
return
case ssntp.NodeDisconnected:
//handled by DisconnectNotify()
return
case ssntp.PublicIPAssigned:
var publicIPAssignedEvent payloads.EventPublicIPAssigned

Expand Down Expand Up @@ -323,7 +327,7 @@ func (ctl *SsntpTestController) EventNotify(event ssntp.Event, frame *ssntp.Fram
fmt.Fprintf(os.Stderr, "controller unhandled event: %s\n", event.String())
}

ctl.SendResultAndDelEventChan(event, result)
go ctl.SendResultAndDelEventChan(event, result)
}

// ErrorNotify implements the SSNTP client ErrorNotify callback for SsntpTestController
Expand All @@ -347,5 +351,5 @@ func (ctl *SsntpTestController) ErrorNotify(error ssntp.Error, frame *ssntp.Fram
fmt.Fprintf(os.Stderr, "controller unhandled error %s\n", error.String())
}

ctl.SendResultAndDelErrorChan(error, result)
go ctl.SendResultAndDelErrorChan(error, result)
}
22 changes: 11 additions & 11 deletions testutil/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (server *SsntpTestServer) GetCmdChanResult(c *chan Result, cmd ssntp.Comman
if result.Err != nil {
err = fmt.Errorf("Server error on %s command: %s\n", cmd, result.Err)
}
case <-time.After(5 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for server %s command result\n", cmd)
}

Expand Down Expand Up @@ -104,7 +104,7 @@ func (server *SsntpTestServer) GetEventChanResult(c *chan Result, evt ssntp.Even
if result.Err != nil {
err = fmt.Errorf("Server error handling %s event: %s\n", evt, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for server %s event result\n", evt)
}

Expand Down Expand Up @@ -143,7 +143,7 @@ func (server *SsntpTestServer) GetErrorChanResult(c *chan Result, error ssntp.Er
if result.Err != nil {
err = fmt.Errorf("Server error handling %s error: %s\n", error, result.Err)
}
case <-time.After(20 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for server %s error result\n", error)
}

Expand Down Expand Up @@ -182,7 +182,7 @@ func (server *SsntpTestServer) GetStatusChanResult(c *chan Result, status ssntp.
if result.Err != nil {
err = fmt.Errorf("Server error handling %s status: %s\n", status, result.Err)
}
case <-time.After(5 * time.Second):
case <-time.After(25 * time.Second):
err = fmt.Errorf("Timeout waiting for server %s status result\n", status)
}

Expand Down Expand Up @@ -267,7 +267,7 @@ func (server *SsntpTestServer) ConnectNotify(uuid string, role ssntp.Role) {
server.netClients = append(server.netClients, uuid)
}

server.SendResultAndDelEventChan(ssntp.NodeConnected, result)
go server.SendResultAndDelEventChan(ssntp.NodeConnected, result)
}

// DisconnectNotify implements an SSNTP DisconnectNotify callback for SsntpTestServer
Expand Down Expand Up @@ -296,7 +296,7 @@ func (server *SsntpTestServer) DisconnectNotify(uuid string, role ssntp.Role) {
server.netClientsLock.Unlock()
}

server.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
go server.SendResultAndDelEventChan(ssntp.NodeDisconnected, result)
}

// StatusNotify is an SSNTP callback stub for SsntpTestServer
Expand All @@ -310,7 +310,7 @@ func (server *SsntpTestServer) StatusNotify(uuid string, status ssntp.Status, fr
fmt.Fprintf(os.Stderr, "server unhandled status frame from node %s\n", uuid)
}

server.SendResultAndDelStatusChan(status, result)
go server.SendResultAndDelStatusChan(status, result)
}

// CommandNotify implements an SSNTP CommandNotify callback for SsntpTestServer
Expand Down Expand Up @@ -395,7 +395,7 @@ func (server *SsntpTestServer) CommandNotify(uuid string, command ssntp.Command,
fmt.Fprintf(os.Stderr, "server unhandled command %s\n", command.String())
}

server.SendResultAndDelCmdChan(command, result)
go server.SendResultAndDelCmdChan(command, result)
}

// EventNotify implements an SSNTP EventNotify callback for SsntpTestServer
Expand Down Expand Up @@ -431,7 +431,7 @@ func (server *SsntpTestServer) EventNotify(uuid string, event ssntp.Event, frame
fmt.Fprintf(os.Stderr, "server unhandled event %s\n", event.String())
}

server.SendResultAndDelEventChan(event, result)
go server.SendResultAndDelEventChan(event, result)
}

func getConcentratorUUID(event ssntp.Event, payload []byte) (string, error) {
Expand Down Expand Up @@ -479,7 +479,7 @@ func (server *SsntpTestServer) EventForward(uuid string, event ssntp.Event, fram
result.Err = err
}

server.SendResultAndDelEventChan(event, result)
go server.SendResultAndDelEventChan(event, result)

return dest
}
Expand Down Expand Up @@ -519,7 +519,7 @@ func (server *SsntpTestServer) ErrorNotify(uuid string, error ssntp.Error, frame
fmt.Fprintf(os.Stderr, "server unhandled error %s\n", error.String())
}

server.SendResultAndDelErrorChan(error, result)
go server.SendResultAndDelErrorChan(error, result)
}

func (server *SsntpTestServer) handleStart(payload []byte) (dest ssntp.ForwardDestination) {
Expand Down

0 comments on commit 7281399

Please sign in to comment.