temp

README.md

@@ -33,6 +33,31 @@ import _ "github.com/my-org/my-service/tests"
 
 For examples see [./cmd/example-server-bootstrap/main.go] and [./internal/packagetestexample].
 
+## Build
+
+```sh
+templ generate
+go build ./cmd/example-server-bootstrap/
+```
+
+## Run
+
+```sh
+./example-server-bootstrap
+```
+
+## Web
+
+If your server is running you can open your browser to e.g. `http://localhost:1337/suites`.
+
+This will show you all available test suites.
+
+To create a new test run you can use the post request in `./requests.http`
+
+```sh
+httpyac requests.http
+```
+
 ## Test best practices
 
 * Pass in the test context for longer running operations and check if it was cancelled.
@@ -45,7 +70,6 @@ For examples see [./cmd/example-server-bootstrap/main.go] and [./internal/packag
 - [ ] (Feature) Automatic test run retries/backoff on failures
 - [ ] (Feature) CLI program to run tests and wait for results
 - [ ] (Feature) Opt-in test timeouts through t.Context and / or providing wrapped handoff functions ( e.g. http clients) to be used in tests  that implement test timeouts
-- [ ] (Feature) Flaky test metric
 - [ ] (Feature) Add test-suite labels
 - [ ] (Feature) Add an option to the helm chart to support remote debugging through dlv
 - [ ] (Feature) Image for [helm chart](https://helm.sh/docs/topics/chart_tests/) tests for automated helm release rollbacks
@@ -71,7 +95,7 @@ For examples see [./cmd/example-server-bootstrap/main.go] and [./internal/packag
 - [x] (Feature) Skip individual tests by calling t.Skip() within a test
 - [x] (Feature) Scheduled / recurring test runs (e.g. for soak tests)
 - [x] (Feature) Skip test subsets via regex filters passed into a test run
-- [x] (Feature) Flaky test detection
+- [x] (Feature) Flaky test detection + metric label
 - [x] (Feature) Support existing assertion libraries like stretch/testify
 - [x] (Feature) Prometheus /metrics endpoint that exposes test metrics
 - [x] (Feature) Basic support for plugins to hook into the test lifecycle

cmd/example-server-bootstrap/main.go

@@ -42,7 +42,6 @@ func main() {
 				LoggingTest,
 			},
 		}),
-		handoff.WithScheduledRun(handoff.ScheduledRun{TestSuiteName: "my-app", Schedule: "@every 5s"}),
 	)
 
 	if err := h.Run(os.Args); err != nil {

go.mod

@@ -3,15 +3,15 @@ module github.com/raphi011/handoff
 go 1.21.1
 
 require (
-	github.com/a-h/templ v0.2.364
+	github.com/a-h/templ v0.2.793
 	github.com/alexflint/go-arg v1.4.3
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/elastic/go-elasticsearch/v8 v8.6.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/stretchr/testify v1.8.1
-	golang.org/x/tools v0.13.0
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/tools v0.24.0
 )
 
 require (
@@ -28,16 +28,18 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v2.0.8+incompatible // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/slack-go/slack v0.12.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -35,6 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/a-h/templ v0.2.364 h1:4vseUATyAzacrOOU1jPmLfkxet6Yg/NAmsA/4rDDhfo=
 github.com/a-h/templ v0.2.364/go.mod h1:6Lfhsl3Z4/vXl7jjEjkJRCqoWDGjDnuKgzjYMDSddas=
+github.com/a-h/templ v0.2.793 h1:Io+/ocnfGWYO4VHdR0zBbf39PQlnzVCVVD+wEEs6/qY=
+github.com/a-h/templ v0.2.793/go.mod h1:lq48JXoUvuQrU0VThrK31yFwdRjTCnIE5bcPCM9IP1w=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -89,6 +91,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -140,6 +143,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -156,6 +160,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -228,6 +235,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/slack-go/slack v0.12.3 h1:92/dfFU8Q5XP6Wp5rr5/T5JHLM5c5Smtn53fhToAP88=
+github.com/slack-go/slack v0.12.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -240,6 +249,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -324,6 +335,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -383,6 +396,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -439,6 +454,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

handoff.go

@@ -37,13 +37,14 @@ type Server struct {
 	// initialisation.
 	readOnlyTestSuites map[string]model.TestSuite
 
+	// a map of statically configured scheduled runs  that must not be modified after
+	// initialisation.
+	readOnlySchedules []model.ScheduledRun
+
 	// _userProvidedTestSuites is a list of all test suites provided
 	// by the user and will be mapped to `readOnlyTestSuites` on startup.
 	_userProvidedTestSuites []TestSuite
 
-	// scheduled runs configured by the user
-	schedules []ScheduledRun
-
 	// started will be closed when the service has started.
 	started chan any
 
@@ -118,31 +119,26 @@ type TestContext = model.TestContext
 
 type Option func(s *Server)
 
-var (
-	registeredSuites    []TestSuite
-	registeredSchedules []ScheduledRun
-)
+var registeredSuites []TestSuite
 
 // Register registers test suites and schedules for the handoff server. Has to be called
 // before `handoff.New()` to take effect.
-func Register(suites []TestSuite, schedules []ScheduledRun) {
+func RegisterSuites(suites []TestSuite) {
 	registeredSuites = append(registeredSuites, suites...)
-	registeredSchedules = append(registeredSchedules, schedules...)
 }
 
 // New configures a new Handoff instance.
 func New(opts ...Option) *Server {
 	s := &Server{
 		_userProvidedTestSuites: registeredSuites,
-		schedules:               registeredSchedules,
 		readOnlyTestSuites:      map[string]model.TestSuite{},
 		started:                 make(chan any),
 		hasShutdown:             make(chan error, 1),
 		shutdown:                make(chan any),
 		log:                     slog.New(slog.NewTextHandler(os.Stderr, nil)),
 	}
 
-	s.hooks = newHookManager(s.asyncHookCallback)
+	s.hooks = newHookManager(s.asyncHookCallback, s.log)
 
 	for _, o := range opts {
 		o(s)
@@ -189,7 +185,8 @@ func (s *Server) Run(args []string) error {
 		return fmt.Errorf("init hooks: %w", err)
 	}
 
-	if err := s.startSchedules(); err != nil {
+	// TODO: also start schedules from DB
+	if err := s.startStaticSchedules(); err != nil {
 		return fmt.Errorf("start schedules: %w", err)
 	}
 
@@ -343,38 +340,53 @@ func (s *Server) printTestSuites() {
 	os.Exit(0)
 }
 
-func (s *Server) startSchedules() error {
-
-	for i := range s.schedules {
-		schedule := s.schedules[i]
+func (s *Server) startStaticSchedules() error {
+	for _, sr := range s.readOnlySchedules {
 
-		ts, ok := s.readOnlyTestSuites[schedule.TestSuiteName]
-		if !ok {
-			return fmt.Errorf("test suite %q not found", schedule.TestSuiteName)
+		if _, err := s.startSchedule(sr, false); err != nil {
+			return err
 		}
+	}
+
+	return nil
+}
+
+func (s *Server) startSchedule(sr model.ScheduledRun, persist bool) (cron.EntryID, error) {
+	ts, ok := s.readOnlyTestSuites[sr.TestSuiteName]
+	if !ok {
+		return -1, fmt.Errorf("test suite %q not found", sr.TestSuiteName)
+	}
 
-		if len(ts.FilterTests(schedule.TestFilter)) == 0 {
-			return errors.New("no tests match filter regex %s")
+	if len(ts.FilterTests(sr.TestFilter)) == 0 {
+		return -1, errors.New("no tests match filter regex %s")
+	}
+
+	ctx := context.Background()
+
+	if persist {
+		err := s.storage.InsertScheduledRun(ctx, sr)
+		if err != nil {
+			return -1, fmt.Errorf("persisting schedule failed: %w", err)
 		}
+	}
 
-		entryID, err := s.cron.AddFunc(schedule.Schedule, func() {
-			_, err := s.startNewTestSuiteRun(ts, model.RunParams{
-				TriggeredBy:     "scheduled",
-				TestFilter:      schedule.TestFilter,
-				MaxTestAttempts: ts.MaxTestAttempts,
-			})
-			if err != nil {
-				s.log.Error("starting new scheduled test suite run failed", "error", err, "test-suite", ts.Name)
-			}
+	entryID, err := s.cron.AddFunc(sr.Schedule, func() {
+		_, err := s.startNewTestSuiteRun(ts, model.RunParams{
+			TriggeredBy:     "scheduled",
+			TestFilter:      sr.TestFilter,
+			MaxTestAttempts: ts.MaxTestAttempts,
 		})
 		if err != nil {
-			return fmt.Errorf("adding scheduled test suite run %q: %w", schedule.TestSuiteName, err)
+			s.log.Error("starting new scheduled test suite run failed", "error", err, "test-suite", ts.Name)
 		}
-
-		s.schedules[i].EntryID = entryID
+	})
+	if err != nil {
+		// TODO: log this error?
+		_ = s.storage.DeleteScheduledRun(ctx, sr.Name)
+		return -1, fmt.Errorf("adding scheduled test suite run %q: %w", sr.TestSuiteName, err)
 	}
 
-	return nil
+	return entryID, nil
 }
 
 // startNewTestSuiteRun is used to start new test suite runs. It persists the
@@ -537,7 +549,7 @@ func (s *Server) runTestSuite(
 	if tsr.Result != model.ResultPending {
 		s.hooks.notifyTestSuiteFinishedAsync(suite, tsr)
 
-		metric.TestSuitesRun.WithLabelValues(s.config.Instance, suite.Namespace, suite.Name, string(tsr.Result)).Inc()
+		metric.TestSuiteFinished(s.config.Instance, suite, tsr)
 	}
 }
 

handoff_test.go

@@ -150,11 +150,6 @@ func TestShutdownSucceeds(t *testing.T) {
 				Sleep(2 * time.Second),
 			},
 		}},
-		[]handoff.ScheduledRun{
-			{
-				TestSuiteName: "success",
-				Schedule:      "@every 1.5s",
-			}},
 		[]string{"handoff-test", "-p", "0", "-d", ""},
 	)
 
@@ -177,13 +172,7 @@ func TestScheduledRunWithTestFilter(t *testing.T) {
 		Tests: []model.TestFunc{Success, LogAttempt},
 	}}
 
-	scheduledRuns := []handoff.ScheduledRun{{
-		TestSuiteName: suiteName,
-		Schedule:      "@every 1.5s",
-		TestFilter:    regexp.MustCompile(filteredTestName),
-	}}
-
-	i := handoffInstance(suites, scheduledRuns, []string{"handoff-test", "-p", "0", "-d", ""})
+	i := handoffInstance(suites, []string{"handoff-test", "-p", "0", "-d", ""})
 
 	// make sure the scheduled run has started
 	time.Sleep(2 * time.Second)