Feat skip states for workflows

Taskfile.yaml

@@ -55,8 +55,8 @@ tasks:
   recreate-db-from-scratch:
     cmds:
       - docker compose down
-      - docker volume rm oss_hatchet_postgres_data
-      - docker volume rm oss_hatchet_rabbitmq_data
+      - docker volume rm oss_hatchet_postgres_data || true
+      - docker volume rm oss_hatchet_rabbitmq_data || true
       - docker compose up -d
       - task: setup
       - task: init-dev-env

api/v1/server/handlers/workflows/trigger.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hatchet-dev/hatchet/internal/services/shared/tasktypes"
 	"github.com/hatchet-dev/hatchet/pkg/repository"
 	"github.com/hatchet-dev/hatchet/pkg/repository/metered"
+	"github.com/hatchet-dev/hatchet/pkg/repository/prisma"
 	"github.com/hatchet-dev/hatchet/pkg/repository/prisma/db"
 	"github.com/hatchet-dev/hatchet/pkg/repository/prisma/dbsqlc"
 	"github.com/hatchet-dev/hatchet/pkg/repository/prisma/sqlchelpers"
@@ -95,21 +96,37 @@ func (t *WorkflowService) WorkflowRunCreate(ctx echo.Context, request gen.Workfl
 		return nil, fmt.Errorf("trigger.go could not create workflow run: %w", err)
 	}
 
-	// send to workflow processing queue
-	err = t.config.MessageQueue.AddMessage(
-		ctx.Request().Context(),
-		msgqueue.WORKFLOW_PROCESSING_QUEUE,
-		tasktypes.WorkflowRunQueuedToTask(
-			sqlchelpers.UUIDToStr(createdWorkflowRun.TenantId),
-			sqlchelpers.UUIDToStr(createdWorkflowRun.ID),
-		),
-	)
+	if !prisma.CanShortCircuit(createdWorkflowRun.Row) {
+		// send to workflow processing queue
+		err = t.config.MessageQueue.AddMessage(
+			ctx.Request().Context(),
+			msgqueue.WORKFLOW_PROCESSING_QUEUE,
+			tasktypes.WorkflowRunQueuedToTask(
+				sqlchelpers.UUIDToStr(createdWorkflowRun.Row.WorkflowRun.TenantId),
+				sqlchelpers.UUIDToStr(createdWorkflowRun.Row.WorkflowRun.ID),
+			),
+		)
 
-	if err != nil {
-		return nil, fmt.Errorf("could not add workflow run to queue: %w", err)
+		if err != nil {
+			return nil, fmt.Errorf("could not add workflow run to queue: %w", err)
+		}
 	}
 
-	workflowRun, err := t.config.APIRepository.WorkflowRun().GetWorkflowRunById(ctx.Request().Context(), tenant.ID, sqlchelpers.UUIDToStr(createdWorkflowRun.ID))
+	for _, queueName := range createdWorkflowRun.StepRunQueueNames {
+
+		if schedPartitionId, ok := tenant.SchedulerPartitionID(); ok {
+			err = t.config.MessageQueue.AddMessage(
+				ctx.Request().Context(),
+				msgqueue.QueueTypeFromPartitionIDAndController(schedPartitionId, msgqueue.Scheduler),
+				tasktypes.CheckTenantQueueToTask(tenant.ID, queueName, true, false),
+			)
+
+			if err != nil {
+				t.config.Logger.Err(err).Msg("could not add message to scheduler partition queue")
+			}
+		}
+	}
+	workflowRun, err := t.config.APIRepository.WorkflowRun().GetWorkflowRunById(ctx.Request().Context(), tenant.ID, sqlchelpers.UUIDToStr(createdWorkflowRun.Row.WorkflowRun.ID))
 
 	if err != nil {
 		return nil, fmt.Errorf("could not get workflow run: %w", err)

cmd/hatchet-engine/engine/run.go

@@ -370,6 +370,7 @@ func runV0Config(ctx context.Context, sc *server.ServerConfig) ([]Teardown, erro
 			admin.WithRepository(sc.EngineRepository),
 			admin.WithMessageQueue(sc.MessageQueue),
 			admin.WithEntitlementsRepository(sc.EntitlementRepository),
+			admin.WithLogger(sc.Logger),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("could not create admin service: %w", err)
@@ -730,6 +731,7 @@ func runV1Config(ctx context.Context, sc *server.ServerConfig) ([]Teardown, erro
 			admin.WithRepository(sc.EngineRepository),
 			admin.WithMessageQueue(sc.MessageQueue),
 			admin.WithEntitlementsRepository(sc.EntitlementRepository),
+			admin.WithLogger(sc.Logger),
 		)
 
 		if err != nil {

examples/bulk_imports/main.go

@@ -83,9 +83,9 @@ func run() (func() error, error) {
 
 	var events []client.EventWithAdditionalMetadata
 
-	// 20000 times to test the bulk push
+	// 999 (max amount) times to test the bulk push
 
-	for i := 0; i < 20000; i++ {
+	for i := 0; i < 999; i++ {
 		testEvent := userCreateEvent{
 			Username: "echo-test",
 			UserID:   "1234 " + fmt.Sprint(i),

examples/concurrency/main.go

@@ -30,27 +30,63 @@ func main() {
 	}
 
 	events := make(chan string, 50)
+	wfrIds := make(chan *client.Workflow, 50)
 	interrupt := cmdutils.InterruptChan()
+	c, err := client.New()
 
-	cleanup, err := run(events)
+	if err != nil {
+		log.Fatalf("error creating client: %v", err)
+	}
+	cleanup, err := run(c, events, wfrIds)
 	if err != nil {
 		panic(err)
 	}
+selectLoop:
+	for {
+		select {
+
+		case <-interrupt:
+			log.Print("Interrupted")
+			break selectLoop
+		case wfrId := <-wfrIds:
+			log.Printf("Workflow run id: %s", wfrId.WorkflowRunId())
+			wfResult, err := wfrId.Result()
+			if err != nil {
 
-	<-interrupt
+				if err.Error() == "step output for step-one not found" {
+					log.Printf("Step output for step-one not found because it was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT")
+					continue
+				}
+				panic(fmt.Errorf("error getting workflow run result: %w", err))
+			}
+
+			stepOneOutput := &stepOneOutput{}
+
+			err = wfResult.StepOutput("step-one", stepOneOutput)
+
+			if err != nil {
+				if err.Error() == "step run failed: this step run was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT" {
+					log.Printf("Workflow run was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT")
+					continue
+				}
+				if err.Error() == "step output for step-one not found" {
+					log.Printf("Step output for step-one not found because it was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT")
+					continue
+				}
+				panic(fmt.Errorf("error getting workflow run result: %w", err))
+			}
+		case e := <-events:
+			log.Printf("Event: %s", e)
+		}
+	}
 
 	if err := cleanup(); err != nil {
 
 		panic(fmt.Errorf("error cleaning up: %w", err))
 	}
 }
 
-func run(events chan<- string) (func() error, error) {
-	c, err := client.New()
-
-	if err != nil {
-		return nil, fmt.Errorf("error creating client: %w", err)
-	}
+func run(c client.Client, events chan<- string, wfrIds chan<- *client.Workflow) (func() error, error) {
 
 	w, err := worker.NewWorker(
 		worker.WithClient(
@@ -74,7 +110,8 @@ func run(events chan<- string) (func() error, error) {
 					err = ctx.WorkflowInput(input)
 
 					// we sleep to simulate a long running task
-					time.Sleep(10 * time.Second)
+
+					time.Sleep(7 * time.Second)
 
 					if err != nil {
 
@@ -98,7 +135,11 @@ func run(events chan<- string) (func() error, error) {
 					err = ctx.StepOutput("step-one", input)
 
 					if err != nil {
-						return nil, err
+
+						if err.Error() == "step run failed: this step run was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT" {
+							return nil, nil
+						}
+
 					}
 
 					if ctx.Err() != nil {
@@ -125,18 +166,39 @@ func run(events chan<- string) (func() error, error) {
 			"test": "test",
 		},
 	}
+
+	// I want some to be in Running and some to be in Pending so we cancel both
+
 	go func() {
 		// do this 10 times to test concurrency
-		for i := 0; i < 10; i++ {
+		for i := 0; i < 7; i++ {
 
 			wfr_id, err := c.Admin().RunWorkflow("simple-concurrency", testEvent)
 
-			log.Println("Starting workflow run id: ", wfr_id)
+			log.Println("Starting workflow run id: ", wfr_id.WorkflowRunId())
 
 			if err != nil {
 				panic(fmt.Errorf("error running workflow: %w", err))
 			}
 
+			wfrIds <- wfr_id
+			time.Sleep(400 * time.Millisecond)
+		}
+	}()
+	go func() {
+		// do this 10 times to test concurrency
+		for i := 0; i < 13; i++ {
+
+			wfr_id, err := c.Admin().RunWorkflow("simple-concurrency", testEvent)
+
+			log.Println("Starting workflow run id: ", wfr_id.WorkflowRunId())
+
+			if err != nil {
+				panic(fmt.Errorf("error running workflow: %w", err))
+			}
+
+			wfrIds <- wfr_id
+
 		}
 	}()
 

examples/concurrency/main_e2e_test.go

@@ -4,49 +4,126 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 
 	"github.com/hatchet-dev/hatchet/internal/testutils"
+	"github.com/hatchet-dev/hatchet/pkg/client"
 )
 
 func TestConcurrency(t *testing.T) {
 	testutils.Prepare(t)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 
 	events := make(chan string, 50)
+	wfrIds := make(chan *client.Workflow, 50)
+	c, err := client.New()
 
-	cleanup, err := run(events)
+	if err != nil {
+		panic("error creating client: " + err.Error())
+	}
+	cleanup, err := run(c, events, wfrIds)
 	if err != nil {
 		t.Fatalf("/run() error = %v", err)
 	}
 
 	var items []string
-
+	var workflowRunIds []*client.WorkflowResult
+	var wg sync.WaitGroup
+	done := make(chan struct{})
 outer:
 	for {
+
 		select {
 		case item := <-events:
 			items = append(items, item)
 			if len(items) > 2 {
+				fmt.Println("got 2 events")
 				break outer
 			}
 		case <-ctx.Done():
+			fmt.Println("context done")
 			break outer
+
+		case wfrId := <-wfrIds:
+			fmt.Println("got wfr id")
+			go func(workflow *client.Workflow) {
+				wg.Add(1)
+				defer wg.Done()
+				wfr, err := workflow.Result()
+				workflowRunIds = append(workflowRunIds, wfr)
+				if err != nil {
+					panic(fmt.Errorf("error getting workflow run result: %w", err))
+				}
+			}(wfrId)
+
+		}
+	}
+
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+
+	case <-time.After(20 * time.Second):
+		t.Fatalf("timed out waiting for workflow results")
+	case <-done:
+
+	}
+
+	// our workflow run ids should have only one succeeded everyone else should have failed
+	stateCount := make(map[string]int)
+
+	if len(workflowRunIds) != 20 {
+		t.Fatalf("expected 20 workflow run ids, got %d", len(workflowRunIds))
+	}
+
+	for _, wfrId := range workflowRunIds {
+		state, err := getWorkflowStateForWorkflowRunId(c, ctx, wfrId)
+
+		fmt.Println("state: ", state)
+		if err != nil {
+			t.Fatalf("error getting workflow state: %v", err)
 		}
+		stateCount[state]++
 	}
 
-	assert.Equal(t, []string{
-		"step-one",
-		"step-two",
-	}, items)
+	assert.Equal(t, 1, stateCount["SUCCEEDED"])
+	assert.Equal(t, 19, stateCount["CANCELLED_BY_CONCURRENCY_LIMIT"])
 
 	if err := cleanup(); err != nil {
 		t.Fatalf("cleanup() error = %v", err)
 	}
 
 }
+
+func getWorkflowStateForWorkflowRunId(client client.Client, ctx context.Context, wfr *client.WorkflowResult) (string, error) {
+
+	stepOneOutput := &stepOneOutput{}
+
+	err := wfr.StepOutput("step-one", stepOneOutput)
+	if err != nil {
+
+		if err.Error() == "step run failed: this step run was cancelled due to CANCELLED_BY_CONCURRENCY_LIMIT" {
+			return "CANCELLED_BY_CONCURRENCY_LIMIT", nil
+		}
+
+		// this happens if we cancel before the workflow is run
+		if err.Error() == "step output for step-one not found" {
+			return "CANCELLED_BY_CONCURRENCY_LIMIT", nil
+		}
+
+		fmt.Println("error getting step output: %w", err)
+		return "", err
+	}
+
+	return "SUCCEEDED", nil
+}