Track progress updates from long running tasks for better debugging and rich audit logs. (#189)

* TaskWorker accepts interim results from tasks that are still processing.
* Can now push interim updates for a task still processing using the JS SDK.
* Can now push interim updates for a task still processing using the Python SDK.
* Updated Orchestration View to handle interim task results.
* Can display interim task results as short and long progress updates on CLI.
* Prepared changelog for latest feature release.

Author: ezodude

README.md

@@ -82,11 +82,11 @@ Download the latest CLI binary for your platform from our [releases page](https:
 
 ```shell
 # macOS
-curl -L https://github.com/orra-dev/orra/releases/download/v0.2.2/orra-darwin-arm64 -o /usr/local/bin/orra
+curl -L https://github.com/orra-dev/orra/releases/download/v0.2.3/orra-darwin-arm64 -o /usr/local/bin/orra
 chmod +x /usr/local/bin/orra
 
 # Linux
-curl -L https://github.com/ezodude/orra/releases/download/v0.2.2/orra-linux-amd64 -o /usr/local/bin/orra
+curl -L https://github.com/ezodude/orra/releases/download/v0.2.3/orra-linux-amd64 -o /usr/local/bin/orra
 chmod +x /usr/local/bin/orra
 
 # Verify installation

cli/cmd/inspect.go

@@ -21,6 +21,8 @@ import (
 
 func newInspectCmd(opts *CliOpts) *cobra.Command {
 	var detailed bool
+	var shortUpdates bool
+	var longUpdates bool
 
 	cmd := &cobra.Command{
 		Use:   "inspect [orchestration-id]",
@@ -101,6 +103,48 @@ func newInspectCmd(opts *CliOpts) *cobra.Command {
 						fmt.Println(statusLine)
 					}
 
+					if (shortUpdates || longUpdates) && len(task.InterimResults) > 0 {
+						// Show the simplified timeline
+						timelineStr := renderTaskTimeline(task.StatusHistory, task.InterimResults)
+						if timelineStr != "" {
+							fmt.Println(timelineStr)
+						}
+
+						fmt.Printf("│\n│ Progress Updates %s\n", strings.Repeat("─", 35))
+
+						if longUpdates {
+							// Show all updates
+							for j, result := range task.InterimResults {
+								fmt.Printf("│ %s  Update %d/%d\n",
+									result.Timestamp.Format("15:04:05"),
+									j+1,
+									len(task.InterimResults))
+								printJSONWithPrefix(result.Data, "│   ")
+								fmt.Printf("│\n")
+							}
+						} else {
+							// Show only first and last updates
+							first := task.InterimResults[0]
+							fmt.Printf("│ %s  First Update\n", first.Timestamp.Format("15:04:05"))
+							printJSONWithPrefix(first.Data, "│   ")
+
+							// Show last update if different from first
+							if len(task.InterimResults) > 1 {
+								last := task.InterimResults[len(task.InterimResults)-1]
+								fmt.Printf("│\n│ %s  Latest Update\n", last.Timestamp.Format("15:04:05"))
+								printJSONWithPrefix(last.Data, "│   ")
+							}
+
+							if len(task.InterimResults) > 2 {
+								fmt.Printf("│\n│ Use --long-updates to show all %d updates\n",
+									len(task.InterimResults))
+							}
+						}
+
+						// Add divider after progress updates section
+						fmt.Printf("│\n│ %s\n", strings.Repeat("─", 50))
+					}
+
 					// Print compensation history if present
 					if len(task.CompensationHistory) > 0 {
 						fmt.Printf("│ Compensating %s\n", strings.Repeat("─", 37))
@@ -149,6 +193,16 @@ func newInspectCmd(opts *CliOpts) *cobra.Command {
 	}
 
 	cmd.Flags().BoolVarP(&detailed, "detailed", "d", false, "Show detailed task history with I/O")
+	cmd.Flags().BoolVarP(&shortUpdates, "updates", "u", false, "Show summarized progress updates (first and last only)")
+	cmd.Flags().BoolVar(&longUpdates, "long-updates", false, "Show all progress updates with complete details")
+
+	cmd.PreRunE = func(cmd *cobra.Command, args []string) error {
+		if shortUpdates && longUpdates {
+			return fmt.Errorf("--updates and --long-updates flags cannot be used together")
+		}
+		return nil
+	}
+
 	return cmd
 }
 
@@ -199,3 +253,53 @@ func formatInspectionError(err string) string {
 	}
 	return err
 }
+
+func renderTaskTimeline(statusHistory []api.TaskStatusEvent, interimResults []api.InterimTaskResult) string {
+	if len(statusHistory) == 0 || len(interimResults) == 0 {
+		return ""
+	}
+
+	// Extract key information only
+	startTime := statusHistory[0].Timestamp
+	endTime := statusHistory[len(statusHistory)-1].Timestamp
+	duration := endTime.Sub(startTime).Seconds()
+
+	// Extract latest progress percentage if available
+	latestProgress := 0
+	for _, result := range interimResults {
+		var data map[string]interface{}
+		if err := json.Unmarshal(result.Data, &data); err == nil {
+			if p, ok := data["progress"].(float64); ok {
+				latestProgress = int(p)
+			} else if p, ok := data["percentComplete"].(float64); ok {
+				latestProgress = int(p)
+			}
+		}
+	}
+
+	// Build simple progress bar (inspired by kubectl)
+	var sb strings.Builder
+	sb.WriteString("│ Progress: [")
+
+	// Progress bar (20 chars max)
+	progressChars := 20
+	filledChars := int(float64(latestProgress) / 100 * float64(progressChars))
+	sb.WriteString(strings.Repeat("=", filledChars))
+
+	if filledChars < progressChars {
+		sb.WriteString(">") // Progress indicator
+		sb.WriteString(strings.Repeat(" ", progressChars-filledChars-1))
+	}
+
+	sb.WriteString(fmt.Sprintf("] %d%%", latestProgress))
+
+	// Add timing information
+	sb.WriteString(fmt.Sprintf("  (%.1fs elapsed)", duration))
+	sb.WriteString("\n")
+
+	// Add update count
+	sb.WriteString(fmt.Sprintf("│ Updates: %d", len(interimResults)))
+	sb.WriteString("\n")
+
+	return sb.String()
+}

cli/cmd/version.go

@@ -17,7 +17,7 @@ func newVersionCmd(_ *CliOpts) *cobra.Command {
 		Use:   "version",
 		Short: "Print the client and server version information",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			fmt.Println("Client Version: v0.2.2")
+			fmt.Println("Client Version: v0.2.3")
 			// TODO: Implement server version check
 			return nil
 		},

cli/internal/api/api.go

@@ -137,6 +137,7 @@ type TaskInspectResponse struct {
 	Output              json.RawMessage           `json:"output,omitempty"`
 	Error               string                    `json:"error,omitempty"`
 	Duration            time.Duration             `json:"duration"`
+	InterimResults      []InterimTaskResult       `json:"interimResults,omitempty"`
 	Compensation        *TaskCompensationStatus   `json:"compensation,omitempty"`
 	CompensationHistory []CompensationStatusEvent `json:"compensationHistory,omitempty"`
 	IsRevertible        bool                      `json:"isRevertible"`
@@ -149,6 +150,12 @@ type TaskCompensationStatus struct {
 	Timestamp   time.Time `json:"timestamp"`    // When compensation started
 }
 
+type InterimTaskResult struct {
+	ID        string          `json:"id"`
+	Timestamp time.Time       `json:"timestamp"`
+	Data      json.RawMessage `json:"data"`
+}
+
 type CompensationStatusEvent struct {
 	ID          string    `json:"id"`
 	TaskID      string    `json:"taskId"`

docs/cli.md

@@ -88,6 +88,21 @@ await svc.register({
 // Start handling tasks
 svc.start(async (task) => {
   // Your service logic here
+  
+  // Report progress during long-running tasks
+  await task.pushUpdate({
+    progress: 25,
+    message: "Processing customer data..."
+  });
+  
+  // Continue processing and report more progress
+  await task.pushUpdate({
+    progress: 50,
+    message: "Analyzing request..."
+  });
+  
+  // Complete the task with final result
+  return { success: true, data: { /* your result */ } };
 });
 ```
 
@@ -182,6 +197,12 @@ orra inspect o_abc123
 
 # Get comprehensive details with inputs/outputs
 orra inspect -d o_abc123
+
+# View progress updates for tasks
+orra inspect -d o_abc123 --updates
+
+# View complete progress details for long-running tasks
+orra inspect -d o_abc123 --long-updates
 ```
 
 ### Grounding Management
@@ -237,6 +258,10 @@ orra ps
 orra inspect -d o_failed123
 
 # Shows full task history, inputs, and outputs
+orra inspect -d o_failed123 --updates
+
+# View all progress updates for problematic tasks
+orra inspect -d o_failed123 --long-updates
 ```
 
 3. **Multiple Projects or environments**
@@ -282,9 +307,14 @@ orra config reset
    orra inspect -d <action-id>
    ```
 
-2. View recent actions:
+2. View task progress and updates:
+   ```bash
+   orra inspect -d <action-id> --updates
+   ```
+
+3. View recent actions:
    ```bash
    orra ps
    ```
 
-3. Visit our documentation: [https://orra.dev/docs](https://orra.dev/docs)
+4. Visit our documentation: [https://orra.dev/docs](https://orra.dev/docs)

docs/core.md

@@ -156,17 +156,31 @@ Orra uses an append-only log (similar to Kafka) for task coordination:
 - Immutable history for debugging
 - Basis for exactly-once execution
 - Clean recovery after failures
+- Storage for progress updates from long-running tasks
 
 Example log sequence:
 ```
 [0001] Task customer_context started
 [0002] Task order_details started
-[0003] Task customer_context completed
-[0004] Task order_details completed
-[0005] Task generate_response started
-[0006] Task generate_response completed
+[0003] Task customer_context interim_output (20% complete)
+[0004] Task customer_context interim_output (50% complete)
+[0005] Task order_details completed
+[0006] Task customer_context interim_output (90% complete)
+[0007] Task customer_context completed
+[0008] Task generate_response started
+[0009] Task generate_response completed
 ```
 
+#### Progress Updates in the Log
+
+The Plan Engine stores interim task results in the log as special entries with type `task_interim_output`. These entries:
+
+- Are appended while a task is running (before completion)
+- Maintain chronological ordering with other log entries
+- Are retrievable for debugging and monitoring purposes (e.g. inspect with the CLI `--updates` or `--long-updates`)
+
+This mechanism provides transparency into task execution without compromising the reliability of the core task processing model.
+
 ### Advanced Usage Patterns
 
 #### 1. Dynamic Service/Agent Registration
@@ -221,7 +235,25 @@ orra inspect -d o_xxxxxxxxxxxxxx
 │ inventory-service (task1)
 │ ──────────────────────────────────────────────────
 │ 14:07:43  ◎ Processing
-│ 14:07:43  ● Completed
+│ 14:07:44  ● Completed
+│
+│ Progress: [====================] 100%  (1.0s elapsed)
+│ Updates: 3
+│
+│ Progress Updates ───────────────────────────────────
+│ 14:07:43  First Update
+│   {
+│     "progress": 30,
+│     "status": "processing",
+│     "message": "Checking inventory"
+│   }
+│
+│ 14:07:44  Latest Update
+│   {
+│     "progress": 100,
+│     "status": "processing",
+│     "message": "Product located"
+│   }
 │
 │ Input:
 │   {

docs/sdks/js-sdk.md

@@ -187,6 +187,73 @@ service.onRevert(async (task, result) => {
 
 ## Advanced Features
 
+### Progress Updates
+
+For long-running tasks, you can send interim progress updates to the Plan Engine. This allows monitoring task execution in real-time and provides valuable information for debugging and audit logs.
+
+The update is any object with properties that make sense for the task underway.
+
+```javascript
+service.start(async (task) => {
+  try {
+    // 1. Begin processing
+    await task.pushUpdate({
+      progress: 20,
+      status: "processing",
+      message: "Starting data analysis"
+    });
+    
+    // 2. Continue with more steps
+    await someFunction();
+    await task.pushUpdate({
+      progress: 50,
+      status: "processing",
+      message: "Processing halfway complete"
+    });
+    
+    // 3. Almost done
+    await finalSteps();
+    await task.pushUpdate({
+      progress: 90,
+      status: "processing",
+      message: "Finalizing results"
+    });
+    
+    // 4. Return final result
+    return { success: true, results: [...] };
+  } catch (error) {
+    // Handle errors
+    throw error;
+  }
+});
+```
+
+#### Benefits of Progress Updates
+
+- **Visibility**: Track execution of long-running tasks in real-time
+- **Debugging**: Identify exactly where tasks slow down or fail
+- **Audit Trail**: Maintain a complete history of task execution
+- **User Experience**: Forward progress information to end-users
+
+#### Viewing Progress Updates
+
+Use the Orra CLI to view progress updates:
+
+```bash
+# View summarized progress updates
+orra inspect -d <orchestration-id> --updates
+
+# View all detailed progress updates
+orra inspect -d <orchestration-id> --long-updates
+```
+
+#### Best Practices
+
+- Send updates at meaningful milestones (not too frequent)
+- Include percentage completion when possible
+- Keep messages concise and informative
+- Use consistent status terminology
+
 ### Persistence Configuration
 
 Orra's Plan Engine maintains service/agent identity across restarts using persistence. This is crucial for: