fix: async jobs stuck in processing on marshal failure (#1907)

## Summary

Fixes async jobs that become stuck in "processing" state when JSON marshaling fails during job execution. Previously, marshal failures would cause the job to exit without updating its status, leaving it permanently in the processing state.

## Changes

- Added panic recovery mechanism to ensure async jobs always reach a terminal state even if unexpected panics occur
- Introduced `markFailed` helper function to consistently update job status to "failed" with appropriate error details
- Added explicit error handling for JSON marshal failures that now properly transition jobs to "failed" status instead of leaving them stuck
- Jobs that fail during error serialization or result serialization now get marked as failed with descriptive error messages

## Type of change

- [x] Bug fix
- [ ] Feature
- [ ] Refactor
- [ ] Documentation
- [ ] Chore/CI

## Affected areas

- [x] Core (Go)
- [ ] Transports (HTTP)
- [ ] Providers/Integrations
- [ ] Plugins
- [ ] UI (Next.js)
- [ ] Docs

## How to test

Test async job execution with marshal failures to verify jobs transition to failed state:

```sh
# Core/Transports
go version
go test ./...

# Test specific async job scenarios
go test ./framework/logstore -v -run TestAsyncJob
```

## Screenshots/Recordings

N/A

## Breaking changes

- [ ] Yes
- [x] No

## Related issues

N/A

## Security considerations

No security implications. This change improves system reliability by preventing jobs from being stuck in processing state.

## Checklist

- [ ] I read `docs/contributing/README.md` and followed the guidelines
- [ ] I added/updated tests where appropriate
- [ ] I updated documentation where needed
- [ ] I verified builds succeed (Go and UI)
- [ ] I verified the CI pipeline passes locally if applicable

Author: Pratham-Mishra04

framework/logstore/asyncjob.go

@@ -118,6 +118,31 @@ func (e *AsyncJobExecutor) SubmitJob(virtualKeyValue *string, resultTTL int, ope
 func (e *AsyncJobExecutor) executeJob(jobID string, resultTTL int, operation AsyncOperation) {
 	ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline)
 
+	markFailed := func(msg string) {
+		now := time.Now().UTC()
+		expiresAt := now.Add(time.Duration(resultTTL) * time.Second)
+		errJSON, _ := sonic.Marshal(&schemas.BifrostError{Error: &schemas.ErrorField{Message: msg}})
+		if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{
+			"status":       schemas.AsyncJobStatusFailed,
+			"status_code":  fasthttp.StatusInternalServerError,
+			"error":        string(errJSON),
+			"completed_at": now,
+			"expires_at":   expiresAt,
+		}); err != nil {
+			e.logger.Warn("failed to update async job to failed: %v", err)
+		}
+	}
+
+	// The bifrost execution flow is very stable and panics are not expected.
+	// This recover is purely defensive to ensure the job always reaches a terminal
+	// state rather than being stuck in "processing" if an unexpected panic occurs.
+	defer func() {
+		if r := recover(); r != nil {
+			e.logger.Warn("async job %s panicked: %v", jobID, r)
+			markFailed(fmt.Sprintf("internal error: %v", r))
+		}
+	}()
+
 	// Mark as processing
 	if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{
 		"status": schemas.AsyncJobStatusProcessing,
@@ -137,6 +162,7 @@ func (e *AsyncJobExecutor) executeJob(jobID string, resultTTL int, operation Asy
 		errJSON, err := sonic.Marshal(bifrostErr)
 		if err != nil {
 			e.logger.Warn("failed to marshal bifrost error: %v", err)
+			markFailed(fmt.Sprintf("failed to serialize error response: %v", err))
 			return
 		}
 		statusCode := fasthttp.StatusInternalServerError
@@ -158,6 +184,7 @@ func (e *AsyncJobExecutor) executeJob(jobID string, resultTTL int, operation Asy
 	respJSON, err := sonic.Marshal(resp)
 	if err != nil {
 		e.logger.Warn("failed to marshal result: %v", err)
+		markFailed(fmt.Sprintf("failed to serialize result: %v", err))
 		return
 	}
 	if err := e.logstore.UpdateAsyncJob(ctx, jobID, map[string]interface{}{

transports/changelog.md

@@ -1 +1,2 @@
-- fix: preserve original audio filename in transcription requests
+- fix: preserve original audio filename in transcription requests
+- fix: async jobs stuck in "processing" on marshal failure now correctly transition to "failed"