chore: log stack trace during run crash

pkg/server/handler/stack/execute_async.go

@@ -76,7 +76,8 @@ func (h *Handler) PreviewStackAsync() http.HandlerFunc {
 			logger.Info("Async preview in progress")
 			var previewChanges any
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {
@@ -178,7 +179,8 @@ func (h *Handler) ApplyStackAsync() http.HandlerFunc {
 			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async apply in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {
@@ -279,9 +281,10 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc {
 			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async generate in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			var sp *apiv1.Spec
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
+			var sp *apiv1.Spec
 			// update status of the run when exiting the async run
 			defer func() {
 				select {
@@ -371,10 +374,10 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc {
 
 		// Starts a safe goroutine using given recover handler
 		inBufferZone := h.workerPool.Do(func() {
-			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async destroy in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {

pkg/server/handler/stack/utils.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -169,3 +170,20 @@ func CopyToNewContextWithTimeout(ctx context.Context, timeout time.Duration) (co
 	newCtxWithTimeout, cancel := context.WithTimeout(newCtx, timeout)
 	return newCtxWithTimeout, cancel
 }
+
+func logStackTrace(runLogger *httplog.Logger) {
+	buf := make([]byte, 1<<16) // 64KB
+	stackSize := runtime.Stack(buf, true)
+	runLogger.Error("Stack trace:")
+	runLogger.Error(string(buf[:stackSize]))
+}
+
+func handleCrash(ctx context.Context) {
+	if r := recover(); r != nil {
+		logger := logutil.GetLogger(ctx)
+		runLogger := logutil.GetRunLogger(ctx)
+		logger.Error("Recovered from panic during async execution:", "error", r)
+		runLogger.Error("Panic recovered", "error", r)
+		logStackTrace(runLogger)
+	}
+}

pkg/server/manager/stack/execute.go

@@ -65,9 +65,6 @@ func (m *StackManager) GenerateSpec(ctx context.Context, params *StackRequestPar
 
 	// Otherwise, generate spec from stack entity using the default generator
 	project, stack, wsBackend, err := m.getStackProjectAndBackend(ctx, stackEntity, params.Workspace)
-	if err != nil {
-		return "", nil, err
-	}
 	wsStorage, err := wsBackend.WorkspaceStorage()
 	if err != nil {
 		return "", nil, err