Add tags to connector run failures indicating if it's user or framewo…

…rk error (elastic#197818)

Resolves elastic#197315

In this PR, I'm adding the following tags to the connector failure logs
so it makes it easier to filter for systematic errors.

- `connector-run-failed` for logs specific to connector run failures
- `user-error` for errors caused by the user
- `framework-error` for systematic errors

## To verify

You can either use the jest test to observe the returned flags or set
your logging to JSON and make connectors fail.

kibana.yml to set logging to JSON
```
logging:
  appenders:
    json-layout:
      type: console
      layout:
        type: json
  root:
    appenders: [json-layout]
```

x-pack/plugins/actions/server/lib/task_runner_factory.test.ts

@@ -885,7 +885,8 @@ describe('Task Runner Factory', () => {
     expect(err).toBeDefined();
     expect(isRetryableError(err)).toEqual(false);
     expect(taskRunnerFactoryInitializerParams.logger.error as jest.Mock).toHaveBeenCalledWith(
-      `Action '2' failed: Error message`
+      `Action '2' failed: Error message`,
+      { tags: ['connector-run-failed', 'framework-error'] }
     );
     expect(getErrorSource(err)).toBe(TaskErrorSource.FRAMEWORK);
   });
@@ -934,7 +935,8 @@ describe('Task Runner Factory', () => {
 
     expect(err).toBeDefined();
     expect(taskRunnerFactoryInitializerParams.logger.error as jest.Mock).toHaveBeenCalledWith(
-      `Action '2' failed: Error message: Service message`
+      `Action '2' failed: Error message: Service message`,
+      { tags: ['connector-run-failed', 'framework-error'] }
     );
   });
 
@@ -1033,7 +1035,8 @@ describe('Task Runner Factory', () => {
     }
     expect(err).toBeDefined();
     expect(taskRunnerFactoryInitializerParams.logger.error as jest.Mock).toHaveBeenCalledWith(
-      `Action '2' failed: Fail`
+      `Action '2' failed: Fail`,
+      { tags: ['connector-run-failed', 'framework-error'] }
     );
     expect(thrownError).toEqual(err);
     expect(getErrorSource(err)).toBe(TaskErrorSource.FRAMEWORK);
@@ -1140,10 +1143,16 @@ describe('Task Runner Factory', () => {
 
     try {
       await taskRunner.run();
+      throw new Error('Should have thrown');
     } catch (e) {
       expect(mockedEncryptedSavedObjectsClient.getDecryptedAsInternalUser).toHaveBeenCalledTimes(1);
       expect(getErrorSource(e)).toBe(TaskErrorSource.FRAMEWORK);
       expect(e).toEqual(error);
+
+      expect(taskRunnerFactoryInitializerParams.logger.error).toHaveBeenCalledWith(
+        `Failed to load action task params ${mockedTaskInstance.params.actionTaskParamsId}: test`,
+        { tags: ['connector-run-failed', 'framework-error'] }
+      );
     }
   });
 });

x-pack/plugins/actions/server/lib/task_runner_factory.ts

@@ -115,7 +115,8 @@ export class TaskRunnerFactory {
         } = await getActionTaskParams(
           actionTaskExecutorParams,
           encryptedSavedObjectsClient,
-          spaceIdToNamespace
+          spaceIdToNamespace,
+          logger
         );
 
         const { spaceId } = actionTaskExecutorParams;
@@ -139,12 +140,18 @@ export class TaskRunnerFactory {
             ...getSource(references, source),
           });
         } catch (e) {
-          logger.error(`Action '${actionId}' failed: ${e.message}`);
+          const errorSource =
+            e instanceof ActionTypeDisabledError
+              ? TaskErrorSource.USER
+              : getErrorSource(e) || TaskErrorSource.FRAMEWORK;
+          logger.error(`Action '${actionId}' failed: ${e.message}`, {
+            tags: ['connector-run-failed', `${errorSource}-error`],
+          });
           if (e instanceof ActionTypeDisabledError) {
             // We'll stop re-trying due to action being forbidden
-            throwUnrecoverableError(createTaskRunError(e, TaskErrorSource.USER));
+            throwUnrecoverableError(createTaskRunError(e, errorSource));
           }
-          throw createTaskRunError(e, getErrorSource(e) || TaskErrorSource.FRAMEWORK);
+          throw createTaskRunError(e, errorSource);
         }
 
         inMemoryMetrics.increment(IN_MEMORY_METRICS.ACTION_EXECUTIONS);
@@ -155,7 +162,9 @@ export class TaskRunnerFactory {
           if (executorResult.serviceMessage) {
             message = `${message}: ${executorResult.serviceMessage}`;
           }
-          logger.error(`Action '${actionId}' failed: ${message}`);
+          logger.error(`Action '${actionId}' failed: ${message}`, {
+            tags: ['connector-run-failed', `${executorResult.errorSource}-error`],
+          });
 
           // Task manager error handler only kicks in when an error thrown (at this time)
           // So what we have to do is throw when the return status is `error`.
@@ -175,7 +184,8 @@ export class TaskRunnerFactory {
         } = await getActionTaskParams(
           actionTaskExecutorParams,
           encryptedSavedObjectsClient,
-          spaceIdToNamespace
+          spaceIdToNamespace,
+          logger
         );
 
         const request = getFakeRequest(apiKey);
@@ -239,7 +249,8 @@ function getFakeRequest(apiKey?: string) {
 async function getActionTaskParams(
   executorParams: ActionTaskExecutorParams,
   encryptedSavedObjectsClient: EncryptedSavedObjectsClient,
-  spaceIdToNamespace: SpaceIdToNamespaceFunction
+  spaceIdToNamespace: SpaceIdToNamespaceFunction,
+  logger: Logger
 ): Promise<TaskParams> {
   const { spaceId } = executorParams;
   const namespace = spaceIdToNamespace(spaceId);
@@ -268,10 +279,17 @@ async function getActionTaskParams(
         },
       };
     } catch (e) {
+      const errorSource = SavedObjectsErrorHelpers.isNotFoundError(e)
+        ? TaskErrorSource.USER
+        : TaskErrorSource.FRAMEWORK;
+      logger.error(
+        `Failed to load action task params ${executorParams.actionTaskParamsId}: ${e.message}`,
+        { tags: ['connector-run-failed', `${errorSource}-error`] }
+      );
       if (SavedObjectsErrorHelpers.isNotFoundError(e)) {
-        throw createRetryableError(createTaskRunError(e, TaskErrorSource.USER), true);
+        throw createRetryableError(createTaskRunError(e, errorSource), true);
       }
-      throw createRetryableError(createTaskRunError(e, TaskErrorSource.FRAMEWORK), true);
+      throw createRetryableError(createTaskRunError(e, errorSource), true);
     }
   } else {
     return { attributes: executorParams.taskParams, references: executorParams.references ?? [] };