Add application err category (#2485)

* update protos

* add ApplicationErrorCategory enum

* Add category field to ApplicationFailure

* add category proto converter logic

* add logging/metrics checks for benign application failures

* fixes, added test for workflow failure metric

* fixes for activity failures, added test for activity failures

* address PR review

* only check immediate failure

* define code enum, convert to/from proto representation

* cleanup

Author: THardy98

temporal-sdk/src/main/java/io/temporal/failure/ApplicationErrorCategory.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2022 Temporal Technologies, Inc. All Rights Reserved.
+ *
+ * Copyright (C) 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Modifications copyright (C) 2017 Uber Technologies, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this material except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.temporal.failure;
+
+/**
+ * Used to categorize application failures, for example, to distinguish benign errors from others.
+ *
+ * @see io.temporal.api.enums.v1.ApplicationErrorCategory
+ */
+public enum ApplicationErrorCategory {
+  UNSPECIFIED,
+  /** Expected application error with little/no severity. */
+  BENIGN,
+  ;
+}

temporal-sdk/src/main/java/io/temporal/failure/ApplicationFailure.java

@@ -51,13 +51,15 @@
  *   <li>nonRetryable is set to false
  *   <li>details are set to null
  *   <li>stack trace is copied from the original exception
+ *   <li>category is set to ApplicationErrorCategory.APPLICATION_ERROR_CATEGORY_UNSPECIFIED
  * </ul>
  */
 public final class ApplicationFailure extends TemporalFailure {
   private final String type;
   private final Values details;
   private boolean nonRetryable;
   private Duration nextRetryDelay;
+  private final ApplicationErrorCategory category;
 
   /**
    * New ApplicationFailure with {@link #isNonRetryable()} flag set to false.
@@ -92,7 +94,14 @@ public static ApplicationFailure newFailure(String message, String type, Object.
    */
   public static ApplicationFailure newFailureWithCause(
       String message, String type, @Nullable Throwable cause, Object... details) {
-    return new ApplicationFailure(message, type, false, new EncodedValues(details), cause, null);
+    return new ApplicationFailure(
+        message,
+        type,
+        false,
+        new EncodedValues(details),
+        cause,
+        null,
+        ApplicationErrorCategory.UNSPECIFIED);
   }
 
   /**
@@ -118,7 +127,13 @@ public static ApplicationFailure newFailureWithCauseAndDelay(
       Duration nextRetryDelay,
       Object... details) {
     return new ApplicationFailure(
-        message, type, false, new EncodedValues(details), cause, nextRetryDelay);
+        message,
+        type,
+        false,
+        new EncodedValues(details),
+        cause,
+        nextRetryDelay,
+        ApplicationErrorCategory.UNSPECIFIED);
   }
 
   /**
@@ -153,7 +168,40 @@ public static ApplicationFailure newNonRetryableFailure(
    */
   public static ApplicationFailure newNonRetryableFailureWithCause(
       String message, String type, @Nullable Throwable cause, Object... details) {
-    return new ApplicationFailure(message, type, true, new EncodedValues(details), cause, null);
+    return new ApplicationFailure(
+        message,
+        type,
+        true,
+        new EncodedValues(details),
+        cause,
+        null,
+        ApplicationErrorCategory.UNSPECIFIED);
+  }
+
+  /**
+   * New ApplicationFailure with a specified category and {@link #isNonRetryable()} flag set to
+   * false.
+   *
+   * <p>Note that this exception still may not be retried by the service if its type is included in
+   * the doNotRetry property of the correspondent retry policy.
+   *
+   * @param message optional error message
+   * @param type error type
+   * @param category the category of the application failure.
+   * @param cause failure cause. Each element of the cause chain will be converted to
+   *     ApplicationFailure for network transmission across network if it doesn't extend {@link
+   *     TemporalFailure}
+   * @param details optional details about the failure. They are serialized using the same approach
+   *     as arguments and results.
+   */
+  public static ApplicationFailure newFailureWithCategory(
+      String message,
+      String type,
+      ApplicationErrorCategory category,
+      @Nullable Throwable cause,
+      Object... details) {
+    return new ApplicationFailure(
+        message, type, false, new EncodedValues(details), cause, null, category);
   }
 
   static ApplicationFailure newFromValues(
@@ -162,8 +210,10 @@ static ApplicationFailure newFromValues(
       boolean nonRetryable,
       Values details,
       Throwable cause,
-      Duration nextRetryDelay) {
-    return new ApplicationFailure(message, type, nonRetryable, details, cause, nextRetryDelay);
+      Duration nextRetryDelay,
+      ApplicationErrorCategory category) {
+    return new ApplicationFailure(
+        message, type, nonRetryable, details, cause, nextRetryDelay, category);
   }
 
   ApplicationFailure(
@@ -172,12 +222,14 @@ static ApplicationFailure newFromValues(
       boolean nonRetryable,
       Values details,
       Throwable cause,
-      Duration nextRetryDelay) {
+      Duration nextRetryDelay,
+      ApplicationErrorCategory category) {
     super(getMessage(message, Objects.requireNonNull(type), nonRetryable), message, cause);
     this.type = type;
     this.details = details;
     this.nonRetryable = nonRetryable;
     this.nextRetryDelay = nextRetryDelay;
+    this.category = category;
   }
 
   public String getType() {
@@ -210,6 +262,10 @@ public void setNextRetryDelay(Duration nextRetryDelay) {
     this.nextRetryDelay = nextRetryDelay;
   }
 
+  public ApplicationErrorCategory getApplicationErrorCategory() {
+    return category;
+  }
+
   private static String getMessage(String message, String type, boolean nonRetryable) {
     return (Strings.isNullOrEmpty(message) ? "" : "message='" + message + "', ")
         + "type='"

temporal-sdk/src/main/java/io/temporal/failure/DefaultFailureConverter.java

@@ -34,6 +34,7 @@
 import io.temporal.common.converter.EncodedValues;
 import io.temporal.common.converter.FailureConverter;
 import io.temporal.internal.activity.ActivityTaskHandlerImpl;
+import io.temporal.internal.common.FailureUtils;
 import io.temporal.internal.common.ProtobufTimeUtils;
 import io.temporal.internal.sync.POJOWorkflowImplementationFactory;
 import io.temporal.serviceclient.CheckedExceptionWrapper;
@@ -106,7 +107,8 @@ private RuntimeException failureToExceptionImpl(Failure failure, DataConverter d
               cause,
               info.hasNextRetryDelay()
                   ? ProtobufTimeUtils.toJavaDuration(info.getNextRetryDelay())
-                  : null);
+                  : null,
+              FailureUtils.categoryFromProto(info.getCategory()));
         }
       case TIMEOUT_FAILURE_INFO:
         {
@@ -146,13 +148,14 @@ private RuntimeException failureToExceptionImpl(Failure failure, DataConverter d
               info.hasLastHeartbeatDetails()
                   ? Optional.of(info.getLastHeartbeatDetails())
                   : Optional.empty();
-          return new ApplicationFailure(
+          return ApplicationFailure.newFromValues(
               failure.getMessage(),
               "ResetWorkflow",
               false,
               new EncodedValues(details, dataConverter),
               cause,
-              null);
+              null,
+              ApplicationErrorCategory.UNSPECIFIED);
         }
       case ACTIVITY_FAILURE_INFO:
         {
@@ -214,7 +217,8 @@ private RuntimeException failureToExceptionImpl(Failure failure, DataConverter d
             false,
             new EncodedValues(Optional.empty(), dataConverter),
             cause,
-            null);
+            null,
+            ApplicationErrorCategory.UNSPECIFIED);
     }
   }
 
@@ -260,7 +264,8 @@ private Failure exceptionToFailure(Throwable throwable) {
       ApplicationFailureInfo.Builder info =
           ApplicationFailureInfo.newBuilder()
               .setType(ae.getType())
-              .setNonRetryable(ae.isNonRetryable());
+              .setNonRetryable(ae.isNonRetryable())
+              .setCategory(FailureUtils.categoryToProto(ae.getApplicationErrorCategory()));
       Optional<Payloads> details = ((EncodedValues) ae.getDetails()).toPayloads();
       if (details.isPresent()) {
         info.setDetails(details.get());
@@ -352,7 +357,10 @@ private Failure exceptionToFailure(Throwable throwable) {
       ApplicationFailureInfo.Builder info =
           ApplicationFailureInfo.newBuilder()
               .setType(throwable.getClass().getName())
-              .setNonRetryable(false);
+              .setNonRetryable(false)
+              .setCategory(
+                  io.temporal.api.enums.v1.ApplicationErrorCategory
+                      .APPLICATION_ERROR_CATEGORY_UNSPECIFIED);
       failure.setApplicationFailureInfo(info);
     }
     return failure.build();

temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java

@@ -36,6 +36,7 @@
 import io.temporal.common.interceptors.ActivityInboundCallsInterceptor.ActivityOutput;
 import io.temporal.common.interceptors.Header;
 import io.temporal.common.interceptors.WorkerInterceptor;
+import io.temporal.internal.common.FailureUtils;
 import io.temporal.internal.worker.ActivityTaskHandler;
 import io.temporal.payload.context.ActivitySerializationContext;
 import io.temporal.serviceclient.CheckedExceptionWrapper;
@@ -122,6 +123,14 @@ public ActivityTaskHandler.Result execute(ActivityInfoInternal info, Scope metri
               info.getActivityId(),
               info.getActivityType(),
               info.getAttempt());
+        } else if (FailureUtils.isBenignApplicationFailure(ex)) {
+          log.debug(
+              "{} failure. ActivityId={}, activityType={}, attempt={}",
+              local ? "Local activity" : "Activity",
+              info.getActivityId(),
+              info.getActivityType(),
+              info.getAttempt(),
+              ex);
         } else {
           log.warn(
               "{} failure. ActivityId={}, activityType={}, attempt={}",

temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskHandlerImpl.java

@@ -36,6 +36,7 @@
 import io.temporal.common.metadata.POJOActivityImplMetadata;
 import io.temporal.common.metadata.POJOActivityMethodMetadata;
 import io.temporal.internal.activity.ActivityTaskExecutors.ActivityTaskExecutor;
+import io.temporal.internal.common.FailureUtils;
 import io.temporal.internal.common.env.ReflectionUtils;
 import io.temporal.internal.worker.ActivityTask;
 import io.temporal.internal.worker.ActivityTaskHandler;
@@ -209,11 +210,13 @@ static ActivityTaskHandler.Result mapToActivityFailure(
     Scope ms =
         metricsScope.tagged(
             ImmutableMap.of(MetricsTag.EXCEPTION, exception.getClass().getSimpleName()));
-    if (isLocalActivity) {
-      ms.counter(MetricsType.LOCAL_ACTIVITY_EXEC_FAILED_COUNTER).inc(1);
-      ms.counter(MetricsType.LOCAL_ACTIVITY_FAILED_COUNTER).inc(1);
-    } else {
-      ms.counter(MetricsType.ACTIVITY_EXEC_FAILED_COUNTER).inc(1);
+    if (!FailureUtils.isBenignApplicationFailure(exception)) {
+      if (isLocalActivity) {
+        ms.counter(MetricsType.LOCAL_ACTIVITY_EXEC_FAILED_COUNTER).inc(1);
+        ms.counter(MetricsType.LOCAL_ACTIVITY_FAILED_COUNTER).inc(1);
+      } else {
+        ms.counter(MetricsType.ACTIVITY_EXEC_FAILED_COUNTER).inc(1);
+      }
     }
     Failure failure = dataConverter.exceptionToFailure(exception);
     RespondActivityTaskFailedRequest.Builder result =

temporal-sdk/src/main/java/io/temporal/internal/common/FailureUtils.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2022 Temporal Technologies, Inc. All Rights Reserved.
+ *
+ * Copyright (C) 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Modifications copyright (C) 2017 Uber Technologies, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this material except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.temporal.internal.common;
+
+import io.temporal.api.failure.v1.Failure;
+import io.temporal.failure.ApplicationErrorCategory;
+import io.temporal.failure.ApplicationFailure;
+import javax.annotation.Nullable;
+
+public class FailureUtils {
+  private FailureUtils() {}
+
+  public static boolean isBenignApplicationFailure(@Nullable Throwable t) {
+    if (t instanceof ApplicationFailure
+        && ((ApplicationFailure) t).getApplicationErrorCategory()
+            == ApplicationErrorCategory.BENIGN) {
+      return true;
+    }
+    return false;
+  }
+
+  public static boolean isBenignApplicationFailure(@Nullable Failure failure) {
+    if (failure != null
+        && failure.getApplicationFailureInfo() != null
+        && FailureUtils.categoryFromProto(failure.getApplicationFailureInfo().getCategory())
+            == ApplicationErrorCategory.BENIGN) {
+      return true;
+    }
+    return false;
+  }
+
+  public static ApplicationErrorCategory categoryFromProto(
+      io.temporal.api.enums.v1.ApplicationErrorCategory protoCategory) {
+    if (protoCategory == null) {
+      return ApplicationErrorCategory.UNSPECIFIED;
+    }
+    switch (protoCategory) {
+      case APPLICATION_ERROR_CATEGORY_BENIGN:
+        return ApplicationErrorCategory.BENIGN;
+      case APPLICATION_ERROR_CATEGORY_UNSPECIFIED:
+      case UNRECOGNIZED:
+      default:
+        // Fallback unrecognized or unspecified proto values as UNSPECIFIED
+        return ApplicationErrorCategory.UNSPECIFIED;
+    }
+  }
+
+  public static io.temporal.api.enums.v1.ApplicationErrorCategory categoryToProto(
+      io.temporal.failure.ApplicationErrorCategory category) {
+    switch (category) {
+      case BENIGN:
+        return io.temporal.api.enums.v1.ApplicationErrorCategory.APPLICATION_ERROR_CATEGORY_BENIGN;
+      case UNSPECIFIED:
+      default:
+        // Fallback to UNSPECIFIED for unknown values
+        return io.temporal.api.enums.v1.ApplicationErrorCategory
+            .APPLICATION_ERROR_CATEGORY_UNSPECIFIED;
+    }
+  }
+}

temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowExecutor.java

@@ -34,6 +34,7 @@
 import io.temporal.api.update.v1.Input;
 import io.temporal.api.update.v1.Request;
 import io.temporal.failure.CanceledFailure;
+import io.temporal.internal.common.FailureUtils;
 import io.temporal.internal.common.ProtobufTimeUtils;
 import io.temporal.internal.common.UpdateMessage;
 import io.temporal.internal.statemachines.WorkflowStateMachines;
@@ -153,7 +154,9 @@ private void completeWorkflow(@Nullable WorkflowExecutionException failure) {
       metricsScope.counter(MetricsType.WORKFLOW_CANCELED_COUNTER).inc(1);
     } else if (failure != null) {
       workflowStateMachines.failWorkflow(failure.getFailure());
-      metricsScope.counter(MetricsType.WORKFLOW_FAILED_COUNTER).inc(1);
+      if (!FailureUtils.isBenignApplicationFailure(failure.getFailure())) {
+        metricsScope.counter(MetricsType.WORKFLOW_FAILED_COUNTER).inc(1);
+      }
     } else {
       ContinueAsNewWorkflowExecutionCommandAttributes attributes =
           context.getContinueAsNewOnCompletion();