HARMONY-2159: Add support in Harmony to handle non-retriable errors from service.

Author: ygliuvt

services/harmony/app/backends/workflow-orchestration/work-item-updates.ts

@@ -708,7 +708,13 @@ export async function processWorkItem(
 
     // retry failed work-items up to a limit
     if (status === WorkItemStatus.FAILED) {
-      if (workItem.retryCount < env.workItemRetryLimit) {
+      if (message_category === 'noretry') {
+        logger.warn('This error is not retriable.');
+        logger.warn(
+          `Updating work item for ${workItemID} to ${status} with message ${message}`,
+          { workFailureMessage: message, serviceId: workItem.serviceID, status },
+        );
+      } else if (workItem.retryCount < env.workItemRetryLimit) {
         logger.info(`Retrying failed work-item ${workItemID}`);
         workItem.retryCount += 1;
         workItem.status = WorkItemStatus.READY;

services/harmony/test/work-items/retries.ts

@@ -87,7 +87,7 @@ describe('Work item failure retries', function () {
       });
     });
 
-    describe('And a work-item fails the first time', async function () {
+    describe('And a work-item fails two times', async function () {
       hookRangesetRequest('1.0.0', collection, 'all', { query: { ...reprojectQuery, ...{ maxResults: 1 } } });
       hookRedirect('joe');
       before(async function () {
@@ -147,7 +147,8 @@ describe('Work item failure retries', function () {
         });
       });
     });
-    describe('And a work-item fails the first time', async function () {
+
+    describe('And a work-item fails three times', async function () {
       hookRangesetRequest('1.0.0', collection, 'all', { query: { ...reprojectQuery, ...{ maxResults: 1 } } });
       hookRedirect('joe');
       before(async function () {
@@ -209,5 +210,32 @@ describe('Work item failure retries', function () {
         });
       });
     });
+
+    describe('And a work-item fails with no retry error', async function () {
+      hookRangesetRequest('1.0.0', collection, 'all', { query: { ...reprojectQuery, ...{ maxResults: 1 } } });
+      hookRedirect('joe');
+      before(async function () {
+        const res = await getWorkForService(this.backend, 'harmonyservices/query-cmr:stable');
+        const { workItem } = JSON.parse(res.text);
+
+        workItem.status = WorkItemStatus.FAILED;
+        workItem.message = 'The service returns noretry error';
+        workItem.message_category = 'noretry';
+        workItem.results = [];
+
+        await updateWorkItem(this.backend, workItem);
+
+        this.workItem = await getWorkItemById(db, workItem.id);
+      });
+
+      it('Changes the work-item status to failed immediately without retries', async function () {
+        expect(this.workItem.status).to.equal(WorkItemStatus.FAILED);
+        expect(this.workItem.retryCount).to.equal(0);
+      });
+      it('changes the work-item status to failed', async function () {
+        const { job } = await Job.byJobID(db, this.workItem.jobID);
+        expect(job.status).to.equal(JobStatus.FAILED);
+      });
+    });
   });
 });