feat(Statement): Add alternative publish to queue logic without using Redis (LLC-2162) (#890)

Author: PrinceWaune

.env.example

@@ -95,6 +95,10 @@ EXPRESS_PORT=8081
 ###############################
 # Uncomment next line if you want to enable statement handling priority
 #ENABLE_QUEUE_PRIORITY=true
+# Event provider(redis|sqs). Redis by default
+#EVENTS_REPO=redis
+# Queue namespace
+#QUEUE_NAMESPACE=DEV
 
 ##########
 # Misc #

package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.100.0",
+    "@aws-sdk/client-sqs": "^3.282.0",
     "@aws-sdk/lib-storage": "^3.100.0",
     "@azure/storage-blob": "^10.3.0",
     "@google-cloud/storage": "^5.8.1",
@@ -208,6 +209,8 @@
       "functional/prefer-type-literal": "off",
       "functional/no-throw-statement": "off",
       "functional/no-try-statement": "off",
+      "functional/no-let": "off",
+      "functional/no-loop-statement": "off",
       "functional/prefer-readonly-type": [
         "error",
         {

src/apps/AppConfig.ts

@@ -3,6 +3,7 @@ import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
+import { SQSClient } from '@aws-sdk/client-sqs';
 
 export default interface AppConfig {
   readonly repo: {
@@ -46,6 +47,11 @@ export default interface AppConfig {
       readonly client: () => Promise<Redis>;
       readonly isQueuePriorityEnabled: boolean;
     };
+    readonly sqs: {
+      readonly prefix: string;
+      readonly client: () => Promise<SQSClient>;
+      readonly isQueuePriorityEnabled: boolean;
+    };
   };
   readonly service: {
     readonly statements: {

src/apps/app.ts

@@ -81,6 +81,7 @@ export default (appConfig: AppConfig): Router => {
       local: appConfig.repo.local,
       mongo: appConfig.repo.mongo,
       redis: appConfig.repo.redis,
+      sqs: appConfig.repo.sqs,
       s3: appConfig.repo.s3,
       storageSubFolder: appConfig.repo.storageSubFolders.statements,
     },

src/apps/statements/AppConfig.ts

@@ -3,6 +3,7 @@ import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
+import { SQSClient } from '@aws-sdk/client-sqs';
 
 export default interface AppConfig {
   readonly logger: LoggerInstance;
@@ -62,5 +63,10 @@ export default interface AppConfig {
       readonly client: () => Promise<Redis>;
       readonly isQueuePriorityEnabled: boolean;
     };
+    readonly sqs: {
+      readonly prefix: string;
+      readonly client: () => Promise<SQSClient>;
+      readonly isQueuePriorityEnabled: boolean;
+    };
   };
 }

src/apps/statements/app.ts

@@ -21,6 +21,11 @@ export default (appConfig: AppConfig): Result => {
         prefix: appConfig.repo.redis.prefix,
         isQueuePriorityEnabled: appConfig.repo.redis.isQueuePriorityEnabled,
       },
+      sqs: {
+        client: appConfig.repo.sqs.client,
+        prefix: appConfig.repo.sqs.prefix,
+        isQueuePriorityEnabled: appConfig.repo.sqs.isQueuePriorityEnabled,
+      },
     },
     models: {
       facade: appConfig.repo.factory.modelsRepoName,

src/apps/statements/repo/eventsRepo/FactoryConfig.ts

@@ -1,6 +1,8 @@
 import RedisFactoryConfig from './utils/redisEvents/FactoryConfig';
+import SQSFactoryConfig from './utils/sqsEvents/FactoryConfig';
 
 export default interface FactoryConfig {
   readonly facade?: string;
   readonly redis?: RedisFactoryConfig;
+  readonly sqs?: SQSFactoryConfig;
 }

src/apps/statements/repo/eventsRepo/emitNewStatements/sqs.ts

@@ -0,0 +1,70 @@
+import {
+  GetQueueUrlCommand,
+  SQSClient,
+  SendMessageBatchCommand,
+  SendMessageBatchRequestEntry,
+} from '@aws-sdk/client-sqs';
+import { v4 } from 'uuid';
+import { getPrefixWithProcessingPriority } from '../utils/getPrefixWithProcessingPriority';
+import { StatementProcessingPriority } from '../../../enums/statementProcessingPriority.enum';
+import FacadeConfig from '../utils/sqsEvents/FacadeConfig';
+import { STATEMENT_QUEUE } from '../utils/constants';
+import Signature from './Signature';
+
+const MAX_BATCH_SIZE = 10;
+
+let queueUrl: string | undefined;
+
+const publishMessages = async (sqsClient: SQSClient, statementProperties: string[]) => {
+  const statementPropertiesBatchRequest = statementProperties.map(
+    (statementProperty): SendMessageBatchRequestEntry => ({
+      Id: v4(),
+      MessageBody: statementProperty,
+    }),
+  );
+
+  for (let index = 0; index < statementPropertiesBatchRequest.length; index += MAX_BATCH_SIZE) {
+    await sqsClient.send(
+      new SendMessageBatchCommand({
+        QueueUrl: queueUrl,
+        Entries: statementPropertiesBatchRequest.slice(index, index + MAX_BATCH_SIZE),
+      }),
+    );
+  }
+};
+
+const getQueueUrl = async (
+  sqsClient: SQSClient,
+  prefix: string,
+  priority: StatementProcessingPriority,
+  isQueuePriorityEnabled: boolean,
+) => {
+  if (queueUrl) {
+    return queueUrl;
+  }
+
+  const prefixWithPriority = getPrefixWithProcessingPriority(
+    prefix,
+    priority,
+    isQueuePriorityEnabled,
+  );
+
+  const getQueueUrlCommand = new GetQueueUrlCommand({
+    QueueName: `${prefixWithPriority}_${STATEMENT_QUEUE}`,
+  });
+
+  const commandResult = await sqsClient.send(getQueueUrlCommand);
+
+  queueUrl = commandResult.QueueUrl;
+
+  return queueUrl;
+};
+
+export default (config: FacadeConfig): Signature => {
+  return async ({ statementProperties, priority }) => {
+    const sqsClient = await config.client();
+
+    await getQueueUrl(sqsClient, config.prefix, priority, config.isQueuePriorityEnabled);
+    await publishMessages(sqsClient, statementProperties);
+  };
+};

src/apps/statements/repo/eventsRepo/factory.ts

@@ -1,11 +1,14 @@
 import Facade from './Facade';
 import FactoryConfig from './FactoryConfig';
 import redisFactory from './utils/redisEvents/factory';
+import sqsFactory from './utils/sqsEvents/factory';
 
 export default (config: FactoryConfig): Facade => {
   switch (config.facade) {
     default:
     case 'redis':
       return redisFactory(config.redis);
+    case 'sqs':
+      return sqsFactory(config.sqs);
   }
 };

src/apps/statements/repo/eventsRepo/utils/constants.ts

@@ -1,2 +1,3 @@
 export const EVENT_NAME = 'statement.new';
 export const CHANNEL_NAME = 'statement.notify';
+export const STATEMENT_QUEUE = 'STATEMENT_QUEUE';