Alternative Collection Concept Id in the signed s3 urls

services/cron-service/test/resources/test-env-defaults

@@ -164,6 +164,12 @@ PROVIDER_CACHE_SIZE=5000000
 # Provider Cache TTL in milliseconds, 1 hour
 PROVIDER_CACHE_TTL=3600000
 
+# Maximum size (in bytes) of the collection cache, 5MB
+COLLECTION_CACHE_SIZE=5000000
+
+# Collection Cache TTL in milliseconds, 1 hour
+COLLECTION_CACHE_TTL=3600000
+
 # Maximum size (in bytes) of the job status cache, 50MB
 JOB_STATUS_CACHE_SIZE=50000000
 

services/harmony/app/frontends/service-results.ts

@@ -76,6 +76,32 @@ export const providerIdCache = new LRUCache({
   fetchMethod: fetchProviderId,
 });
 
+/**
+ * Wrapper function of getCollectionIdForJobId to be set to fetchMethod of LRUCache.
+ *
+ * @param jobId - the job identifier
+ * @param _sv - stale value parameter of LRUCache fetchMethod, unused here
+ * @param options - options parameter of LRUCache fetchMethod, carries the request context
+ * @returns resolves to the collection ids for the job
+ */
+async function fetchCollectionId(
+  jobId: string,
+  _sv: string,
+  { context },
+): Promise<string> {
+  context.logger.info(`Fetching collection id for job id ${jobId}`);
+  return Job.getCollectionIdForJobId(db, jobId);
+}
+
+// In memory cache for Job ID to Collection Id
+export const collectionIdCache = new LRUCache({
+  ttl: env.collectionCacheTtl,
+  maxSize: env.collectionCacheSize,
+  sizeCalculation: (value: string): number => value.length,
+  fetchMethod: fetchCollectionId,
+});
+
+
 /**
  * Express.js handler that returns redirects to pre-signed URLs
  *
@@ -97,6 +123,7 @@ export async function getServiceResult(
   const url = `s3://${bucket}/${key}`;
 
   const provider = jobId ? await providerIdCache.fetch(jobId, { context: req.context }) : undefined;
+  const collectionIds = jobId ? await collectionIdCache.fetch(jobId, { context: req.context }) : undefined;
 
   const objectStore = objectStoreForProtocol('s3');
   if (objectStore) {
@@ -108,6 +135,9 @@ export async function getServiceResult(
       if (provider) {
         customParams['A-provider'] = provider.toUpperCase();
       }
+      if (collectionIds) {
+        customParams['A-collection-concept-ids'] = collectionIds.toUpperCase();
+      }
       req.context.logger.info(`Signing ${url} with params ${JSON.stringify(customParams)}`);
       const result = await objectStore.signGetObject(url, customParams);
       // Direct clients to reuse the redirect for 10 minutes before asking for a new one

services/harmony/app/models/job.ts

@@ -701,6 +701,25 @@ export class Job extends DBRecord implements JobRecord {
     return results[0]?.provider_id;
   }
 
+  /**
+   * Returns the Collection Id for the given jobID
+   *
+   * @param tx - the database transaction to use for querying
+   * @param jobID - the jobID for the job that should be retrieved
+   * @returns the Collection id for the job
+   */
+  static async getCollectionIdForJobId(
+    tx: Transaction,
+    jobID: string,
+  ): Promise<string> {
+    const results = await tx(Job.table).select('collectionIds').where({ jobID });
+    const collection_ids =  typeof results[0]?.collectionIds === 'string'
+      ? JSON.parse(results[0]?.collectionIds).join(',')
+      : undefined;
+    return collection_ids;
+  }
+
+
   /**
    * Returns the job matching the given username and job ID, or null if
    * no such job exists.

services/harmony/app/util/env.ts

@@ -129,6 +129,14 @@ class HarmonyServerEnv extends HarmonyEnv {
   @Min(1)
   providerCacheTtl: number;
 
+  @IsInt()
+  @Min(1)
+  collectionCacheSize: number;
+
+  @IsInt()
+  @Min(1)
+  collectionCacheTtl: number;
+
   @IsInt()
   @Min(1)
   edlCacheSize: number;

services/harmony/env-defaults

@@ -164,6 +164,12 @@ PROVIDER_CACHE_SIZE=5000000
 # Provider Cache TTL in milliseconds, 1 hour
 PROVIDER_CACHE_TTL=3600000
 
+# Maximum size (in bytes) of the collection cache, 5MB
+COLLECTION_CACHE_SIZE=5000000
+
+# Collection Cache TTL in milliseconds, 1 hour
+COLLECTION_CACHE_TTL=3600000
+
 # Maximum size (in bytes) of the job status cache, 50MB
 JOB_STATUS_CACHE_SIZE=50000000
 

services/harmony/test/service-results.ts

@@ -2,7 +2,7 @@ import { expect } from 'chai';
 import { describe, it } from 'mocha';
 import sinon, { stub } from 'sinon';
 
-import { createPublicPermalink, providerIdCache } from '../app/frontends/service-results';
+import { createPublicPermalink, providerIdCache, collectionIdCache } from '../app/frontends/service-results';
 import { FileStore } from '../app/util/object-store/file-store';
 import { hookUrl } from './helpers/hooks';
 import hookServersStartStop from './helpers/servers';
@@ -65,19 +65,23 @@ describe('service-results', function () {
 
   describe('getServiceResult', function () {
     describe('when given a URL containing an output created by harmony with a job ID and work item ID in the URL', function () {
-      let providerIdCacheStub;
+      let providerIdCacheStub, collectionIdCacheStub;
 
       before(function () {
         providerIdCacheStub = sinon.stub(providerIdCache, 'fetch').resolves('eedtest');
+        collectionIdCacheStub = sinon.stub(collectionIdCache, 'fetch').resolves('collection-1,collection-2');
       });
 
       after(function () {
         providerIdCacheStub.restore();
+        collectionIdCacheStub.restore();
       });
 
       hookUrl('/service-results/some-bucket/public/some-job-id/some-work-item-id/some-path.tif', 'jdoe');
       it('passes the user\'s Earthdata Login username to the signing function for tracking', function () {
         expect(this.res.headers.location).to.include('A-userid=jdoe');
+        expect(this.res.headers.location).to.include('A-provider=EEDTEST');
+        expect(this.res.headers.location).to.include('A-collection-concept-ids=COLLECTION-1%2CCOLLECTION-2');
       });
 
       it('redirects temporarily to a presigned URL', function () {
@@ -98,6 +102,27 @@ describe('service-results', function () {
       });
     });
 
+    describe('when the job has no collection IDs', function () {
+      let providerIdCacheStub, collectionIdCacheStub;
+
+      before(function () {
+        providerIdCacheStub = sinon.stub(providerIdCache, 'fetch').resolves('eedtest');
+        collectionIdCacheStub = sinon.stub(collectionIdCache, 'fetch').resolves(undefined);
+      });
+
+      after(function () {
+        providerIdCacheStub.restore();
+        collectionIdCacheStub.restore();
+      });
+
+      hookUrl('/service-results/some-bucket/public/some-job-id/some-work-item-id/some-path.tif', 'jdoe');
+
+      it('does not include the A-collection-concept-ids header', function () {
+        expect(this.res.headers.location).to.not.include('A-collection-concept-ids');
+        expect(this.res.headers.location).to.include('A-provider=EEDTEST');
+      });
+    });
+
     describe('when given a valid bucket and key that cannot be signed', function () {
       let stubObject;
       before(function () {