Dependency updates

.github/workflows/dependencies.yml

@@ -14,13 +14,13 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v6.0.2
         with:
           token: ${{ secrets.ACCESS_TOKEN }}
           ref: development
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6.2.0
         with:
           python-version: '3.12'
           cache: pip
@@ -55,7 +55,7 @@ jobs:
 
       - name: Create Pull Request
         if: always()
-        uses: peter-evans/create-pull-request@v7.0.7
+        uses: peter-evans/create-pull-request@v8.1.0
         with:
           token: ${{ secrets.ACCESS_TOKEN }}
           branch: dependency-updates

.github/workflows/deploy.yml

@@ -15,24 +15,24 @@ jobs:
       name: ${{ github.ref_name }}
 
     env:
-      APP_NAME: digital_ingest_assembly
+      APP_NAME: data_fetch
 
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6.0.2
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4.0.2
+        uses: aws-actions/configure-aws-credentials@v6.0.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
           role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE }}
           role-skip-session-tagging: true
           role-duration-seconds: 900
           aws-region: ${{ secrets.AWS_REGION }}
 
       - name: Login to Amazon ECR
-        uses: aws-actions/amazon-ecr-login@v2.0.1
+        uses: aws-actions/amazon-ecr-login@v2.1.2
         with:
           registries: ${{ secrets.ECR_REGISTRIES }}
 

.github/workflows/tests.yml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6.0.2
 
       - name: Set up Python and cache pip
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6.2.0
         with:
           python-version: '3.12'
           cache: 'pip'

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
-- repo: https://github.com/pre-commit/mirrors-autopep8
-  rev: v2.0.4
+- repo: https://github.com/hhatto/autopep8
+  rev: v2.3.2
   hooks:
   - id: autopep8
     args:
@@ -11,11 +11,11 @@ repos:
   hooks:
   - id: flake8
 - repo: https://github.com/pycqa/isort
-  rev: 7.0.0
+  rev: 9.0.0a3
   hooks:
   - id: isort
 - repo: https://github.com/jazzband/pip-tools
-  rev: v7.5.2
+  rev: v7.5.3
   hooks:
     - id: pip-compile
       args:

README.md

@@ -16,10 +16,22 @@ If you have [git](https://git-scm.com/) and [Docker](https://store.docker.com/se
 ## Service Flow
 
 The service processes packages as follows:
-
+- Marks the service instance as running
+- Instantiates the necessary client for the desired data source
+- If fetching updated objects, the service:
+    - Fetches identifiers for all objects that been updated
+    - Fetches data for updated objects
+    - The data is sent to an SNS topic with an indication of whether the data should be indexed or removed from the index
+- If fetching deleted objects, the service:
+    - Fetches identifiers for all objects that been deleted
+    - Sends the identifier to an SNS topic, indicating that it should be deleted from the index
+- Sends a success message to an SNS topic
+- Updates the last run time of the service
+- Marks the service instance as not running
 
 If errors are encountered during any of the above steps, the service:
-
+- Sends a failure message to an SNS topic
+- Marks service instance as not running
 
 
 ## Usage

requirements.txt

@@ -1,44 +1,44 @@
 archivessnake==0.10.1
     # via -r requirements.in
-attrs==25.4.0
+attrs==26.1.0
     # via archivessnake
 aws-assume-role-lib==2.10.0
     # via -r requirements.in
 boltons==25.0.0
     # via archivessnake
-boto3==1.42.42
+boto3==1.43.1
     # via
     #   -r requirements.in
     #   aws-assume-role-lib
-botocore==1.42.42
+botocore==1.43.1
     # via
     #   boto3
     #   s3transfer
-certifi==2026.1.4
+certifi==2026.4.22
     # via requests
-charset-normalizer==3.4.4
+charset-normalizer==3.4.7
     # via requests
 electronbonder==1.1
     # via -r requirements.in
-idna==3.11
+idna==3.13
     # via requests
 jmespath==1.1.0
     # via
     #   boto3
     #   botocore
-more-itertools==10.8.0
+more-itertools==11.0.2
     # via archivessnake
 python-dateutil==2.9.0.post0
     # via botocore
 pyyaml==6.0.3
     # via archivessnake
-rapidfuzz==3.14.3
+rapidfuzz==3.14.5
     # via archivessnake
-requests==2.32.5
+requests==2.33.1
     # via
     #   archivessnake
     #   electronbonder
-s3transfer==0.16.0
+s3transfer==0.17.0
     # via boto3
 shortuuid==1.0.13
     # via -r requirements.in

src/fetch_data.py

@@ -51,7 +51,9 @@ def __init__(self, environment,
 
     def fetch(self):
         """Main method, which calls all other methods."""
+        logging.info(f"Checking to see if fetch for {self.object_status} {self.object_type} is already running.")
         if not self.is_running(self.object_status, self.object_type):
+            logging.info(f"Fetching {self.object_status} {self.object_type} from {self.source_system}.")
             try:
                 start_time = int(time.time())
                 self.set_is_running(self.object_status, self.object_type)
@@ -80,10 +82,11 @@ def fetch(self):
                     for to_delete in fetched_ids:
                         self.send_delete_request(to_delete)
                 self.send_success_message()
+                self.set_last_run_time(self.object_status, self.object_type, start_time)
             except Exception as e:
+                logging.error(e)
                 self.send_failure_message(e)
             self.set_is_running(self.object_status, self.object_type, status=False)
-            self.set_last_run_time(self.object_status, self.object_type, start_time)
 
     def get_client_with_role(self, resource, role_arn):
         """Gets Boto3 client which authenticates with a specific IAM role."""
@@ -202,13 +205,18 @@ def send_data_to_sns(self, data):
                 'requested_action': {
                     'DataType': 'String',
                     'StringValue': 'merge',
+                },
+                'object_type': {
+                    'DataType': 'String',
+                    'StringValue': self.object_type,
                 }
             })
 
     def send_delete_request(self, data):
         """Sends delete request to SNS topic."""
         client = self.get_client_with_role('sns', self.sns_role_arn)
         es_id = get_es_id(data)
+
         client.publish(
             TopicArn=self.sns_topic,
             MessageGroupId=f'{self.service_name}-{es_id}',
@@ -222,6 +230,14 @@ def send_delete_request(self, data):
                 'requested_action': {
                     'DataType': 'String',
                     'StringValue': 'delete',
+                },
+                'es_id': {
+                    'DataType': 'String',
+                    'StringValue': es_id,
+                },
+                'object_type': {
+                    'DataType': 'String',
+                    'StringValue': self.object_type,
                 }
             })
 

tests/test_fetch.py

@@ -364,7 +364,7 @@ def test_fetch_with_exception(
         mock_failure_message.assert_called_once()
         mock_set_is_running.assert_has_calls(
             [call(self.fetcher.object_status, self.fetcher.object_type), call(self.fetcher.object_status, self.fetcher.object_type, status=False)])
-        mock_set_last_run_time.assert_called_once_with(self.fetcher.object_status, self.fetcher.object_type, ANY)
+        mock_set_last_run_time.assert_not_called()
 
     @mock_aws
     def test_get_last_run_time(self):
@@ -440,14 +440,19 @@ def test_send_data_to_sns(self):
         self.assertEqual(message_body['Message'], json.dumps({"uri": "12345"}))
         self.assertEqual(
             message_body['MessageAttributes'],
-            {'service': {
-                'Type': 'String',
-                'Value': self.fetcher.service_name,
-            },
+            {
+                'service': {
+                    'Type': 'String',
+                    'Value': self.fetcher.service_name,
+                },
                 'requested_action': {
-                'Type': 'String',
-                'Value': 'merge',
-            }})
+                    'Type': 'String',
+                    'Value': 'merge',
+                },
+                'object_type': {
+                    'Type': 'String',
+                    'Value': self.fetcher.object_type,
+                }})
 
     @mock_aws
     def test_send_delete_request(self):
@@ -458,14 +463,23 @@ def test_send_delete_request(self):
         self.assertEqual(message_body['Message'], '3aai9usY3AZzCSFkB3RSQ9')
         self.assertEqual(
             message_body['MessageAttributes'],
-            {'service': {
-                'Type': 'String',
-                'Value': self.fetcher.service_name,
-            },
+            {
+                'service': {
+                    'Type': 'String',
+                    'Value': self.fetcher.service_name,
+                },
                 'requested_action': {
-                'Type': 'String',
-                'Value': 'delete',
-            }})
+                    'Type': 'String',
+                    'Value': 'delete',
+                },
+                'es_id': {
+                    'Type': 'String',
+                    'Value': '3aai9usY3AZzCSFkB3RSQ9',
+                },
+                'object_type': {
+                    'Type': 'String',
+                    'Value': self.fetcher.object_type,
+                }})
 
     @mock_aws
     def test_send_success_message(self):

tests/test_requirements.txt

@@ -1,21 +1,21 @@
-boto3==1.42.42
+boto3==1.43.1
     # via moto
-botocore==1.42.42
+botocore==1.43.1
     # via
     #   boto3
     #   moto
     #   s3transfer
-certifi==2026.1.4
+certifi==2026.4.22
     # via requests
 cffi==2.0.0
     # via cryptography
-charset-normalizer==3.4.4
+charset-normalizer==3.4.7
     # via requests
-coverage==7.13.3
+coverage==7.13.5
     # via -r tests/test_requirements.in
-cryptography==46.0.4
+cryptography==47.0.0
     # via moto
-idna==3.11
+idna==3.13
     # via requests
 iniconfig==2.3.0
     # via pytest
@@ -29,31 +29,31 @@ markupsafe==3.0.3
     # via
     #   jinja2
     #   werkzeug
-moto==5.1.20
+moto==5.1.22
     # via -r tests/test_requirements.in
-packaging==26.0
+packaging==26.2
     # via pytest
 pluggy==1.6.0
     # via pytest
 pycparser==3.0
     # via cffi
-pygments==2.19.2
+pygments==2.20.0
     # via pytest
-pytest==9.0.2
+pytest==9.0.3
     # via -r tests/test_requirements.in
 python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   moto
 pyyaml==6.0.3
     # via responses
-requests==2.32.5
+requests==2.33.1
     # via
     #   moto
     #   responses
-responses==0.25.8
+responses==0.26.0
     # via moto
-s3transfer==0.16.0
+s3transfer==0.17.0
     # via boto3
 six==1.17.0
     # via python-dateutil
@@ -62,7 +62,7 @@ urllib3==2.6.3
     #   botocore
     #   requests
     #   responses
-werkzeug==3.1.5
+werkzeug==3.1.8
     # via moto
-xmltodict==1.0.2
+xmltodict==1.0.4
     # via moto

tox.ini

@@ -8,8 +8,8 @@ skip_install = True
 [testenv:py312]
 allowlist_externals = docker
 commands =
-  docker build -t digital_ingest_assembly_test --target test .
-  docker run digital_ingest_assembly_test sh -c "coverage run -m pytest -s && coverage report -m"
+  docker build -t data_fetch --target test .
+  docker run data_fetch sh -c "coverage run -m pytest -s && coverage report -m"
 
 [testenv:linting]
 allowlist_externals = pre-commit