Fix: Race between layer and Lambda update (#5927, PR #7436)

Author: achave11-ucsc

.mypy.ini

@@ -68,6 +68,7 @@ modules =
     scripts.post_deploy_tdr,
     scripts.find_in_snapshots,
     scripts.can_bundle,
+    scripts.delete_older_function_versions,
     scripts.zenhub_to_github,
     azul.plugins.metadata.anvil.bundle,
     azul.plugins.metadata.anvil.schema,

scripts/delete_older_function_versions.py

@@ -0,0 +1,48 @@
+"""
+Delete all versions of a Lambda function prior to the specified one.
+"""
+import argparse
+import logging
+import sys
+
+from azul import (
+    R,
+    config,
+)
+from azul.args import (
+    AzulArgumentHelpFormatter,
+)
+from azul.lambdas import (
+    LambdaFunctions,
+)
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+
+def main(argv: list[str]):
+    assert config.terraform_component == '', R(
+        'This script cannot be run with a Terraform component selected',
+        config.terraform_component)
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=AzulArgumentHelpFormatter)
+    parser.add_argument('--function-name', '-f',
+                        required=True,
+                        help='The name of the Lambda function.')
+    parser.add_argument('--function-version', '-v',
+                        type=int,
+                        required=True,
+                        help='The Lambda function version to keep. Must be an '
+                             'integer.')
+    args = parser.parse_args(argv)
+    log.info('Deleting function %r versions older than %r',
+             args.function_name, args.function_version)
+    functions = LambdaFunctions()
+    functions.delete_older_versions(args.function_name, args.function_version)
+
+
+if __name__ == '__main__':
+    configure_script_logging(log)
+    main(sys.argv[1:])

scripts/generate_openapi_document.py

@@ -42,14 +42,14 @@ def main():
                                sources=set())
     }
 
-    lambda_name = Path.cwd().name
-    assert lambda_name in config.lambda_names(), lambda_name
+    app_name = Path.cwd().name
+    assert app_name in config.app_names(), app_name
 
     # To create a normalized OpenAPI document, we patch any
     # deployment-specific variables that affect the document.
     with (
         patch_config('catalogs', catalogs),
-        patch_config(f'{lambda_name}_function_name', f'azul-{lambda_name}-dev'),
+        patch_config(f'{app_name}_function_name', f'azul-{app_name}-dev'),
         patch_config('enable_log_forwarding', False),
         patch_config('enable_replicas', True),
         patch_config('monitoring_email', '[email protected]')
@@ -58,10 +58,10 @@ def main():
         with patch.object(target=AzulChaliceApp,
                           attribute='base_url',
                           new=lambda_endpoint):
-            app_module = load_app_module(lambda_name)
+            app_module = load_app_module(app_name)
             assert app_module.app.base_url == lambda_endpoint
             app_spec = app_module.app.spec()
-            doc_path = Path(config.project_root) / 'lambdas' / lambda_name / 'openapi.json'
+            doc_path = Path(config.project_root) / 'lambdas' / app_name / 'openapi.json'
             with write_file_atomically(doc_path) as file:
                 json.dump(app_spec, file, indent=4)
 

scripts/manage_lambdas.py

@@ -2,7 +2,7 @@
 import logging
 
 from azul.lambdas import (
-    Lambdas,
+    LambdaFunctions,
 )
 from azul.logging import (
     configure_script_logging,
@@ -18,4 +18,5 @@
     group.add_argument('--disable', dest='enabled', action='store_false')
     args = parser.parse_args()
     assert args.enabled is not None
-    Lambdas().manage_lambdas(args.enabled)
+    functions = LambdaFunctions()
+    functions.manage_lambdas(args.enabled)

scripts/reset_lambda_role.py

@@ -1,10 +1,16 @@
+"""
+Attempt to fix KMSAccessDeniedException when invoking a function.
+
+See Troubleshooting section in README.md for details.
+"""
 from azul.lambdas import (
-    Lambdas,
+    LambdaFunctions,
 )
 
 
 def main():
-    Lambdas().reset_lambda_roles()
+    functions = LambdaFunctions()
+    functions.reset_lambda_roles()
 
 
 if __name__ == '__main__':

scripts/sell_unused_slots.py

@@ -27,8 +27,8 @@
     aws,
 )
 from azul.lambdas import (
-    Lambda,
-    Lambdas,
+    LambdaFunction,
+    LambdaFunctions,
 )
 from azul.logging import (
     configure_script_logging,
@@ -65,17 +65,18 @@ def is_reindex_active(self) -> bool:
 
     @classmethod
     @cache
-    def _list_contribution_lambda_functions(cls) -> list[Lambda]:
+    def _list_contribution_lambda_functions(cls) -> list[LambdaFunction]:
         """
         Search Lambda functions for the names of contribution Lambdas.
         """
+        functions = LambdaFunctions()
         return [
-            lambda_
-            for lambda_ in Lambdas().list_lambdas()
-            if lambda_.is_contribution_lambda
+            function
+            for function in functions.list_functions()
+            if function.contributes
         ]
 
-    def _lambda_invocation_counts(self) -> dict[Lambda, int]:
+    def _lambda_invocation_counts(self) -> dict[LambdaFunction, int]:
         # FIXME: DeprecationWarning for datetime methods in Python 3.12
         #        https://github.com/DataBiosphere/azul/issues/5953
         end = datetime.utcnow()
@@ -95,6 +96,9 @@ def _lambda_invocation_counts(self) -> dict[Lambda, int]:
                                 'Namespace': 'AWS/Lambda',
                                 'MetricName': 'Invocations',
                                 'Dimensions': [{
+                                    # The 'FunctionName' dimension returns
+                                    # aggregate metrics for all versions and
+                                    # aliases of the function.
                                     'Name': 'FunctionName',
                                     'Value': lambda_.name
                                 }]

src/azul/__init__.py

@@ -743,7 +743,7 @@ def drs_endpoint(self) -> mutable_furl:
         else:
             return self.service_endpoint
 
-    def lambda_names(self) -> list[str]:
+    def app_names(self) -> list[str]:
         return ['indexer', 'service']
 
     @property
@@ -760,13 +760,15 @@ def indexer_function_name(self, handler_name: str | None = None):
     def service_function_name(self, handler_name: str | None = None):
         return self._function_name('service', handler_name)
 
-    def _function_name(self, lambda_name: str, handler_name: str | None):
+    def _function_name(self, app_name: str, handler_name: str | None):
         if handler_name is None:
-            return self.qualified_resource_name(lambda_name)
+            return self.qualified_resource_name(app_name)
         else:
             # FIXME: Eliminate hardcoded separator
             #        https://github.com/databiosphere/azul/issues/2964
-            return self.qualified_resource_name(lambda_name, suffix='-' + handler_name)
+            return self.qualified_resource_name(app_name, suffix='-' + handler_name)
+
+    active_function_alias_name = 'active'
 
     qualifier_re = re.compile(r'[a-z][a-z0-9]{1,16}')
 

src/azul/chalice.py

@@ -26,14 +26,14 @@
 )
 
 import attrs
-import chalice
 from chalice import (
     Chalice,
     ChaliceViewError,
 )
 from chalice.app import (
     BadRequestError,
     CaseInsensitiveMapping,
+    EventSourceHandler,
     HeadersType,
     MultiDict,
     NotFoundError,
@@ -598,7 +598,7 @@ class metric_alarm(HandlerDecorator):
         period: int
 
         def __call__(self, f):
-            assert isinstance(f, chalice.app.EventSourceHandler), f
+            assert isinstance(f, EventSourceHandler), f
             try:
                 metric_alarms = getattr(f, 'metric_alarms')
             except AttributeError:
@@ -611,6 +611,14 @@ def __call__(self, f):
         def tf_resource_name(self) -> str:
             return f'{self.tf_function_resource_name}_{self.metric.name}'
 
+    @property
+    def event_source_handlers(self) -> dict[str, EventSourceHandler]:
+        return {
+            handler_name: handler
+            for handler_name, handler in self.handler_map.items()
+            if isinstance(handler, EventSourceHandler)
+        }
+
     @property
     def metric_alarms(self) -> Iterator[metric_alarm]:
         for metric in LambdaMetric:
@@ -622,19 +630,16 @@ def metric_alarms(self) -> Iterator[metric_alarm]:
                                       threshold=0,
                                       period=60 * 60 if for_errors else 5 * 60)
             yield alarm.bind(self)
-        for handler_name, handler in self.handler_map.items():
-            if isinstance(handler, chalice.app.EventSourceHandler):
-                try:
-                    metric_alarms = getattr(handler, 'metric_alarms')
-                except AttributeError:
-                    metric_alarms = (
-                        self.metric_alarm(metric=metric,
-                                          threshold=0,
-                                          period=5 * 60)
-                        for metric in LambdaMetric
-                    )
-                for metric_alarm in metric_alarms:
-                    yield metric_alarm.bind(self, handler_name)
+        for handler_name, handler in self.event_source_handlers.items():
+            try:
+                metric_alarms = getattr(handler, 'metric_alarms')
+            except AttributeError:
+                metric_alarms = (
+                    self.metric_alarm(metric=metric, threshold=0, period=5 * 60)
+                    for metric in LambdaMetric
+                )
+            for metric_alarm in metric_alarms:
+                yield metric_alarm.bind(self, handler_name)
 
     # noinspection PyPep8Naming
     @attrs.frozen
@@ -650,20 +655,25 @@ class retry(HandlerDecorator):
         num_retries: int
 
         def __call__(self, f):
-            assert isinstance(f, chalice.app.EventSourceHandler), f
+            assert isinstance(f, EventSourceHandler), f
             setattr(f, 'retry', self)
             return f
 
     @property
     def retries(self) -> Iterator[retry]:
-        for handler_name, handler in self.handler_map.items():
-            if isinstance(handler, chalice.app.EventSourceHandler):
-                try:
-                    retry = getattr(handler, 'retry')
-                except AttributeError:
-                    pass
-                else:
-                    yield retry.bind(self, handler_name)
+        for handler_name, handler in self.event_source_handlers.items():
+            try:
+                retry = getattr(handler, 'retry')
+            except AttributeError:
+                pass
+            else:
+                yield retry.bind(self, handler_name)
+
+    @property
+    def tf_function_resource_names(self) -> Iterator[str]:
+        yield self.unqualified_app_name
+        for handler_name in self.event_source_handlers:
+            yield f'{self.unqualified_app_name}_{handler_name}'
 
     def default_routes(self):
 

src/azul/health.py

@@ -93,7 +93,7 @@ def description(self):
 
 @attr.s(frozen=True, kw_only=True, auto_attribs=True)
 class HealthController(AppController):
-    lambda_name: str
+    app_name: str
 
     @cached_property
     def storage_service(self):
@@ -134,7 +134,7 @@ def cached_health(self) -> JSON:
                                 self.app.catalog, config.default_catalog)
         else:
             try:
-                cache = json.loads(self.storage_service.get(f'health/{self.lambda_name}'))
+                cache = json.loads(self.storage_service.get(f'health/{self.app_name}'))
             except StorageObjectNotFound:
                 raise NotFoundError('Cached health object does not exist')
             else:
@@ -148,7 +148,7 @@ def cached_health(self) -> JSON:
     def update_cache(self) -> None:
         assert self.app.catalog == config.default_catalog
         health_object = dict(time=time.time(), health=self._health.as_json_fast())
-        self.storage_service.put(object_key=f'health/{self.lambda_name}',
+        self.storage_service.put(object_key=f'health/{self.app_name}',
                                  data=json.dumps(health_object).encode())
 
     @property
@@ -182,7 +182,7 @@ class does not examine any resources, only accessing the individual
 
     @property
     def lambda_name(self):
-        return self.controller.lambda_name
+        return self.controller.app_name
 
     def as_json(self, keys: Iterable[str]) -> JSON:
         keys = frozenset(keys)
@@ -200,9 +200,9 @@ def other_lambdas(self) -> JSON:
         Indicates whether the companion REST API responds to HTTP requests.
         """
         response = {
-            lambda_name: self._lambda(lambda_name)
-            for lambda_name in config.lambda_names()
-            if lambda_name != self.lambda_name
+            app_name: self._lambda(app_name)
+            for app_name in config.app_names()
+            if app_name != self.lambda_name
         }
         return {
             'up': all(json_bool(v['up']) for v in response.values()),
@@ -333,7 +333,7 @@ class HealthApp(AzulChaliceApp):
 
     @cached_property
     def health_controller(self) -> HealthController:
-        return HealthController(app=self, lambda_name=self.unqualified_app_name)
+        return HealthController(app=self, app_name=self.unqualified_app_name)
 
     def default_routes(self):
         _routes = super().default_routes()