Merge pull request #3495 from aws/release-v1.83.0

Release 1.83.0 (to main)

Author: xazhao

.github/workflows/build.yml

@@ -20,7 +20,6 @@ jobs:
         os:
           - ubuntu-latest
         python:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"

DEVELOPMENT_GUIDE.md

@@ -26,7 +26,7 @@ Environment setup
 -----------------
 ### 1. Install Python versions
 
-Our officially supported Python versions are 3.7, 3.8, 3.9 and 3.10. 
+Our officially supported Python versions are 3.8, 3.9 and 3.10. 
 Our CI/CD pipeline is setup to run unit tests against Python 3 versions. Make sure you test it before sending a Pull Request.
 See [Unit testing with multiple Python versions](#unit-testing-with-multiple-python-versions).
 
@@ -40,12 +40,11 @@ easily setup multiple Python versions. For
 1.  Install PyEnv -
     `curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash`
 1. Restart shell so the path changes take effect - `exec $SHELL`
-1.  `pyenv install 3.7.16`
 1.  `pyenv install 3.8.16`
 1.  `pyenv install 3.9.16`
 1.  `pyenv install 3.10.9`
 3.  Make Python versions available in the project:
-    `pyenv local 3.7.16 3.8.16 3.9.16 3.10.9`
+    `pyenv local 3.8.16 3.9.16 3.10.9`
 
 Note: also make sure the following lines were written into your `.bashrc` (or `.zshrc`, depending on which shell you are using):
 ```
@@ -66,7 +65,7 @@ can be found [here](https://black.readthedocs.io/en/stable/editor_integration.ht
 Since black is installed in virtualenv, when you follow [this instruction](https://black.readthedocs.io/en/stable/editor_integration.html), `which black` might give you this
 
 ```bash
-(sam37) $ where black
+(sam38) $ where black
 /Users/<username>/.pyenv/shims/black
 ```
 
@@ -77,11 +76,11 @@ and this will happen:
 pyenv: black: command not found
 
 The `black' command exists in these Python versions:
-  3.7.9/envs/sam37
-  sam37
+  3.8.16/envs/sam38
+  sam38
 ``` 
 
-A simple workaround is to use `/Users/<username>/.pyenv/versions/sam37/bin/black` 
+A simple workaround is to use `/Users/<username>/.pyenv/versions/sam38/bin/black` 
 instead of `/Users/<username>/.pyenv/shims/black`.
 
 #### Pre-commit
@@ -99,15 +98,15 @@ handy plugin that can create virtualenv.
 Depending on the python version, the following commands would change to
 be the appropriate python version.
 
-1.  Create Virtualenv `sam37` for Python3.7: `pyenv virtualenv 3.7.9 sam37`
-1.  Activate Virtualenv: `pyenv activate sam37`
+1.  Create Virtualenv `sam38` for Python3.8: `pyenv virtualenv 3.8.16 sam38`
+1.  Activate Virtualenv: `pyenv activate sam38`
 
 ### 4. Install dev version of SAM transform
 
 We will install a development version of SAM transform from source into the
 virtualenv.
 
-1.  Activate Virtualenv: `pyenv activate sam37`
+1.  Activate Virtualenv: `pyenv activate sam38`
 1.  Install dev version of SAM transform: `make init`
 
 Running tests
@@ -121,8 +120,8 @@ Run `make test` or `make test-fast`. Once all tests pass make sure to run
 
 ### Unit testing with multiple Python versions
 
-Currently, our officially supported Python versions are 3.7, 3.8, 3.9 and 3.10. For the most
-part, code that works in Python3.7 will work in Pythons 3.8, 3.9 and 3.10. You only run into problems if you are
+Currently, our officially supported Python versions are 3.8, 3.9 and 3.10. For the most
+part, code that works in Python3.8 will work in Pythons 3.9 and 3.10. You only run into problems if you are
 trying to use features released in a higher version (for example features introduced into Python3.10
 will not work in Python3.9). If you want to test in many versions, you can create a virtualenv for
 each version and flip between them (sourcing the activate script). Typically, we run all tests in

HOWTO.md

@@ -21,7 +21,7 @@ Check out the [latest specification](versions/2016-10-31.md) for details on how
 You could also use the [aws-sam-cli](https://github.com/awslabs/aws-sam-cli) to get started 
 
 ```shell
-$ sam init --runtime python3.7
+$ sam init --runtime python3.8
 ```
 ## Packing Artifacts
 Before you can deploy a SAM template, you should first upload your Lambda 

Makefile

@@ -65,7 +65,8 @@ prepare-companion-stack:
 fetch-schema-data:
 	mkdir -p .tmp
 
-	curl -o .tmp/cfn-docs.json https://raw.githubusercontent.com/aws/aws-cdk/main/packages/%40aws-cdk/cfnspec/spec-source/cfn-docs/cfn-docs.json
+	# aws-cdk updated where they store the cfn doc json files. See https://github.com/aws/aws-cdk/blob/main/packages/%40aws-cdk/cfnspec/README.md
+	bin/git_lfs_download.sh "https://raw.githubusercontent.com/cdklabs/awscdk-service-spec/main/sources/CloudFormationDocumentation/CloudFormationDocumentation.json"
 
 	curl -o .tmp/cloudformation.schema.json https://raw.githubusercontent.com/awslabs/goformation/master/schema/cloudformation.schema.json
 

README.md

@@ -85,7 +85,7 @@ For a more thorough introduction, see the [this tutorial](https://docs.aws.amazo
 
 ### Setting up development environment
 
-You'll need to have Python 3.7+ installed.
+You'll need to have Python 3.8+ installed.
 
 Create a [virtual environment](https://docs.python.org/3/library/venv.html):
 

bin/git_lfs_download.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -eux
+
+# Here is the reference I found on how to download Git LFS file 
+# https://gist.github.com/fkraeutli/66fa741d9a8c2a6a238a01d17ed0edc5#retrieving-lfs-files
+
+# Check if a URL parameter is provided
+if [ $# -eq 0 ]; then
+    echo "Script Usage: $0 <URL>"
+    exit 1
+fi
+
+# Get the URL from the first command-line parameter
+url="$1"
+
+# Fetch the metadata from the URL
+response=$(curl -s "$url")
+
+# Extract oid and size from the metadata
+oid=$(echo "$response" | grep '^oid' | cut -d: -f2)
+size=$(echo "$response" | grep 'size' | cut -d ' ' -f 2)
+
+# String interpolation to create the request JSON content
+request_json=$(jq -nc --arg oid "$oid" --argjson size "$size" '{"operation":"download","objects":[{"oid":$oid,"size":$size}],"transfers":["basic"]}')
+
+# Send a POST request to Git LFS with the retrieved metadata JSON content
+response=$(curl \
+  -X POST \
+  -H "Accept: application/vnd.git-lfs+json" \
+  -H "Content-type: application/json" \
+  -d "$request_json" \
+  https://github.com/cdklabs/awscdk-service-spec.git/info/lfs/objects/batch)
+
+# The above command should return a JSON object that tells you where the file is stored
+href=$(echo "$response" | jq -r '.objects[0].actions.download.href')
+
+# Download the file and store it in .tmp/cfn-docs.json
+curl -o .tmp/cfn-docs.json $href

bin/run_cfn_lint.sh

@@ -10,4 +10,6 @@ if [ ! -d "${VENV}" ]; then
 fi
 
 "${VENV}/bin/python" -m pip install cfn-lint==0.75.0 --upgrade --quiet
+# update cfn schema
+"${VENV}/bin/cfn-lint" -u
 "${VENV}/bin/cfn-lint" --format parseable

bin/transform-test-error-json-format.py

@@ -8,13 +8,11 @@
 import sys
 from pathlib import Path
 
-from typing_extensions import Final
-
 # To allow this script to be executed from other directories
 sys.path.insert(0, str(Path(__file__).absolute().parent.parent))
 
 import json
-from typing import Type
+from typing import Final, Type
 
 from bin._file_formatter import FileFormatter
 

integration/combination/test_function_with_cwe_dlq_generated.py

@@ -36,7 +36,7 @@ def test_function_with_cwe(self):
 
         # checking policy action
         actions = dlq_policy_statement["Action"]
-        action_list = actions if type(actions) == list else [actions]
+        action_list = actions if isinstance(actions, list) == list else [actions]
         self.assertEqual(len(action_list), 1, "Only one action must be in dead-letter queue policy")
         self.assertEqual(
             action_list[0], "sqs:SendMessage", "Action referenced in dead-letter queue policy must be 'sqs:SendMessage'"

integration/combination/test_function_with_msk.py

@@ -7,6 +7,10 @@
 from integration.helpers.resource import current_region_does_not_support, generate_suffix
 
 
+# Mark this test suite as nonblocking tests since MSK Cluster creation can take
+# up to 30 minutes according to https://docs.aws.amazon.com/msk/latest/developerguide/troubleshooting.html#troubleshooting-cluster-stuck
+# This would cause the test to fail due to MSK Cluster did not stablize.
+# We should investigate any other cause of failures.
 @skipIf(current_region_does_not_support([MSK]), "MSK is not supported in this testing region")
 @nonblocking
 class TestFunctionWithMsk(BaseTest):