Bump awscrt from 0.27.6 to 0.28.1

.changes/next-release/enhancement-awscrt-855.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``awscrt``",
+  "description": "Update awscrt version to 0.28.1"
+}

.changes/next-release/enhancement-crt-82800.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "awscrt",
+  "description": "Exposes new CRT options for S3 file IO"
+}

awscli/customizations/s3/factory.py

@@ -138,6 +138,15 @@ def _create_crt_client(self, params, runtime_config):
             create_crt_client_kwargs['crt_credentials_provider'] = (
                 crt_credentials_provider
             )
+        fio_options = {}
+        if (val := runtime_config.get('should_stream')) is not None:
+            fio_options['should_stream'] = val
+        if (val := runtime_config.get('disk_throughput')) is not None:
+            # Convert bytes to gigabits.
+            fio_options['disk_throughput_gbps'] = val * 8 / 1_000_000_000
+        if (val := runtime_config.get('direct_io')) is not None:
+            fio_options['direct_io'] = val
+        create_crt_client_kwargs['fio_options'] = fio_options
 
         return create_s3_crt_client(**create_crt_client_kwargs)
 

awscli/customizations/s3/transferconfig.py

@@ -15,6 +15,7 @@
 # commands.
 import logging
 
+from botocore.utils import ensure_boolean
 from s3transfer.manager import TransferConfig
 
 from awscli.customizations.s3 import constants
@@ -31,6 +32,9 @@
     'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
     'target_bandwidth': None,
     'io_chunksize': 256 * 1024,
+    'should_stream': None,
+    'disk_throughput': None,
+    'direct_io': None,
 }
 
 
@@ -47,9 +51,18 @@ class RuntimeConfig:
         'max_bandwidth',
         'target_bandwidth',
         'io_chunksize',
+        'disk_throughput',
+    ]
+    HUMAN_READABLE_SIZES = [
+        'multipart_chunksize',
+        'multipart_threshold',
+        'io_chunksize',
+    ]
+    HUMAN_READABLE_RATES = [
+        'max_bandwidth',
+        'target_bandwidth',
+        'disk_throughput',
     ]
-    HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold', 'io_chunksize']
-    HUMAN_READABLE_RATES = ['max_bandwidth', 'target_bandwidth']
     SUPPORTED_CHOICES = {
         'preferred_transfer_client': [
             constants.AUTO_RESOLVE_TRANSFER_CLIENT,
@@ -62,6 +75,7 @@ class RuntimeConfig:
             'default': constants.CLASSIC_TRANSFER_CLIENT
         }
     }
+    BOOLEANS = ['should_stream', 'direct_io']
 
     @staticmethod
     def defaults():
@@ -83,6 +97,7 @@ def build_config(self, **kwargs):
             runtime_config.update(kwargs)
         self._convert_human_readable_sizes(runtime_config)
         self._convert_human_readable_rates(runtime_config)
+        self._convert_booleans(runtime_config)
         self._resolve_choice_aliases(runtime_config)
         self._validate_config(runtime_config)
         return runtime_config
@@ -116,6 +131,12 @@ def _convert_human_readable_rates(self, runtime_config):
                         'second (e.g. 10Mb/s or 800Kb/s)' % value
                     )
 
+    def _convert_booleans(self, runtime_config):
+        for attr in self.BOOLEANS:
+            value = runtime_config.get(attr)
+            if value is not None:
+                runtime_config[attr] = ensure_boolean(value)
+
     def _human_readable_rate_to_int(self, value):
         # The human_readable_to_int() utility only supports integers (e.g. 1024)
         # as strings and human readable sizes (e.g. 10MB, 5GB). It does not

awscli/s3transfer/crt.py

@@ -33,6 +33,7 @@
 )
 from awscrt.s3 import (
     S3Client,
+    S3FileIoOptions,
     S3RequestTlsMode,
     S3RequestType,
     S3ResponseError,
@@ -87,6 +88,7 @@ def create_s3_crt_client(
     part_size=8 * MB,
     use_ssl=True,
     verify=None,
+    fio_options=None,
 ):
     """
     :type region: str
@@ -130,6 +132,9 @@ def create_s3_crt_client(
         * path/to/cert/bundle.pem - A filename of the CA cert bundle to
             use. Specify this argument if you want to use a custom CA cert
             bundle instead of the default one on your system.
+
+    :type fio_options: Optional[dict]
+    :param fio_options: Kwargs to use to build an `awscrt.s3.S3FileIoOptions`.
     """
 
     event_loop_group = EventLoopGroup(num_threads)
@@ -153,6 +158,9 @@ def create_s3_crt_client(
     target_gbps = _get_crt_throughput_target_gbps(
         provided_throughput_target_bytes=target_throughput
     )
+    crt_fio_options = None
+    if fio_options:
+        crt_fio_options = S3FileIoOptions(**fio_options)
     return S3Client(
         bootstrap=bootstrap,
         region=region,
@@ -162,6 +170,7 @@ def create_s3_crt_client(
         tls_connection_options=tls_connection_options,
         throughput_target_gbps=target_gbps,
         enable_s3express=True,
+        fio_options=crt_fio_options,
     )
 
 

awscli/topics/s3-config.rst

@@ -382,6 +382,50 @@ adjustments mid-transfer command in order to increase throughput and reach the
 requested bandwidth.
 
 
+should_stream
+-------------
+.. note::
+   This configuration option is only supported when the ``preferred_transfer_client``
+   configuration value is set to or resolves to ``crt``. The ``classic`` transfer
+   client does not support this configuration option.
+
+**Default** - ``false``
+
+If set to ``true``, the CRT client will skip buffering parts in-memory before
+sending PUT requests.
+
+
+disk_throughput
+---------------
+.. note::
+   This configuration option is only supported when the ``preferred_transfer_client``
+   configuration value is set to or resolves to ``crt``. The ``classic`` transfer
+   client does not support this configuration option.
+
+**Default** - ``10.0``
+
+The estimated target disk throughput. This value is only applied if
+``should_stream`` is set to ``true``.This value can be specified using
+the same semantics as ``target_throughput``, that is either as the
+number of bytes per second as an integer, or using a rate suffix.
+
+
+direct_io
+---------
+.. note::
+   This configuration option is only supported when the ``preferred_transfer_client``
+   configuration value is set to or resolves to ``crt``. The ``classic`` transfer
+   client does not support this configuration option.
+
+.. note::
+   This configuration option is only supported on Linux.
+
+**Default** - ``false``
+
+If set to ``true``, the CRT client will enable direct IO to bypass the OS
+cache when sending PUT requests. Enabling direct IO may be useful in cases
+where the disk IO outperforms the kernel cache.
+
 Experimental Configuration Values
 =================================
 

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
     "ruamel.yaml.clib>=0.2.0,<=0.2.12",
     "prompt-toolkit>=3.0.24,<3.0.52",
     "distro>=1.5.0,<1.9.0",
-    "awscrt==0.27.6",
+    "awscrt==0.28.1",
     "python-dateutil>=2.1,<=2.9.0",
     "jmespath>=0.7.1,<1.1.0",
     "urllib3>=1.25.4,<1.27",

requirements-docs-lock.txt

@@ -8,47 +8,47 @@ alabaster==0.7.16 \
     --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \
     --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92
     # via sphinx
-awscrt==0.27.6 \
-    --hash=sha256:030eb7d70b7a6a40f503dde0df70cf5d9f1d55bd5e30c222f317ea9e2c9172c9 \
-    --hash=sha256:0acf59ef704e09d51ce7bd16b22475ecaef6e3bcac84287c7500682705c4c38b \
-    --hash=sha256:0f41b4cba30e85882dd9551b4b596a13b01e665455f9d4911d0a0015ff2f24c8 \
-    --hash=sha256:11b5cb5823e5b76977a2ffa473a155e937b65e55691e690016dfc9b7ec5767f9 \
-    --hash=sha256:12652f75c6f4a56d096405beac7c5c89bb7cf4d5eed7edf7d23a214e97379d2f \
-    --hash=sha256:181f5bcb4703bc6f91ed528616f86a642c5342b1ed04543937dba3b13c868c88 \
-    --hash=sha256:2b79917a5a6a3f0229b3cbc2857d0b9254be5eb203f9b55fec086324372050f4 \
-    --hash=sha256:32c9aa4b82a3dc07ebd6c549e6e8690995569e2f648a67729068f4313f12f8f4 \
-    --hash=sha256:3f66fa80906f06ab64a1bf8be67c20aead8ba484aee50e9467dfd6611bf2c583 \
-    --hash=sha256:4256487bdec095de39f72e67125c77d2256d53add26420405949a52441c6dc8f \
-    --hash=sha256:45f3dd0b3fb13dfbea856dd96c9acfe77beba57b9b019444ee962ed2b76276dd \
-    --hash=sha256:4d97f4568a3624715e576052e95c1a93c7844729feddc75c2123ce67e645d26c \
-    --hash=sha256:4ecc74234cc7d8e2f7d1783052d777350d218e28d951b09bc2605f4cd56bb858 \
-    --hash=sha256:50e300d6840d99bdbe57aec871d9958fae9dc54aa71430f1278470a79843b982 \
-    --hash=sha256:57fd911787791ad731c592c7b8427f242043fcb0606c17f539c9bd1991a66239 \
-    --hash=sha256:66991c84992f18165e4e0d33730c447697f1696484d350d5b8f0e474ef70adda \
-    --hash=sha256:6d9a4a928f83618864fbe37901cb60df6bf456f10986be57d6bc36bf7ca2be07 \
-    --hash=sha256:705f591eb5354e9295e01169a2077dfc6198f477bc730079610e55a2ebb46c26 \
-    --hash=sha256:718af70271b9e1d32372e7802ee98b5df6b0b7908f4fa9025fcc398091aaf373 \
-    --hash=sha256:7796105413de8d3de8ce58ad3184710f7e533b62aac4662bea4e53bf63ab88ae \
-    --hash=sha256:786476667b414476b152896d13f213a17e55d058bd3da414e43b020b5375e453 \
-    --hash=sha256:795ccafe031198074a09b4ddd0e1ec08e021d205c443163c4060501a415677a9 \
-    --hash=sha256:7f3109f3cbdee9929d90d283547872ca742fc53990ca204527b60d9fba5d5f1d \
-    --hash=sha256:87d085a626ac9e038cd90f01521acca5ade31eba044d9c109118502b06f23ad3 \
-    --hash=sha256:8a36bab2b7994d7622bc5726bea5d6a651edb669083b9acbfe176ef05fd4e1c5 \
-    --hash=sha256:8bcda12718e011c8333435a550ffd48efe3e55ab94f5449af8e2b784cc0007c5 \
-    --hash=sha256:8bda649a0f8ecf2b5b9e7610508e88c8040d51210eaa4339f08acec0ce2811f6 \
-    --hash=sha256:ae8c2d86cfa2aa3784a4d16a5b05fb6ce08383c96089c29321cf8f7572e24915 \
-    --hash=sha256:ba80f479c4da7e7cc3ed9046808cf2de3c0695b135d3c7352568babd872eb902 \
-    --hash=sha256:c1206fb52b7ac481d8014bc07625f10e31c7bd9a27f28affb6662f63732bf7de \
-    --hash=sha256:c249476f87fcd8efcfe25fd09785b6b0362e54241ba6a14fa66e4afe93d419bd \
-    --hash=sha256:c288e989646f074bb59bddd38d8c9d452ff08964cf24555292cbe95f7b699a40 \
-    --hash=sha256:c3bddd393cec9e7470526785956fa01623fcbf38d043e4ea60c38d4b972c1f44 \
-    --hash=sha256:c42ffcc8dc0ea06cab1f66506b5ed3861769785bebdadad1d70a911305b4ec44 \
-    --hash=sha256:d8769b297ebd1bf51a2cec68d5b67e9aff840a6ac87d0aa0ca2962cc74b14efd \
-    --hash=sha256:e3377cee0c494be33b40b86beaab67bfd2b34fd7d51f827262b41fe48bf6301b \
-    --hash=sha256:e93fbf098b021fe23537e5b4d706638f91ab4ff5df57ea145c3b0aefef5ca288 \
-    --hash=sha256:ea2b7d9a4b9109b29878165318447afd16042d91584ec9fb49cb3250218cdd12 \
-    --hash=sha256:eb09d17684e3f6f99a9db5eaf4ac011327589eeee8af2a54b662fd0626ea6d86 \
-    --hash=sha256:fdf6406de9d6ff510cccba6ca020a248d2d673c9c5440c06f2c21a4ae7555672
+awscrt==0.28.1 \
+    --hash=sha256:0860752c93a6dbfeb37d465a63554c25f0da3dea406b5f6a8bf5f0c6b09c016a \
+    --hash=sha256:16659d6424eec7e4c49a3526bbc452dcdcb2d6eb1c199d4dd113ff2f6c2b1aed \
+    --hash=sha256:1dcb33d7cf8f69881ac6ef75a5b9b40816be58678b1bb07ccbe0230281bdbc81 \
+    --hash=sha256:1ea1ac8c3e6915dcfade465fea25b2579c8f0d96c5daab4c61b89195cd39c41f \
+    --hash=sha256:22311d25135b937ee5617e35a6554961727527dcfa3e06efdefe187a6abe65c4 \
+    --hash=sha256:36ad3391fc55f4b7ba17148a0db343a9a1a009979dbcc72faba4e9ccebe3d516 \
+    --hash=sha256:3a060d930939f142345f46a344e19ffc0dada657b04d02216b8adffba550c0a0 \
+    --hash=sha256:3df2316e77ad88c456b7eb2c9928007d379ed892154c1969d35b98653617e576 \
+    --hash=sha256:3e0a23635aa75b4af163ff9bf5a0873928369b1ac32c8b1351741a95472ccf71 \
+    --hash=sha256:43f81ca6bfe85c38ad9765605aaaa646a1ed6fd7210dbedf67c113dd245f425e \
+    --hash=sha256:47f885104065918d311102e2b08b943966717c0f3b0c5de5908d2fd08de32198 \
+    --hash=sha256:491b8b9c73a288cfd5e0cbdac16aabb5313d5cfc33bbe461763a5ddc26624f70 \
+    --hash=sha256:4c6c7125b7e9fcc999eb685d1cace8d4f2ffc63f8f3d8ef7f77e1a97d9552863 \
+    --hash=sha256:51daa07b1689f2dbd677c2e935887b090c771ff1b4128a3f6c4a331d4c3c012a \
+    --hash=sha256:592f4b234ecafa6cde86e55e42c4fe84c4e1ffe9fb11b0a8b8f0ffb8c62fa2cc \
+    --hash=sha256:5b67c80351991892e9b8ceb02bae94fc8cf2daf7ce62ca9aecc4509a7712670d \
+    --hash=sha256:670caaf556876913bcfb9d8183d43d67a6c7b52998f2f398abd1c21632a006f8 \
+    --hash=sha256:6e7b094587e5332d428300340dcc18794a1fcfa76d636f216fc0f5c8405ba604 \
+    --hash=sha256:70a28fd6ff3e0abb7854ea8a9133bc9e5de681a0d9bdbd8a599a23d13a448685 \
+    --hash=sha256:80583a7e5f36d2b3fb3291d92b696b1450a92eeec59910114805039420a4e85f \
+    --hash=sha256:83276c2942501d1232051d64cbd709ba393a69f779d5eb2b4167f7a2dc5ed2e8 \
+    --hash=sha256:8a7edbc50f573e6c265c2771284ab43c0629bd557efce05e41e76931d2ad7fb7 \
+    --hash=sha256:96e7188b6733ea466f0dc64bc3762a76572453fc6bab17cde579c87ddda134fc \
+    --hash=sha256:9849c88ca0830396724acf988e2759895118fe7dd2a23dab21978c8600d01a11 \
+    --hash=sha256:993708d51c165fc4ed5e8db0668ff0b70b0e4342da2b8981ba1bb694ae52d88b \
+    --hash=sha256:9ad8541719f6a5fff6ab0476f79b77675e826e22c4ef39c0a9969fc2b0c92752 \
+    --hash=sha256:9e69f163a207a8b172abbfea1f51045301ed1ac8bbaf76958a6b5e81d72e5b89 \
+    --hash=sha256:a046922ddb049c58f1fd2b3f93f75372319f18599dd70f1a2880d2911c71cec6 \
+    --hash=sha256:ac02f10f7384fdb68187f8d5d94743a271b16fa94be81481ce7684942f6a4b35 \
+    --hash=sha256:b16321f1d2bf5b4991a213059c1b5dc07954edfc424d154b093824465ec94ce2 \
+    --hash=sha256:b1870ff748cb2a5e335d7bbc666a39e54b061ca1742fea23b77d0ce50701aedb \
+    --hash=sha256:b3f2dc580f474418b9a39be63b06cf16c583a537f11bb4b30a38a6c428b384b0 \
+    --hash=sha256:cb36052f9aa34e77687a8037559bbea331fc9d5d77cd71ab0cf4e6d72af73f72 \
+    --hash=sha256:cf09b45002718cd86013dfc313320e615660bda7b39c5b605097f2db7e149f16 \
+    --hash=sha256:d2f20bc774599b9d85ce66689415da529ddd1d2215da818e005deedc4688fe61 \
+    --hash=sha256:d6f9e83f03b689351e7f69495919f90d77e278f371192c87967e1c8ae9861a4f \
+    --hash=sha256:e58740cf0e41552fdf7909e10814b312ab090ebe54741354a61507e0c6d4ebfd \
+    --hash=sha256:f14543403b9e392839e717d95a8abc3ba6dc732132f37f107b880a5fc890702f \
+    --hash=sha256:fc59829152a5806eb2708aca5c5084c11dd18ecbe765e03eb314d5a360eeaa62 \
+    --hash=sha256:fc8e2307d9dbe76842015a14701ff7e9cf2619d674621b2d55b769414e17b3fc
     # via awscli (pyproject.toml)
 babel==2.17.0 \
     --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \

requirements/download-deps/bootstrap-lock.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/download-deps/bootstrap-lock.txt --unsafe-package=flit-core --unsafe-package=pip --unsafe-package=setuptools --unsafe-package=wheel requirements/download-deps/bootstrap.txt
+#    pip-compile --allow-unsafe --cert=None --client-cert=None --generate-hashes --index-url=None --output-file=requirements/download-deps/bootstrap-lock.txt --pip-args=None --unsafe-package=flit-core --unsafe-package=pip --unsafe-package=setuptools --unsafe-package=wheel requirements/download-deps/bootstrap.txt
 #
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/download-deps/bootstrap-win-lock.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.13
 # by the following command:
 #
-#    pip-compile --allow-unsafe --generate-hashes --output-file='requirements\download-deps\bootstrap-win-lock.txt' --unsafe-package=flit-core --unsafe-package=pip --unsafe-package=setuptools --unsafe-package=wheel 'requirements\download-deps\bootstrap.txt'
+#    pip-compile --allow-unsafe --cert=None --client-cert=None --generate-hashes --index-url=None --output-file='requirements\download-deps\bootstrap-win-lock.txt' --pip-args=None --unsafe-package=flit-core --unsafe-package=pip --unsafe-package=setuptools --unsafe-package=wheel 'requirements\download-deps\bootstrap.txt'
 #
 
 # The following packages are considered to be unsafe in a requirements file: