Pure python request pipeline

CHANGES.md

@@ -2,4 +2,13 @@
 
 ## Unreleased
 
-* Description of change. (Issue Number)
+### Breaking Changes
+
+* Removed the `http_client` config option in favor of the generic `transport`.
+
+### Features
+
+* Removed code-generated protocol implementations in favor of hand-written
+  implementations based on schemas.
+* Moved documentation for structure members into doc strings after the member's
+  dataclass field declaration.

codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsAuthIntegration.java

@@ -32,6 +32,9 @@ public class AwsAuthIntegration implements PythonIntegration {
 
     @Override
     public List<RuntimeClientPlugin> getClientPlugins(GenerationContext context) {
+        if (!hasSigV4Auth(context)) {
+            return List.of();
+        }
         return List.of(
                 RuntimeClientPlugin.builder()
                         .servicePredicate((model, service) -> service.hasTrait(SigV4Trait.class))
@@ -61,6 +64,24 @@ public List<RuntimeClientPlugin> getClientPlugins(GenerationContext context) {
                                 .nullable(true)
                                 .build())
                         .addConfigProperty(REGION)
+                        .addConfigProperty(ConfigProperty.builder()
+                                .name("aws_access_key_id")
+                                .type(Symbol.builder().name("str").build())
+                                .documentation("The identifier for a secret access key.")
+                                .nullable(true)
+                                .build())
+                        .addConfigProperty(ConfigProperty.builder()
+                                .name("aws_secret_access_key")
+                                .type(Symbol.builder().name("str").build())
+                                .nullable(true)
+                                .documentation("A secret access key that can be used to sign requests.")
+                                .build())
+                        .addConfigProperty(ConfigProperty.builder()
+                                .name("aws_session_token")
+                                .type(Symbol.builder().name("str").build())
+                                .documentation("An access key ID that identifies temporary security credentials.")
+                                .nullable(true)
+                                .build())
                         .authScheme(new Sigv4AuthScheme())
                         .build());
     }
@@ -70,7 +91,6 @@ public void customize(GenerationContext context) {
         if (!hasSigV4Auth(context)) {
             return;
         }
-        var trait = context.settings().service(context.model()).expectTrait(SigV4Trait.class);
         var resolver = CodegenUtils.getHttpAuthSchemeResolverSymbol(context.settings());
 
         // Add a function that generates the http auth option for api key auth.
@@ -91,8 +111,7 @@ public void customize(GenerationContext context) {
                         )
                     """,
                     SIGV4_OPTION_GENERATOR_NAME,
-                    SigV4Trait.ID.toString(),
-                    trait.getName());
+                    SigV4Trait.ID.toString());
             writer.popState();
         });
     }

codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java

@@ -124,13 +124,6 @@ public void generateService(GenerateServiceDirective<GenerationContext, PythonSe
         if (protocolGenerator == null) {
             return;
         }
-
-        protocolGenerator.generateSharedSerializerComponents(directive.context());
-        protocolGenerator.generateRequestSerializers(directive.context());
-
-        protocolGenerator.generateSharedDeserializerComponents(directive.context());
-        protocolGenerator.generateResponseDeserializers(directive.context());
-
         protocolGenerator.generateProtocolTests(directive.context());
     }
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java

@@ -186,7 +186,7 @@ private void generateRequestTest(OperationShape operation, HttpRequestTestCase t
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://$L/$L",
-                                    http_client = $T(),
+                                    transport = $T(),
                                     retry_strategy=SimpleRetryStrategy(max_attempts=1),
                                 )
                                 """,
@@ -432,7 +432,7 @@ private void generateResponseTest(OperationShape operation, HttpResponseTestCase
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://example.com",
-                                    http_client = $T(
+                                    transport = $T(
                                         status=$L,
                                         headers=$J,
                                         body=b$S,
@@ -485,7 +485,7 @@ private void generateErrorResponseTest(
                         writer.write("""
                                 config = $T(
                                     endpoint_uri="https://example.com",
-                                    http_client = $T(
+                                    transport = $T(
                                         status=$L,
                                         headers=$J,
                                         body=b$S,

codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java

@@ -73,6 +73,15 @@ public final class SmithyPythonDependency {
             Type.DEPENDENCY,
             false);
 
+    /**
+     * Smithy core functionality for AWS.
+     */
+    public static final PythonDependency SMITHY_AWS_CORE = new PythonDependency(
+            "smithy_aws_core",
+            "<0.1.0",
+            Type.DEPENDENCY,
+            false);
+
     /**
      * testing framework used in generated functional tests.
      */

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java

@@ -117,38 +117,58 @@ public ConfigGenerator(PythonSettings settings, GenerationContext context) {
         this.settings = settings;
     }
 
-    private static List<ConfigProperty> getHttpProperties(GenerationContext context) {
-        var properties = new ArrayList<ConfigProperty>(HTTP_PROPERTIES.size() + 2);
-        var clientBuilder = ConfigProperty.builder()
-                .name("http_client")
+    private static List<ConfigProperty> getProtocolProperties(GenerationContext context) {
+        var properties = new ArrayList<ConfigProperty>();
+        var protocolBuilder = ConfigProperty.builder()
+                .name("protocol")
                 .type(Symbol.builder()
-                        .name("HTTPClient")
-                        .namespace("smithy_http.aio.interfaces", ".")
-                        .addDependency(SmithyPythonDependency.SMITHY_HTTP)
+                        .name("ClientProtocol[Any, Any]")
+                        .addReference(Symbol.builder()
+                                .name("ClientProtocol")
+                                .namespace("smithy_core.aio.interfaces", ".")
+                                .build())
+                        .build())
+                .documentation("The protocol to serialize and deserialize requests with.")
+                .initialize(w -> {
+                    w.write("self.protocol = protocol or ${C|}",
+                            w.consumer(writer -> context.protocolGenerator().initializeProtocol(context, writer)));
+                });
+
+        var transportBuilder = ConfigProperty.builder()
+                .name("transport")
+                .type(Symbol.builder()
+                        .name("ClientTransport[Any, Any]")
+                        .addReference(Symbol.builder()
+                                .name("ClientTransport")
+                                .namespace("smithy_core.aio.interfaces", ".")
+                                .build())
                         .build())
-                .documentation("The HTTP client used to make requests.")
-                .nullable(false);
+                .documentation("The transport to use to send requests (e.g. an HTTP client).");
 
-        if (usesHttp2(context)) {
-            clientBuilder
-                    .initialize(writer -> {
-                        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("awscrt"));
-                        writer.addImport("smithy_http.aio.crt", "AWSCRTHTTPClient");
-                        writer.write("self.http_client = http_client or AWSCRTHTTPClient()");
-                    });
+        if (context.applicationProtocol().isHttpProtocol()) {
+            properties.addAll(HTTP_PROPERTIES);
+            if (usesHttp2(context)) {
+                transportBuilder
+                        .initialize(writer -> {
+                            writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("awscrt"));
+                            writer.addImport("smithy_http.aio.crt", "AWSCRTHTTPClient");
+                            writer.write("self.transport = transport or AWSCRTHTTPClient()");
+                        });
 
-        } else {
-            clientBuilder
-                    .initialize(writer -> {
-                        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("aiohttp"));
-                        writer.addImport("smithy_http.aio.aiohttp", "AIOHTTPClient");
-                        writer.write("self.http_client = http_client or AIOHTTPClient()");
-                    });
+            } else {
+                transportBuilder
+                        .initialize(writer -> {
+                            writer.addDependency(
+                                    SmithyPythonDependency.SMITHY_HTTP.withOptionalDependencies("aiohttp"));
+                            writer.addImport("smithy_http.aio.aiohttp", "AIOHTTPClient");
+                            writer.write("self.transport = transport or AIOHTTPClient()");
+                        });
+            }
         }
-        properties.add(clientBuilder.build());
 
-        properties.addAll(HTTP_PROPERTIES);
-        return List.copyOf(properties);
+        properties.add(protocolBuilder.build());
+        properties.add(transportBuilder.build());
+        return properties;
     }
 
     private static boolean usesHttp2(GenerationContext context) {
@@ -289,6 +309,7 @@ private void generateConfig(GenerationContext context, PythonWriter writer) {
         // Initialize a set of config properties with our base properties.
         var properties = new TreeSet<>(Comparator.comparing(ConfigProperty::name));
         properties.addAll(BASE_PROPERTIES);
+        properties.addAll(getProtocolProperties(context));
 
         // Add in auth configuration if the service supports auth.
         var serviceIndex = ServiceIndex.of(context.model());
@@ -297,13 +318,6 @@ private void generateConfig(GenerationContext context, PythonWriter writer) {
             writer.onSection(new AddAuthHelper());
         }
 
-        // Smithy is transport agnostic, so we don't add http-related properties by default.
-        // Nevertheless, HTTP is the most common use case so we standardize those settings
-        // and add them in if the protocol is going to need them.
-        if (context.applicationProtocol().isHttpProtocol()) {
-            properties.addAll(getHttpProperties(context));
-        }
-
         var model = context.model();
         var service = context.settings().service(model);
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java

@@ -28,6 +28,7 @@
 import software.amazon.smithy.model.shapes.StructureShape;
 import software.amazon.smithy.model.shapes.TimestampShape;
 import software.amazon.smithy.model.shapes.UnionShape;
+import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.python.codegen.GenerationContext;
 import software.amazon.smithy.python.codegen.SymbolProperties;
 import software.amazon.smithy.python.codegen.writer.PythonWriter;
@@ -100,7 +101,11 @@ private void writeSchema() {
 
     @Override
     public Void blobShape(BlobShape shape) {
-        writeDeserializer(shape);
+        if (shape.hasTrait(StreamingTrait.class)) {
+            writeDeserializer("data_stream");
+        } else {
+            writeDeserializer(shape);
+        }
         return null;
     }
 

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberSerializerGenerator.java

@@ -29,6 +29,7 @@
 import software.amazon.smithy.model.shapes.StructureShape;
 import software.amazon.smithy.model.shapes.TimestampShape;
 import software.amazon.smithy.model.shapes.UnionShape;
+import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.python.codegen.GenerationContext;
 import software.amazon.smithy.python.codegen.SymbolProperties;
 import software.amazon.smithy.python.codegen.writer.PythonWriter;
@@ -107,7 +108,11 @@ private void writeSchema() {
 
     @Override
     public Void blobShape(BlobShape shape) {
-        writeSerializer(shape);
+        if (shape.hasTrait(StreamingTrait.class)) {
+            writeSerializer("data_stream");
+        } else {
+            writeSerializer(shape);
+        }
         return null;
     }