chore!: Python and Go codegen runs in polymorph_code_gen target

.github/workflows/manual.yml

@@ -16,6 +16,10 @@ jobs:
     uses: ./.github/workflows/test_models_dafny_verification.yml
     with:
       dafny: ${{ inputs.dafny }}
+  manual-ci-polymorph-code-gen:
+    uses: ./.github/workflows/test_models_polymorph_code_gen.yml
+    with:
+      dafny: ${{ inputs.dafny }}
   manual-ci-java:
     uses: ./.github/workflows/test_models_java_tests.yml
     with:

.github/workflows/pull.yml

@@ -27,6 +27,15 @@ jobs:
     uses: ./.github/workflows/test_models_dafny_verification.yml
     with:
       dafny: ${{ matrix.dafny-version }}
+  pr-ci-polymorph-code-gen:
+    needs: pr-populate-dafny-versions
+    strategy:
+      fail-fast: false
+      matrix:
+        dafny-version: ${{ fromJson(needs.pr-populate-dafny-versions.outputs.dafny-version-list) }}
+    uses: ./.github/workflows/test_models_polymorph_code_gen.yml
+    with:
+      dafny: ${{ matrix.dafny-version }}
   pr-ci-java:
     needs: pr-populate-dafny-versions
     strategy:

.github/workflows/push.yml

@@ -29,6 +29,15 @@ jobs:
     uses: ./.github/workflows/test_models_dafny_verification.yml
     with:
       dafny: ${{ matrix.dafny-version }}
+  push-ci-polymorph-code-gen:
+    needs: pr-populate-dafny-versions
+    strategy:
+      fail-fast: false
+      matrix:
+        dafny-version: ${{ fromJson(needs.pr-populate-dafny-versions.outputs.dafny-version-list) }}
+    uses: ./.github/workflows/test_models_polymorph_code_gen.yml
+    with:
+      dafny: ${{ matrix.dafny-version }}
   push-ci-java:
     needs: pr-populate-dafny-versions
     strategy:

.github/workflows/test_models_polymorph_code_gen.yml

@@ -0,0 +1,51 @@
+# This workflow runs polymorph_code_gen.
+name: polymorph_code_gen test
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+
+jobs:
+  testPolymorphCodeGen:
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Support longpaths on Git checkout
+        run: |
+          git config --global core.longpaths true
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Dafny
+        uses: dafny-lang/[email protected]
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Setup Java 17 for codegen
+        uses: actions/setup-java@v3
+        with:
+          distribution: "corretto"
+          java-version: 17
+
+      - name: Setup Python for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Install Python formatters
+        run: make setup_smithy_dafny_python
+
+      - name: Build one TestModel with all languages (smoke test for polymorph_code_gen target)
+        shell: bash
+        working-directory: TestModels/SimpleTypes/SimpleString
+        run: make polymorph_code_gen

SmithyDafnyMakefile.mk

@@ -288,13 +288,15 @@ _polymorph:
 	$(OUTPUT_JAVA_TEST) \
 	$(OUTPUT_DOTNET) \
 	$(OUTPUT_GO) \
+	$(if $(strip $(GO_MODULE_NAME)),--go-module-name $(GO_MODULE_NAME),) \
+	$(GO_DEPENDENCY_MODULE_NAMES) \
 	$(OUTPUT_PYTHON) \
-	$(MODULE_NAME) \
+	$(if $(strip $(PYTHON_MODULE_NAME)),--python-module-name $(PYTHON_MODULE_NAME),) \
+	$(PYTHON_DEPENDENCY_MODULE_NAMES) \
 	$(OUTPUT_RUST) \
 	--model $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(SMITHY_MODEL_ROOT)) \
 	--dependent-model $(PROJECT_ROOT)/$(SMITHY_DEPS) \
 	$(patsubst %, --dependent-model $(PROJECT_ROOT)/%/Model, $($(service_deps_var))) \
-	$(DEPENDENCY_MODULE_NAMES) \
 	$(patsubst %, --namespace %, $($(namespace_var))) \
 	$(OUTPUT_LOCAL_SERVICE_$(SERVICE)) \
 	$(AWS_SDK_CMD) \
@@ -313,13 +315,15 @@ _polymorph_wrapped:
 	$(OUTPUT_DOTNET_WRAPPED) \
 	$(OUTPUT_JAVA_WRAPPED) \
 	$(OUTPUT_GO_WRAPPED) \
+	$(if $(strip $(GO_MODULE_NAME)),--go-module-name $(GO_MODULE_NAME),) \
+	$(GO_DEPENDENCY_MODULE_NAMES) \
 	$(OUTPUT_PYTHON_WRAPPED) \
-	$(MODULE_NAME) \
+	$(if $(strip $(PYTHON_MODULE_NAME)),--python-module-name $(PYTHON_MODULE_NAME),) \
+	$(PYTHON_DEPENDENCY_MODULE_NAMES) \
 	$(OUTPUT_RUST_WRAPPED) \
 	--model $(if $(DIR_STRUCTURE_V2),$(LIBRARY_ROOT)/dafny/$(SERVICE)/Model,$(LIBRARY_ROOT)/Model) \
 	--dependent-model $(PROJECT_ROOT)/$(SMITHY_DEPS) \
 	$(patsubst %, --dependent-model $(PROJECT_ROOT)/%/Model, $($(service_deps_var))) \
-	$(DEPENDENCY_MODULE_NAMES) \
 	--namespace $($(namespace_var)) \
 	--local-service-test \
 	$(AWS_SDK_CMD) \
@@ -354,6 +358,7 @@ _polymorph_code_gen: OUTPUT_DOTNET=\
 _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated
 _polymorph_code_gen: OUTPUT_GO=--output-go $(LIBRARY_ROOT)/runtimes/go/
 _polymorph_code_gen: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated
+_polymorph_code_gen: OUTPUT_PYTHON=--output-python $(LIBRARY_ROOT)/runtimes/python/src/$(PYTHON_MODULE_NAME)/smithygenerated
 _polymorph_code_gen: OUTPUT_RUST=--output-rust $(LIBRARY_ROOT)/runtimes/rust
 _polymorph_code_gen: _polymorph
 
@@ -429,10 +434,6 @@ polymorph_python:
 	done
 
 _polymorph_python: OUTPUT_PYTHON=--output-python $(LIBRARY_ROOT)/runtimes/python/src/$(PYTHON_MODULE_NAME)/smithygenerated
-# Defined per-Makefile
-_polymorph_python: MODULE_NAME=--library-name $(PYTHON_MODULE_NAME)
-# Defined per-Makefile
-_polymorph_python: DEPENDENCY_MODULE_NAMES=$(PYTHON_DEPENDENCY_MODULE_NAMES)
 _polymorph_python: _polymorph
 
 # Dependency for formatting generating Java code
@@ -469,8 +470,6 @@ polymorph_go:
 	done
 
 _polymorph_go: OUTPUT_GO=--output-go $(LIBRARY_ROOT)/runtimes/go/
-_polymorph_go: MODULE_NAME=--library-name $(GO_MODULE_NAME)
-_polymorph_go: DEPENDENCY_MODULE_NAMES = $(GO_DEPENDENCY_MODULE_NAMES)
 # TODO: run_goimports should be an independent command. Right now it is required because of import issues in polymorph_go
 _polymorph_go: _polymorph _mv_polymorph_go run_goimports
 
@@ -659,6 +658,9 @@ _clean:
 	rm -rf $(LIBRARY_ROOT)/TestResults
 	rm -rf $(LIBRARY_ROOT)/runtimes/net/Generated $(LIBRARY_ROOT)/runtimes/net/bin $(LIBRARY_ROOT)/runtimes/net/obj
 	rm -rf $(LIBRARY_ROOT)/runtimes/net/tests/bin $(LIBRARY_ROOT)/runtimes/net/tests/obj
+	rm -rf $(LIBRARY_ROOT)/runtimes/python/src/**/smithygenerated
+	rm -rf $(LIBRARY_ROOT)/runtimes/python/src/**/internaldafny/generated
+	rm -rf $(LIBRARY_ROOT)/runtimes/python/test/internaldafny/generated
 
 clean: _clean
 
@@ -706,7 +708,7 @@ setup_python: setup_smithy_dafny_python
 setup_python:
 	python3 -m pip install poetry
 
-net: polymorph_dafny transpile_python polymorph_python test_python
+python: polymorph_dafny transpile_python polymorph_python test_python
 
 # Python MUST transpile dependencies first to generate .dtr files
 transpile_python: $(if $(ENABLE_EXTERN_PROCESSING), _no_extern_pre_transpile, )

TestModels/CallingAWSSDKFromLocalService/Makefile

@@ -46,10 +46,10 @@ WRAPPED_INDEX_FILE_WITHOUT_EXTERN_STRING="module WrappedSimpleCallingawssdkfroml
 # Go
 
 GO_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=com.amazonaws.kms=github.com/smithy-lang/smithy-dafny/kmsv2 \
-	--dependency-library-name=com.amazonaws.dynamodb=github.com/smithy-lang/smithy-dafny/ddbv2 \
-	--dependency-library-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
-	--dependency-library-name=sdk.com.amazonaws.kms=github.com/aws/aws-sdk-go-v2/service/kms
+	--go-dependency-module-name=com.amazonaws.kms=github.com/smithy-lang/smithy-dafny/kmsv2 \
+	--go-dependency-module-name=com.amazonaws.dynamodb=github.com/smithy-lang/smithy-dafny/ddbv2 \
+	--go-dependency-module-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
+	--go-dependency-module-name=sdk.com.amazonaws.kms=github.com/aws/aws-sdk-go-v2/service/kms
 
 GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/TestModels/CallingAWSSDKFromLocalService"
 

TestModels/Dependencies/Makefile

@@ -44,10 +44,10 @@ SMITHY_DEPS=dafny-dependencies/Model/traits.smithy
 GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/TestModels/Dependencies"
 
 GO_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=simple.constraints=github.com/smithy-lang/smithy-dafny/TestModels/Constraints \
-	--dependency-library-name=simple.errors=github.com/smithy-lang/smithy-dafny/TestModels/Errors \
-	--dependency-library-name=simple.extendable.resources=github.com/smithy-lang/smithy-dafny/TestModels/Extendable \
-	--dependency-library-name=simple.resources=github.com/smithy-lang/smithy-dafny/TestModels/Resource
+	--go-dependency-module-name=simple.constraints=github.com/smithy-lang/smithy-dafny/TestModels/Constraints \
+	--go-dependency-module-name=simple.errors=github.com/smithy-lang/smithy-dafny/TestModels/Errors \
+	--go-dependency-module-name=simple.extendable.resources=github.com/smithy-lang/smithy-dafny/TestModels/Extendable \
+	--go-dependency-module-name=simple.resources=github.com/smithy-lang/smithy-dafny/TestModels/Resource
 
 TRANSLATION_RECORD_GO := \
 	Resource/runtimes/go/ImplementationFromDafny-go/ImplementationFromDafny-go.dtr \
@@ -66,10 +66,10 @@ clean: _clean
 PYTHON_MODULE_NAME=simple_dependencies
 
 PYTHON_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=simple.resources=simple_resources \
-	--dependency-library-name=simple.errors=simple_errors \
-	--dependency-library-name=simple.extendable.resources=simple_extendable_resources \
-	--dependency-library-name=simple.constraints=simple_constraints
+	--python-dependency-module-name=simple.resources=simple_resources \
+	--python-dependency-module-name=simple.errors=simple_errors \
+	--python-dependency-module-name=simple.extendable.resources=simple_extendable_resources \
+	--python-dependency-module-name=simple.constraints=simple_constraints
 
 TRANSLATION_RECORD_PYTHON := \
     --translation-record ../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr \

TestModels/MultipleModels/Makefile

@@ -42,7 +42,7 @@ TRANSLATION_RECORD_GO := \
 PYTHON_MODULE_NAME=multiple_models
 
 PYTHON_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=simple.multiplemodels.dependencyproject=multiple_models
+	--python-dependency-module-name=simple.multiplemodels.dependencyproject=multiple_models
 
 TRANSLATION_RECORD_PYTHON := \
     --translation-record ../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr

TestModels/aws-sdks/ddb/Makefile

@@ -28,7 +28,7 @@ AWS_SDK_CMD=--aws-sdk
 GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/ddb"
 
 GO_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
+	--go-dependency-module-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
 
 TRANSLATION_RECORD_GO := \
     dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/ImplementationFromDafny-go.dtr

TestModels/aws-sdks/ddbv2/Makefile

@@ -54,7 +54,7 @@ TRANSLATION_RECORD_PYTHON := \
 GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/ddbv2"
 
 GO_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
+	--go-dependency-module-name=sdk.com.amazonaws.dynamodb=github.com/aws/aws-sdk-go-v2/service/dynamodb \
 
 TRANSLATION_RECORD_GO := \
 	dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/ImplementationFromDafny-go.dtr

TestModels/aws-sdks/kmsv2/Makefile

@@ -55,7 +55,7 @@ TRANSLATION_RECORD_PYTHON := \
 GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/kmsv2"
 
 GO_DEPENDENCY_MODULE_NAMES := \
-	--dependency-library-name=sdk.com.amazonaws.kms=github.com/aws/aws-sdk-go-v2/service/kms \
+	--go-dependency-module-name=sdk.com.amazonaws.kms=github.com/aws/aws-sdk-go-v2/service/kms \
 
 TRANSLATION_RECORD_GO := \
     dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/ImplementationFromDafny-go.dtr

codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java

@@ -129,7 +129,8 @@ public static void main(String[] args) {
       .withLibraryRoot(cliArguments.libraryRoot)
       .withServiceModel(serviceModel)
       .withDependentModelPaths(cliArguments.dependentModelPaths)
-      .withDependencyLibraryNames(cliArguments.dependencyLibraryNames)
+      .withDependencyPythonModuleNames(cliArguments.dependencyPythonModuleNames)
+      .withDependencyGoModuleNames(cliArguments.dependencyGoModuleNames)
       .withNamespaces(cliArguments.namespaces)
       .withTargetLangOutputDirs(outputDirs)
       .withTargetLangTestOutputDirs(testOutputDirs)
@@ -159,7 +160,10 @@ public static void main(String[] args) {
     cliArguments.includeDafnyFile.ifPresent(
       engineBuilder::withIncludeDafnyFile
     );
-    cliArguments.libraryName.ifPresent(engineBuilder::withLibraryName);
+    cliArguments.pythonModuleName.ifPresent(
+      engineBuilder::withPythonModuleName
+    );
+    cliArguments.goModuleName.ifPresent(engineBuilder::withGoModuleName);
     cliArguments.patchFilesDir.ifPresent(engineBuilder::withPatchFilesDir);
     final CodegenEngine engine = engineBuilder.build();
     switch (cliArguments.command) {
@@ -202,10 +206,20 @@ private static Options getCliOptionsForBuild() {
       )
       .addOption(
         Option
-          .builder("dln")
-          .longOpt("dependency-library-name")
+          .builder("pdmn")
+          .longOpt("python-dependency-module-name")
           .desc(
-            "namespace-to-library-name map entry for a dependency namespace"
+            "<optional> if using Python, a dependency namespace-to-Python module name map entry"
+          )
+          .hasArg()
+          .build()
+      )
+      .addOption(
+        Option
+          .builder("gdmn")
+          .longOpt("go-dependency-module-name")
+          .desc(
+            "<optional> if using Go, a dependency namespace-to-Go module name map entry"
           )
           .hasArg()
           .build()
@@ -221,10 +235,20 @@ private static Options getCliOptionsForBuild() {
       )
       .addOption(
         Option
-          .builder("ln")
-          .longOpt("library-name")
+          .builder("pmn")
+          .longOpt("python-module-name")
+          .desc(
+            "<optional> if using Python, the intended Python module name for generated code"
+          )
+          .hasArg()
+          .build()
+      )
+      .addOption(
+        Option
+          .builder("gmn")
+          .longOpt("go-module-name")
           .desc(
-            "if generating for a language that uses library names (go, python), the name of the library in that language"
+            "<optional> if using Go, the intended Go module name for generated code"
           )
           .hasArg()
           .build()
@@ -472,9 +496,11 @@ private record CliArguments(
     Path libraryRoot,
     Path modelPath,
     Path[] dependentModelPaths,
-    Map<String, String> dependencyLibraryNames,
+    Map<String, String> dependencyPythonModuleNames,
+    Map<String, String> dependencyGoModuleNames,
     Set<String> namespaces,
-    Optional<String> libraryName,
+    Optional<String> pythonModuleName,
+    Optional<String> goModuleName,
     Optional<Path> outputDotnetDir,
     Optional<Path> outputJavaDir,
     Optional<Path> outputGoDir,
@@ -538,10 +564,18 @@ static Optional<CliArguments> parse(String[] args) throws ParseException {
       // ex. `dependency-library-name=aws.cryptography.materialproviders=aws_cryptographic_materialproviders`
       // maps the Smithy namespace `aws.cryptography.materialproviders` to a module name `aws_cryptographic_materialproviders`
       // via a map key of "aws.cryptography.materialproviders" and a value of "aws_cryptographic_materialproviders"
-      final Map<String, String> dependencyNamespacesToLibraryNamesMap =
-        commandLine.hasOption("dependency-library-name")
+      final Map<String, String> dependencyNamespacesToPythonModuleNamesMap =
+        commandLine.hasOption("python-dependency-module-name")
           ? Arrays
-            .stream(commandLine.getOptionValues("dln"))
+            .stream(commandLine.getOptionValues("pdmn"))
+            .map(s -> s.split("="))
+            .collect(Collectors.toMap(i -> i[0], i -> i[1]))
+          : new HashMap<>();
+
+      final Map<String, String> dependencyNamespacesToGoModuleNamesMap =
+        commandLine.hasOption("go-dependency-module-name")
+          ? Arrays
+            .stream(commandLine.getOptionValues("gdmn"))
             .map(s -> s.split("="))
             .collect(Collectors.toMap(i -> i[0], i -> i[1]))
           : new HashMap<>();
@@ -551,8 +585,12 @@ static Optional<CliArguments> parse(String[] args) throws ParseException {
         .<Set<String>>map(ns -> new HashSet<>(Arrays.asList(ns)))
         .orElse(Collections.emptySet());
 
-      final Optional<String> libraryName = Optional.ofNullable(
-        commandLine.getOptionValue("library-name")
+      final Optional<String> pythonModuleName = Optional.ofNullable(
+        commandLine.getOptionValue("python-module-name")
+      );
+
+      final Optional<String> goModuleName = Optional.ofNullable(
+        commandLine.getOptionValue("go-module-name")
       );
 
       Optional<Path> outputDafnyDir = Optional
@@ -643,9 +681,11 @@ static Optional<CliArguments> parse(String[] args) throws ParseException {
           libraryRoot,
           modelPath,
           dependentModelPaths,
-          dependencyNamespacesToLibraryNamesMap,
+          dependencyNamespacesToPythonModuleNamesMap,
+          dependencyNamespacesToGoModuleNamesMap,
           namespaces,
-          libraryName,
+          pythonModuleName,
+          goModuleName,
           outputDotnetDir,
           outputJavaDir,
           outputGoDir,