Treat empty string as null during deserialization with configs (#3111)

Author: fengzhou-msft

src/AutoRest.CSharp/Common/AutoRest/Communication/StandaloneGeneratorRunner.cs

@@ -176,8 +176,13 @@ internal static string SaveConfiguration()
                     Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, Configuration.Options.ModelFactoryForHlc, Configuration.ModelFactoryForHlc);
                     WriteIfNotDefault(writer, Configuration.Options.UnreferencedTypesHandling, Configuration.UnreferencedTypesHandling);
                     WriteIfNotDefault(writer, Configuration.Options.ProjectFolder, Configuration.RelativeProjectFolder);
-                    Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.ProtocolMethodList), Configuration.ProtocolMethodList);
-                    Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.SuppressAbstractBaseClasses), Configuration.SuppressAbstractBaseClasses);
+                    Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.Options.ProtocolMethodList), Configuration.ProtocolMethodList);
+                    Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.Options.SuppressAbstractBaseClasses), Configuration.SuppressAbstractBaseClasses);
+                    Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.Options.ModelsToTreatEmptyStringAsNull), Configuration.ModelsToTreatEmptyStringAsNull.ToList<string>());
+                    if (Configuration.ModelsToTreatEmptyStringAsNull.Any())
+                    {
+                        Utf8JsonWriterExtensions.WriteNonEmptyArray(writer, nameof(Configuration.IntrinsicTypesToTreatEmptyStringAsNull), Configuration.IntrinsicTypesToTreatEmptyStringAsNull.ToList<string>());
+                    }
 
                     Configuration.MgmtConfiguration.SaveConfiguration(writer);
 
@@ -239,6 +244,10 @@ internal static void LoadConfiguration(string? projectPath, string outputPath, s
             var protocolMethods = Configuration.DeserializeArray(protocolMethodList);
             root.TryGetProperty(nameof(Configuration.Options.SuppressAbstractBaseClasses), out var suppressAbstractBaseClassesElement);
             var suppressAbstractBaseClasses = Configuration.DeserializeArray(suppressAbstractBaseClassesElement);
+            root.TryGetProperty(nameof(Configuration.Options.ModelsToTreatEmptyStringAsNull), out var modelsToTreatEmptyStringAsNullElement);
+            var modelsToTreatEmptyStringAsNull = Configuration.DeserializeArray(modelsToTreatEmptyStringAsNullElement);
+            root.TryGetProperty(nameof(Configuration.IntrinsicTypesToTreatEmptyStringAsNull), out var intrinsicTypesToTreatEmptyStringAsNullElement);
+            var intrinsicTypesToTreatEmptyStringAsNull = Configuration.DeserializeArray(intrinsicTypesToTreatEmptyStringAsNullElement);
             root.TryGetProperty(nameof(Configuration.Options.ModelFactoryForHlc), out var oldModelFactoryEntriesElement);
             var oldModelFactoryEntries = Configuration.DeserializeArray(oldModelFactoryEntriesElement);
 
@@ -263,6 +272,8 @@ internal static void LoadConfiguration(string? projectPath, string outputPath, s
                 projectPath ?? ReadStringOption(root, Configuration.Options.ProjectFolder),
                 protocolMethods,
                 suppressAbstractBaseClasses,
+                modelsToTreatEmptyStringAsNull,
+                intrinsicTypesToTreatEmptyStringAsNull,
                 MgmtConfiguration.LoadConfiguration(root),
                 MgmtTestConfiguration.LoadConfiguration(root)
             );

src/AutoRest.CSharp/Common/AutoRest/Plugins/Configuration.cs

@@ -7,6 +7,7 @@
 using System.Linq;
 using System.Text.Json;
 using AutoRest.CSharp.AutoRest.Communication;
+using Azure.Core;
 
 namespace AutoRest.CSharp.Input
 {
@@ -37,6 +38,8 @@ public static class Options
             public const string UnreferencedTypesHandling = "unreferenced-types-handling";
             public const string ModelFactoryForHlc = "model-factory-for-hlc";
             public const string GenerateModelFactory = "generate-model-factory";
+            public const string ModelsToTreatEmptyStringAsNull = "models-to-treat-empty-string-as-null";
+            public const string AdditionalIntrinsicTypesToTreatEmptyStringAsNull = "additional-intrinsic-types-to-treat-empty-string-as-null";
         }
 
         public enum UnreferencedTypesHandlingOption
@@ -67,6 +70,8 @@ public static void Initialize(
             string? projectFolder,
             IReadOnlyList<string> protocolMethodList,
             IReadOnlyList<string> suppressAbstractBaseClasses,
+            IReadOnlyList<string> modelsToTreatEmptyStringAsNull,
+            IReadOnlyList<string> additionalIntrinsicTypesToTreatEmptyStringAsNull,
             MgmtConfiguration mgmtConfiguration,
             MgmtTestConfiguration? mgmtTestConfiguration)
         {
@@ -123,6 +128,8 @@ public static void Initialize(
             _mgmtConfiguration = mgmtConfiguration;
             MgmtTestConfiguration = mgmtTestConfiguration;
             _suppressAbstractBaseClasses = suppressAbstractBaseClasses;
+            _modelsToTreatEmptyStringAsNull = new HashSet<string>(modelsToTreatEmptyStringAsNull);
+            _intrinsicTypesToTreatEmptyStringAsNull.UnionWith(additionalIntrinsicTypesToTreatEmptyStringAsNull);
         }
 
         private static string? _outputFolder;
@@ -152,6 +159,12 @@ public static void Initialize(
         private static IReadOnlyList<string>? _protocolMethodList;
         public static IReadOnlyList<string> ProtocolMethodList => _protocolMethodList ?? throw new InvalidOperationException("Configuration has not been initialized");
 
+        private static HashSet<string>? _modelsToTreatEmptyStringAsNull;
+        public static HashSet<string> ModelsToTreatEmptyStringAsNull => _modelsToTreatEmptyStringAsNull ?? throw new InvalidOperationException("Configuration has not been initialized");
+
+        private static HashSet<string> _intrinsicTypesToTreatEmptyStringAsNull = new HashSet<string>() { nameof(Uri), nameof(Guid), nameof(ResourceIdentifier), nameof(DateTimeOffset) };
+        public static HashSet<string> IntrinsicTypesToTreatEmptyStringAsNull => _intrinsicTypesToTreatEmptyStringAsNull;
+
         private static MgmtConfiguration? _mgmtConfiguration;
         public static MgmtConfiguration MgmtConfiguration => _mgmtConfiguration ?? throw new InvalidOperationException("Configuration has not been initialized");
 
@@ -185,6 +198,8 @@ public static void Initialize(IPluginCommunication autoRest)
                 projectFolder: autoRest.GetValue<string?>(Options.ProjectFolder).GetAwaiter().GetResult(),
                 protocolMethodList: autoRest.GetValue<string[]?>(Options.ProtocolMethodList).GetAwaiter().GetResult() ?? Array.Empty<string>(),
                 suppressAbstractBaseClasses: autoRest.GetValue<string[]?>(Options.SuppressAbstractBaseClasses).GetAwaiter().GetResult() ?? Array.Empty<string>(),
+                modelsToTreatEmptyStringAsNull: autoRest.GetValue<string[]?>(Options.ModelsToTreatEmptyStringAsNull).GetAwaiter().GetResult() ?? Array.Empty<string>(),
+                additionalIntrinsicTypesToTreatEmptyStringAsNull: autoRest.GetValue<string[]?>(Options.AdditionalIntrinsicTypesToTreatEmptyStringAsNull).GetAwaiter().GetResult() ?? Array.Empty<string>(),
                 mgmtConfiguration: MgmtConfiguration.GetConfiguration(autoRest),
                 mgmtTestConfiguration: MgmtTestConfiguration.GetConfiguration(autoRest)
             );

src/AutoRest.CSharp/Common/Generation/Writers/JsonCodeWriterExtensions.cs

@@ -306,7 +306,7 @@ private static void ToSerializeCall(this CodeWriter writer, FormattableString wr
             }
         }
 
-        private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnumerable<JsonPropertySerialization> propertySerializations, FormattableString itemVariable, Dictionary<JsonPropertySerialization, ObjectPropertyVariable> propertyVariables)
+        private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnumerable<JsonPropertySerialization> propertySerializations, FormattableString itemVariable, Dictionary<JsonPropertySerialization, ObjectPropertyVariable> propertyVariables, bool shouldTreatEmptyStringAsNull)
         {
             foreach (JsonPropertySerialization property in propertySerializations.Where(p => !p.ShouldSkipDeserialization))
             {
@@ -315,7 +315,8 @@ private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnu
                 {
                     if (property.ValueType?.IsNullable == true)
                     {
-                        using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null)"))
+                        var emptyStringCheck = GetEmptyStringCheckClause(property, itemVariable, shouldTreatEmptyStringAsNull);
+                        using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null{emptyStringCheck})"))
                         {
                             writer.Line($"{propertyVariables[property].Declaration} = null;");
                             writer.Append($"continue;");
@@ -326,17 +327,18 @@ private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnu
                              property.ValueType?.Equals(typeof(string)) !=
                              true) //https://github.com/Azure/autorest.csharp/issues/922
                     {
+                        var emptyStringCheck = GetEmptyStringCheckClause(property, itemVariable, shouldTreatEmptyStringAsNull);
                         if (Configuration.AzureArm && property.ValueType?.Equals(typeof(Uri)) == true)
                         {
-                            using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null)"))
+                            using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null{emptyStringCheck})"))
                             {
                                 writer.Line($"{propertyVariables[property].Declaration} = null;");
                                 writer.Append($"continue;");
                             }
                         }
                         else
                         {
-                            using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null)"))
+                            using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null{emptyStringCheck})"))
                             {
                                 writer.UseNamespace(typeof(JsonElementExtensions).Namespace!);
                                 writer.Line($"{itemVariable}.{nameof(JsonElementExtensions.ThrowNonNullablePropertyIsNull)}();");
@@ -357,7 +359,7 @@ private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnu
                         var nestedItemVariable = new CodeWriterDeclaration("property");
                         using (writer.Scope($"foreach (var {nestedItemVariable:D} in {itemVariable:I}.Value.EnumerateObject())"))
                         {
-                            writer.DeserializeIntoObjectProperties(property.PropertySerializations, $"{nestedItemVariable:I}", propertyVariables);
+                            writer.DeserializeIntoObjectProperties(property.PropertySerializations, $"{nestedItemVariable:I}", propertyVariables, shouldTreatEmptyStringAsNull);
                         }
                     }
                     else
@@ -370,6 +372,22 @@ private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnu
             }
         }
 
+        private static FormattableString GetEmptyStringCheckClause(JsonPropertySerialization property, FormattableString itemVariable, bool shouldTreatEmptyStringAsNull)
+        {
+            FormattableString result = $"";
+            if (!shouldTreatEmptyStringAsNull)
+                return result;
+            if (property.ValueSerialization is JsonValueSerialization valueSerialization
+                && valueSerialization.Type.IsFrameworkType)
+            {
+                if (Configuration.IntrinsicTypesToTreatEmptyStringAsNull.Contains(valueSerialization.Type.FrameworkType.Name))
+                {
+                    result = $" || {itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.{nameof(JsonValueKind.String)} && {itemVariable}.Value.GetString().Length == 0";
+                }
+            }
+            return result;
+        }
+
         private static FormattableString GetOptionalFormattable(JsonPropertySerialization target, ObjectPropertyVariable variable)
         {
             var targetType = target.PropertyType;
@@ -559,7 +577,7 @@ serialization.Discriminator.Value is not null &&
             var itemVariable = new CodeWriterDeclaration("property");
             using (writer.Scope($"foreach (var {itemVariable:D} in element.EnumerateObject())"))
             {
-                writer.DeserializeIntoObjectProperties(serialization.Properties, $"{itemVariable:I}", propertyVariables);
+                writer.DeserializeIntoObjectProperties(serialization.Properties, $"{itemVariable:I}", propertyVariables, Configuration.ModelsToTreatEmptyStringAsNull.Contains(serialization.Type.Name));
 
                 if (objAdditionalProperties?.ValueSerialization != null)
                 {

src/CADL.Extension/Emitter.Csharp/src/emitter.ts

@@ -166,7 +166,28 @@ export async function $onEmit(context: EmitContext<NetEmitterOptions>) {
                 SingleTopLevelClient: options["single-top-level-client"],
                 "unreferenced-types-handling":
                     options["unreferenced-types-handling"],
-                "model-namespace": options["model-namespace"]
+                "model-namespace": options["model-namespace"],
+                ModelsToTreatEmptyStringAsNull:
+                    options["models-to-treat-empty-string-as-null"],
+                IntrinsicTypesToTreatEmptyStringAsNull: options[
+                    "models-to-treat-empty-string-as-null"
+                ]
+                    ? options[
+                          "additional-intrinsic-types-to-treat-empty-string-as-null"
+                      ].concat(
+                          [
+                              "Uri",
+                              "Guid",
+                              "ResourceIdentifier",
+                              "DateTimeOffset"
+                          ].filter(
+                              (item) =>
+                                  options[
+                                      "additional-intrinsic-types-to-treat-empty-string-as-null"
+                                  ].indexOf(item) < 0
+                          )
+                      )
+                    : undefined
             } as Configuration;
 
             await program.host.writeFile(

src/CADL.Extension/Emitter.Csharp/src/options.ts

@@ -19,6 +19,8 @@ export type NetEmitterOptions = {
     "save-inputs"?: boolean;
     "model-namespace"?: boolean;
     debug?: boolean;
+    "models-to-treat-empty-string-as-null"?: string[];
+    "additional-intrinsic-types-to-treat-empty-string-as-null"?: string[];
 } & DpgEmitterOptions;
 
 export const NetEmitterOptionsSchema: JSONSchemaType<NetEmitterOptions> = {
@@ -48,7 +50,17 @@ export const NetEmitterOptionsSchema: JSONSchemaType<NetEmitterOptions> = {
         "generate-protocol-methods": { type: "boolean", nullable: true },
         "generate-convenience-methods": { type: "boolean", nullable: true },
         "package-name": { type: "string", nullable: true },
-        debug: { type: "boolean", nullable: true }
+        debug: { type: "boolean", nullable: true },
+        "models-to-treat-empty-string-as-null": {
+            type: "array",
+            nullable: true,
+            items: { type: "string" }
+        },
+        "additional-intrinsic-types-to-treat-empty-string-as-null": {
+            type: "array",
+            nullable: true,
+            items: { type: "string" }
+        }
     },
     required: []
 };
@@ -64,7 +76,9 @@ const defaultOptions = {
     "generate-protocol-methods": true,
     "generate-convenience-methods": true,
     "package-name": undefined,
-    debug: undefined
+    debug: undefined,
+    "models-to-treat-empty-string-as-null": undefined,
+    "additional-intrinsic-types-to-treat-empty-string-as-null": []
 };
 
 export function resolveOptions(context: EmitContext<NetEmitterOptions>) {

src/CADL.Extension/Emitter.Csharp/src/type/Configuration.ts

@@ -12,4 +12,6 @@ export interface Configuration {
         | "internalize"
         | "keepAll";
     "model-namespace"?: boolean;
+    "models-to-treat-empty-string-as-null"?: string[];
+    "additional-intrinsic-types-to-treat-empty-string-as-null"?: string[];
 }

test/AutoRest.TestServer.Tests/Mgmt/Unit/MgmtRestOperationTests.cs

@@ -70,6 +70,8 @@ public void Setup()
                 projectFolder: "/..",
                 protocolMethodList: Array.Empty<string>(),
                 suppressAbstractBaseClasses: Array.Empty<string>(),
+                modelsToTreatEmptyStringAsNull: Array.Empty<string>(),
+                additionalIntrinsicTypesToTreatEmptyStringAsNull: Array.Empty<string>(),
                 mgmtConfiguration: mgmtConfiguration,
                 mgmtTestConfiguration: null);
         }

test/AutoRest.TestServerLowLevel.Tests/LowLevel/Generation/ModelGenerationTestBase.cs

@@ -55,6 +55,8 @@ public void init()
                 projectFolder: ".",
                 protocolMethodList: Array.Empty<string>(),
                 suppressAbstractBaseClasses: Array.Empty<string>(),
+                modelsToTreatEmptyStringAsNull: Array.Empty<string>(),
+                additionalIntrinsicTypesToTreatEmptyStringAsNull: Array.Empty<string>(),
                 mgmtConfiguration: null,
                 mgmtTestConfiguration: null);
         }

test/CadlRanchProjects/models/property-optional/Generated/Configuration.json

@@ -43,7 +43,7 @@ internal static DatetimeProperty DeserializeDatetimeProperty(JsonElement element
             {
                 if (property0.NameEquals("property"u8))
                 {
-                    if (property0.Value.ValueKind == JsonValueKind.Null)
+                    if (property0.Value.ValueKind == JsonValueKind.Null || property0.Value.ValueKind == JsonValueKind.String && property0.Value.GetString().Length == 0)
                     {
                         property = null;
                         continue;