Merge pull request #1094 from Accenture/develop

Release 5.3

Author: JoernBerkefeld

.github/ISSUE_TEMPLATE/bug.yml

@@ -39,6 +39,7 @@ body:
           label: Version
           description: What version of our software are you running? (mcdev --version)
           options:
+              - 5.3.0
               - 5.2.0
               - 5.1.0
               - 5.0.2

docs/dist/documentation.md

@@ -38,6 +38,7 @@ const MetadataTypeDefinitions = {
     transactionalSMS: require('./metadataTypes/definitions/TransactionalSMS.definition'),
     triggeredSend: require('./metadataTypes/definitions/TriggeredSend.definition'),
     user: require('./metadataTypes/definitions/User.definition'),
+    verification: require('./metadataTypes/definitions/Verification.definition'),
 };
 
 module.exports = MetadataTypeDefinitions;

lib/MetadataTypeDefinitions.js

@@ -38,6 +38,7 @@ const MetadataTypeInfo = {
     transactionalSMS: require('./metadataTypes/TransactionalSMS'),
     triggeredSend: require('./metadataTypes/TriggeredSend'),
     user: require('./metadataTypes/User'),
+    verification: require('./metadataTypes/Verification'),
 };
 
 module.exports = MetadataTypeInfo;

lib/MetadataTypeInfo.js

@@ -45,7 +45,7 @@ yargs
         },
     })
     .command({
-        command: 'deploy [BU] [TYPE] [KEY] [--fromRetrieve] [--refresh]',
+        command: 'deploy [BU] [TYPE] [KEY]',
         aliases: ['d'],
         desc: 'deploys local metadata to a business unit',
         builder: (yargs) => {
@@ -97,6 +97,11 @@ yargs
                     group: 'Options for deploy:',
                     describe:
                         'optionally start existing schedule instead of running item once immediately (only works for automations)',
+                })
+                .option('fixShared', {
+                    group: 'Options for deploy:',
+                    describe:
+                        "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)",
                 });
         },
         handler: (argv) => {

lib/cli.js

@@ -56,12 +56,14 @@ class Mcdev {
             'commitHistory',
             'execute',
             'filter',
+            'fixShared',
             'fromRetrieve',
             'json',
             'like',
             'noLogColors',
             'noLogFile',
             'refresh',
+            '_runningTest',
             'schedule',
             'skipInteraction',
         ];
@@ -310,7 +312,8 @@ class Mcdev {
                 triggeredSend: 'triggeredSendDefinition',
                 user: 'accountUser',
             };
-            Util.logger.info(`:: Retrieving ${cred}/${bu}\n`);
+            Util.logger.info('');
+            Util.logger.info(`:: Retrieving ${cred}/${bu}`);
             const retrieveTypesArr = [];
             if (selectedTypesArr) {
                 for (const selectedType of Array.isArray(selectedTypesArr)

lib/index.js

@@ -31,13 +31,7 @@ class AttributeSet extends MetadataType {
             const result = await AttributeGroup.retrieveForCache();
             cache.setMetadata('attributeGroup', result.metadata);
         }
-        return super.retrieveREST(
-            retrieveDir,
-            '/hub/v1/contacts/schema/setDefinitions',
-            null,
-            null,
-            key
-        );
+        return super.retrieveREST(retrieveDir, '/hub/v1/contacts/schema/setDefinitions', null, key);
     }
     /**
      * Retrieves Metadata of schema set definitions for caching.
@@ -47,6 +41,110 @@ class AttributeSet extends MetadataType {
     static retrieveForCache() {
         return super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
     }
+    /**
+     * used to identify updated shared data extensions that are used in attributeSets.
+     * helper for DataExtension.#fixShared_onBU
+     *
+     * @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
+     * @param {object} fixShared_fields DataExtensionField.fixShared_fields
+     * @returns {Promise.<string[]>} Promise of list of shared dataExtension IDs
+     */
+    static async fixShared_retrieve(sharedDataExtensionMap, fixShared_fields) {
+        if (!Object.keys(sharedDataExtensionMap).length) {
+            return [];
+        }
+        const result = await super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
+        const metadataMap = result?.metadata;
+        if (metadataMap && Object.keys(metadataMap).length) {
+            const sharedDeIds = Object.keys(metadataMap)
+                .filter(
+                    (asKey) =>
+                        metadataMap[asKey].storageLogicalType === 'ExactTargetSchema' ||
+                        metadataMap[asKey].storageLogicalType === 'DataExtension'
+                )
+                .filter((asKey) => {
+                    // check if dataExtension ID is found on any attributeSet of this BU
+                    if (sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]) {
+                        Util.logger.debug(
+                            ` shared dataExtension ID ${metadataMap[asKey].storageReferenceID.value} found in attributeSet ${asKey}`
+                        );
+                        return true;
+                    } else {
+                        return false;
+                    }
+                })
+                .filter((asKey) => {
+                    // check if any of the dataExtension fields dont exist on the attributeSet or are out of date
+                    const deKey =
+                        sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value];
+                    const asFields = metadataMap[asKey].valueDefinitions;
+                    const deFields = Object.values(fixShared_fields[deKey]);
+                    return deFields.some((deField) => {
+                        const search = asFields.filter((asf) => asf.name === deField.Name);
+                        if (!search.length) {
+                            Util.logger.debug(
+                                Util.getGrayMsg(
+                                    ` - Field ${deField.Name} not found in attributeSet; Note: only first recognized difference is printed to log`
+                                )
+                            );
+                            return true;
+                        }
+                        const asField = search[0];
+                        if (asField.dataType !== deField.FieldType) {
+                            Util.logger.debug(
+                                Util.getGrayMsg(
+                                    ` - Field ${deField.Name} FieldType changed (old: ${asField.dataType}; new: ${deField.FieldType}); Note: only first recognized difference is printed to log`
+                                )
+                            );
+                            return true;
+                        }
+                        asField.defaultValue ||= '';
+                        if (
+                            (asField.defaultValue && deField.DefaultValue === '') ||
+                            (deField.FieldType === 'Boolean' &&
+                                deField.DefaultValue !== '' &&
+                                (deField.DefaultValue
+                                    ? 'True'
+                                    : 'False' !== asField.defaultValue)) ||
+                            (deField.FieldType !== 'Boolean' &&
+                                deField.DefaultValue !== asField.defaultValue)
+                        ) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} DefaultValue changed (old: ${asField.defaultValue}; new: ${deField.DefaultValue}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        // some field types don't carry the length property. reset to 0 to ease comparison
+                        asField.length ||= 0;
+                        if (asField.length !== deField.MaxLength) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} MaxLength changed (old: ${asField.length}; new: ${deField.MaxLength}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        if (asField.isNullable !== deField.IsRequired) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} IsRequired changed (old: ${asField.isNullable}; new: ${deField.IsRequired}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        if (asField.isPrimaryKey !== deField.IsPrimaryKey) {
+                            Util.logger.debug(
+                                ` - Field ${deField.Name} IsPrimaryKey changed (old: ${asField.isPrimaryKey}; new: ${deField.IsPrimaryKey}); Note: only first recognized difference is printed to log`
+                            );
+                            return true;
+                        }
+                        return false;
+                    });
+                })
+                .map((key) => metadataMap[key].storageReferenceID.value)
+                .filter(Boolean);
+            return sharedDeIds;
+        } else {
+            // nothing to do - return empty array
+            return [];
+        }
+    }
 
     /**
      * Builds map of metadata entries mapped to their keyfields
@@ -83,7 +181,7 @@ class AttributeSet extends MetadataType {
         switch (metadata.storageLogicalType) {
             case 'ExactTargetSchema': // synced / shared DEs
             case 'DataExtension': {
-                // local DEs
+                // shared / local DEs
                 try {
                     metadata.r__dataExtension_CustomerKey = cache.searchForField(
                         'dataExtension',
@@ -226,6 +324,12 @@ class AttributeSet extends MetadataType {
         // Member ID
         delete metadata.customObjectOwnerMID;
 
+        // remove duplicate ID fields (main field is definitionID)
+        delete metadata.setDefinitionID;
+        if (metadata.dataRetentionProperties?.setDefinitionID) {
+            delete metadata.dataRetentionProperties?.setDefinitionID;
+        }
+
         // connectingID.identifierType seems to be always set to 'FullyQualifiedName' - to be sure we check it here and remove it if it's the case
         if (metadata.connectingID?.identifierType === 'FullyQualifiedName') {
             // remove useless field
@@ -241,16 +345,19 @@ class AttributeSet extends MetadataType {
      * @returns {object[]} all system value definitions
      */
     static _getSystemValueDefinitions() {
-        if (!this.systemValueDefinitions) {
-            this.systemValueDefinitions = Object.values(cache.getCache()['attributeSet'])
+        this.systemValueDefinitions ||= {};
+        if (!this.systemValueDefinitions[this.buObject.mid]) {
+            this.systemValueDefinitions[this.buObject.mid] = Object.values(
+                cache.getCache()['attributeSet']
+            )
                 .flatMap((item) => {
                     if (item.isSystemDefined) {
                         return item.valueDefinitions;
                     }
                 })
                 .filter(Boolean);
         }
-        return this.systemValueDefinitions;
+        return this.systemValueDefinitions[this.buObject.mid];
     }
 }