NIFI-15513 Added unit tests for property migration of processors and controller services in extended bundles e-g, snowflake, social-media, splunk, update-attribute, websocket, windows-event-log, workday and zendesk.

nifi-extension-bundles/nifi-dropbox-bundle/nifi-dropbox-processors/src/test/java/org/apache/nifi/processors/dropbox/ListDropboxTest.java

@@ -28,10 +28,14 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.migration.ProxyServiceMigration;
 import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.util.list.AbstractListProcessor;
+import org.apache.nifi.processor.util.list.ListedEntityTracker;
 import org.apache.nifi.reporting.InitializationException;
 import org.apache.nifi.serialization.RecordSetWriterFactory;
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
 import org.apache.nifi.util.TestRunners;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -43,6 +47,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.Spliterator;
 import java.util.stream.StreamSupport;
 
@@ -201,6 +207,32 @@ void testRecordWriter() throws Exception {
         assertEquals(expectedFileNames, actualFileNames);
     }
 
+    @Test
+    void testMigrateProperties() {
+        final Map<String, String> expectedRenamed = Map.ofEntries(
+                Map.entry(DropboxTrait.OLD_CREDENTIAL_SERVICE_PROPERTY_NAME, DropboxTrait.CREDENTIAL_SERVICE.getName()),
+                Map.entry("folder", ListDropbox.FOLDER.getName()),
+                Map.entry("recursive-search", ListDropbox.RECURSIVE_SEARCH.getName()),
+                Map.entry("min-age", ListDropbox.MIN_AGE.getName()),
+                Map.entry(ListedEntityTracker.OLD_TRACKING_STATE_CACHE_PROPERTY_NAME, ListDropbox.TRACKING_STATE_CACHE.getName()),
+                Map.entry(ListedEntityTracker.OLD_TRACKING_TIME_WINDOW_PROPERTY_NAME, ListDropbox.TRACKING_TIME_WINDOW.getName()),
+                Map.entry(ListedEntityTracker.OLD_INITIAL_LISTING_TARGET_PROPERTY_NAME, ListDropbox.INITIAL_LISTING_TARGET.getName()),
+                Map.entry("target-system-timestamp-precision", AbstractListProcessor.TARGET_SYSTEM_TIMESTAMP_PRECISION.getName()),
+                Map.entry("listing-strategy", AbstractListProcessor.LISTING_STRATEGY.getName()),
+                Map.entry("record-writer", AbstractListProcessor.RECORD_WRITER.getName()),
+                Map.entry(ProxyServiceMigration.OBSOLETE_PROXY_CONFIGURATION_SERVICE, ProxyServiceMigration.PROXY_CONFIGURATION_SERVICE)
+        );
+
+        final PropertyMigrationResult propertyMigrationResult = testRunner.migrateProperties();
+        assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
+
+        final Set<String> expectedRemoved = Set.of(
+                "Distributed Cache Service"
+        );
+
+        assertEquals(expectedRemoved, propertyMigrationResult.getPropertiesRemoved());
+    }
+
     private Metadata createFolderMetadata() {
         return FolderMetadata.newBuilder(FOLDER_ID)
                 .withPathDisplay(TEST_FOLDER + "/" + FOLDER_ID)

nifi-extension-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/TestConsumeEmail.java

@@ -26,6 +26,7 @@
 import jakarta.mail.internet.InternetAddress;
 import jakarta.mail.internet.MimeMessage;
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
 import org.junit.jupiter.api.AfterEach;
@@ -34,6 +35,7 @@
 import org.springframework.integration.mail.inbound.AbstractMailReceiver;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -151,4 +153,24 @@ public void validateProtocol() {
 
         assertEquals("pop3", consume.getProtocol(runner.getProcessContext()));
     }
+
+    @Test
+    void testMigrateProperties() {
+        final TestRunner runner = TestRunners.newTestRunner(ConsumeIMAP.class);
+        final Map<String, String> expectedRenamed = Map.ofEntries(
+                Map.entry("host", AbstractEmailProcessor.HOST.getName()),
+                Map.entry("port", AbstractEmailProcessor.PORT.getName()),
+                Map.entry("authorization-mode", AbstractEmailProcessor.AUTHORIZATION_MODE.getName()),
+                Map.entry("oauth2-access-token-provider", AbstractEmailProcessor.OAUTH2_ACCESS_TOKEN_PROVIDER.getName()),
+                Map.entry("user", AbstractEmailProcessor.USER.getName()),
+                Map.entry("password", AbstractEmailProcessor.PASSWORD.getName()),
+                Map.entry("folder", AbstractEmailProcessor.FOLDER.getName()),
+                Map.entry("fetch.size", AbstractEmailProcessor.FETCH_SIZE.getName()),
+                Map.entry("delete.messages", AbstractEmailProcessor.SHOULD_DELETE_MESSAGES.getName()),
+                Map.entry("connection.timeout", AbstractEmailProcessor.CONNECTION_TIMEOUT.getName())
+        );
+
+        final PropertyMigrationResult propertyMigrationResult = runner.migrateProperties();
+        assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
+    }
 }

nifi-extension-bundles/nifi-email-bundle/nifi-email-processors/src/test/java/org/apache/nifi/processors/email/TestExtractEmailHeaders.java

@@ -18,11 +18,16 @@
 package org.apache.nifi.processors.email;
 
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestExtractEmailHeaders {
     String from = "Alice <[email protected]>";
@@ -32,11 +37,15 @@ public class TestExtractEmailHeaders {
     String hostName = "bermudatriangle";
 
     GenerateAttachment attachmentGenerator = new GenerateAttachment(from, to, subject, message, hostName);
+    private TestRunner runner;
+
+    @BeforeEach
+    void setUp() {
+        runner = TestRunners.newTestRunner(ExtractEmailHeaders.class);
+    }
 
     @Test
     public void testValidEmailWithAttachments() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
-
         byte[] withAttachment = attachmentGenerator.withAttachments(1);
 
         runner.enqueue(withAttachment);
@@ -47,15 +56,14 @@ public void testValidEmailWithAttachments() {
 
         runner.assertQueueEmpty();
         final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(ExtractEmailHeaders.REL_SUCCESS);
-        splits.get(0).assertAttributeEquals("email.headers.from.0", from);
-        splits.get(0).assertAttributeEquals("email.headers.to.0", to);
-        splits.get(0).assertAttributeEquals("email.headers.subject", subject);
-        splits.get(0).assertAttributeEquals("email.attachment_count", "1");
+        splits.getFirst().assertAttributeEquals("email.headers.from.0", from);
+        splits.getFirst().assertAttributeEquals("email.headers.to.0", to);
+        splits.getFirst().assertAttributeEquals("email.headers.subject", subject);
+        splits.getFirst().assertAttributeEquals("email.attachment_count", "1");
     }
 
     @Test
     public void testValidEmailWithoutAttachments() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
         runner.setProperty(ExtractEmailHeaders.CAPTURED_HEADERS, "MIME-Version");
 
         byte[] simpleEmail = attachmentGenerator.simpleMessage(to);
@@ -69,10 +77,10 @@ public void testValidEmailWithoutAttachments() {
 
         runner.assertQueueEmpty();
         final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(ExtractEmailHeaders.REL_SUCCESS);
-        splits.get(0).assertAttributeEquals("email.headers.from.0", from);
-        splits.get(0).assertAttributeEquals("email.headers.to.0", to);
-        splits.get(0).assertAttributeEquals("email.attachment_count", "0");
-        splits.get(0).assertAttributeExists("email.headers.mime-version");
+        splits.getFirst().assertAttributeEquals("email.headers.from.0", from);
+        splits.getFirst().assertAttributeEquals("email.headers.to.0", to);
+        splits.getFirst().assertAttributeEquals("email.attachment_count", "0");
+        splits.getFirst().assertAttributeExists("email.headers.mime-version");
     }
 
     /**
@@ -82,7 +90,6 @@ public void testValidEmailWithoutAttachments() {
      */
     @Test
     public void testValidEmailWithNoRecipients() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
         runner.setProperty(ExtractEmailHeaders.CAPTURED_HEADERS, "MIME-Version");
 
         final byte[] message = attachmentGenerator.simpleMessage();
@@ -94,11 +101,11 @@ public void testValidEmailWithNoRecipients() {
 
         runner.assertQueueEmpty();
         final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(ExtractEmailHeaders.REL_SUCCESS);
-        splits.get(0).assertAttributeEquals("email.headers.from.0", from);
-        splits.get(0).assertAttributeExists("email.headers.mime-version");
-        splits.get(0).assertAttributeNotExists("email.headers.to");
-        splits.get(0).assertAttributeNotExists("email.headers.cc");
-        splits.get(0).assertAttributeNotExists("email.headers.bcc");
+        splits.getFirst().assertAttributeEquals("email.headers.from.0", from);
+        splits.getFirst().assertAttributeExists("email.headers.mime-version");
+        splits.getFirst().assertAttributeNotExists("email.headers.to");
+        splits.getFirst().assertAttributeNotExists("email.headers.cc");
+        splits.getFirst().assertAttributeNotExists("email.headers.bcc");
     }
 
     /**
@@ -109,7 +116,6 @@ public void testValidEmailWithNoRecipients() {
      */
     @Test
     public void testNonStrictParsingPassesForInvalidAddresses() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
         runner.setProperty(ExtractEmailHeaders.STRICT_PARSING, "false");
 
         final byte[] message = attachmentGenerator.simpleMessage("<>, Joe, \"\" <>");
@@ -123,9 +129,9 @@ public void testNonStrictParsingPassesForInvalidAddresses() {
         runner.assertQueueEmpty();
         final List<MockFlowFile> splits = runner.getFlowFilesForRelationship(ExtractEmailHeaders.REL_SUCCESS);
 
-        splits.get(0).assertAttributeEquals("email.headers.to.0", "");
-        splits.get(0).assertAttributeEquals("email.headers.to.1", "Joe");
-        splits.get(0).assertAttributeEquals("email.headers.to.2", "");
+        splits.getFirst().assertAttributeEquals("email.headers.to.0", "");
+        splits.getFirst().assertAttributeEquals("email.headers.to.1", "Joe");
+        splits.getFirst().assertAttributeEquals("email.headers.to.2", "");
     }
 
     /**
@@ -136,7 +142,6 @@ public void testNonStrictParsingPassesForInvalidAddresses() {
      */
     @Test
     public void testStrictParsingFailsForInvalidAddresses() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
         runner.setProperty(ExtractEmailHeaders.STRICT_PARSING, "true");
 
         final byte[] message = attachmentGenerator.simpleMessage("<>, Joe, \"\" <>");
@@ -150,11 +155,21 @@ public void testStrictParsingFailsForInvalidAddresses() {
 
     @Test
     public void testInvalidEmail() {
-        final TestRunner runner = TestRunners.newTestRunner(new ExtractEmailHeaders());
         runner.enqueue("test test test chocolate".getBytes());
         runner.run();
 
         runner.assertTransferCount(ExtractEmailHeaders.REL_SUCCESS, 0);
         runner.assertTransferCount(ExtractEmailHeaders.REL_FAILURE, 1);
     }
+
+    @Test
+    void testMigrateProperties() {
+        final Map<String, String> expectedRenamed = Map.ofEntries(
+                Map.entry("CAPTURED_HEADERS", ExtractEmailHeaders.CAPTURED_HEADERS.getName()),
+                Map.entry("STRICT_ADDRESS_PARSING", ExtractEmailHeaders.STRICT_PARSING.getName())
+        );
+
+        final PropertyMigrationResult propertyMigrationResult = runner.migrateProperties();
+        assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
+    }
 }

nifi-extension-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/TestGeoEnrichIPRecord.java

@@ -17,11 +17,11 @@
 
 package org.apache.nifi.processors;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.maxmind.geoip2.DatabaseReader;
 import com.maxmind.geoip2.model.CityResponse;
 import org.apache.avro.Schema;
-import org.apache.commons.io.IOUtils;
 import org.apache.nifi.annotation.lifecycle.OnScheduled;
 import org.apache.nifi.avro.AvroTypeUtil;
 import org.apache.nifi.components.PropertyDescriptor;
@@ -35,6 +35,7 @@
 import org.apache.nifi.serialization.record.MockSchemaRegistry;
 import org.apache.nifi.serialization.record.RecordSchema;
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
 import org.junit.jupiter.api.BeforeEach;
@@ -43,9 +44,9 @@
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
-import java.io.InputStream;
 import java.net.InetAddress;
-import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -70,20 +71,15 @@ public void setup() throws Exception {
         runner = TestRunners.newTestRunner(new TestableGeoEnrichIPRecord());
         ControllerService reader = new JsonTreeReader();
         ControllerService writer = new JsonRecordSetWriter();
-        ControllerService registry = new MockSchemaRegistry();
+        MockSchemaRegistry registry = new MockSchemaRegistry();
         runner.addControllerService("reader", reader);
         runner.addControllerService("writer", writer);
         runner.addControllerService("registry", registry);
 
 
-        try (InputStream is = getClass().getResourceAsStream("/avro/record_schema.avsc")) {
-            String raw = IOUtils.toString(is, StandardCharsets.UTF_8);
-            RecordSchema parsed = AvroTypeUtil.createSchema(new Schema.Parser().parse(raw));
-            ((MockSchemaRegistry) registry).addSchema("record", parsed);
-
-        } catch (Exception ex) {
-            throw new RuntimeException(ex);
-        }
+        final String raw = Files.readString(Paths.get("src/test/resources/avro/record_schema.avsc"));
+        RecordSchema parsed = AvroTypeUtil.createSchema(new Schema.Parser().parse(raw));
+        registry.addSchema("record", parsed);
 
         runner.setProperty(reader, SchemaAccessUtils.SCHEMA_REGISTRY, "registry");
         runner.setProperty(writer, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_NAME_PROPERTY);
@@ -155,7 +151,7 @@ public void testEnrichSendToFound() throws Exception {
         byte[] raw = runner.getContentAsByteArray(ff);
         String content = new String(raw);
         ObjectMapper mapper = new ObjectMapper();
-        List<Map<String, Object>> result = mapper.readValue(content, List.class);
+        List<Map<String, Object>> result = mapper.readValue(content, new TypeReference<>() { });
 
         assertNotNull(result);
         assertEquals(1, result.size());
@@ -172,6 +168,27 @@ public void testEnrichSendToFound() throws Exception {
         assertNotNull(geo.get("lon"));
     }
 
+    @Test
+    void testMigrateProperties() {
+        final TestRunner runner = TestRunners.newTestRunner(GeoEnrichIPRecord.class);
+        final Map<String, String> expectedRenamed = Map.ofEntries(
+                Map.entry("geo-enrich-ip-record-reader", GeoEnrichIPRecord.READER.getName()),
+                Map.entry("geo-enrich-ip-record-writer", GeoEnrichIPRecord.WRITER.getName()),
+                Map.entry("geo-enrich-ip-ip-record-path", GeoEnrichIPRecord.IP_RECORD_PATH.getName()),
+                Map.entry("geo-enrich-ip-split-found-not-found", GeoEnrichIPRecord.SPLIT_FOUND_NOT_FOUND.getName()),
+                Map.entry("geo-enrich-ip-city-record-path", GeoEnrichIPRecord.GEO_CITY.getName()),
+                Map.entry("geo-enrich-ip-latitude-record-path", GeoEnrichIPRecord.GEO_LATITUDE.getName()),
+                Map.entry("geo-enrich-ip-longitude-record-path", GeoEnrichIPRecord.GEO_LONGITUDE.getName()),
+                Map.entry("geo-enrich-ip-country-record-path", GeoEnrichIPRecord.GEO_COUNTRY.getName()),
+                Map.entry("geo-enrich-ip-country-iso-record-path", GeoEnrichIPRecord.GEO_COUNTRY_ISO.getName()),
+                Map.entry("geo-enrich-ip-country-postal-record-path", GeoEnrichIPRecord.GEO_POSTAL_CODE.getName()),
+                Map.entry("Geo Database File", AbstractEnrichIP.GEO_DATABASE_FILE.getName())
+        );
+
+        final PropertyMigrationResult propertyMigrationResult = runner.migrateProperties();
+        assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
+    }
+
     class TestableGeoEnrichIPRecord extends GeoEnrichIPRecord {
 
         @Override
@@ -190,6 +207,7 @@ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
                     LOG_LEVEL
             );
         }
+
         @Override
         @OnScheduled
         public void onScheduled(ProcessContext context) {
@@ -198,6 +216,7 @@ public void onScheduled(ProcessContext context) {
             writerFactory = context.getProperty(WRITER).asControllerService(RecordSetWriterFactory.class);
             splitOutput = context.getProperty(SPLIT_FOUND_NOT_FOUND).asBoolean();
         }
+
         @Override
         protected void loadDatabaseFile() {
             //  Do nothing, the mock database reader is used

nifi-extension-bundles/nifi-evtx-bundle/nifi-evtx-processors/src/test/java/org/apache/nifi/processors/evtx/ParseEvtxTest.java

@@ -29,6 +29,7 @@
 import org.apache.nifi.processors.evtx.parser.Record;
 import org.apache.nifi.processors.evtx.parser.bxml.RootNode;
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
 import org.apache.nifi.util.TestRunner;
 import org.apache.nifi.util.TestRunners;
 import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
@@ -459,6 +460,17 @@ public void testChunkBasedParseCorrectNumberOfFlowFilesFromAResizedFile() {
         testValidEvents(ParseEvtx.CHUNK, "3778_events_not_exported.evtx", 16);
     }
 
+    @Test
+    void testMigrateProperties() {
+        final TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
+        final Map<String, String> expectedRenamed = Map.of(
+                "granularity", ParseEvtx.GRANULARITY.getName()
+        );
+
+        final PropertyMigrationResult propertyMigrationResult = testRunner.migrateProperties();
+        assertEquals(expectedRenamed, propertyMigrationResult.getPropertiesRenamed());
+    }
+
     private void testValidEvents(String granularity, String filename, int expectedCount) {
         TestRunner testRunner = TestRunners.newTestRunner(ParseEvtx.class);
         testRunner.setProperty(ParseEvtx.GRANULARITY, granularity);