From 748dd3424ccd6fd898f939e1a4f1ae389402a19a Mon Sep 17 00:00:00 2001 From: Krish-cloudsufi Date: Thu, 14 Aug 2025 07:37:49 +0000 Subject: [PATCH 1/3] Changed guava scope --- pom.xml | 43 ++++++++++++++++++- .../etl/BaseSalesforceBatchSourceETLTest.java | 3 +- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 85b62902..e9da62ca 100644 --- a/pom.xml +++ b/pom.xml @@ -90,12 +90,24 @@ + ch.qos.logback logback-classic 1.2.13 + + com.google.guava + guava + 13.0.1 + test + + + com.google.guava + guava + 32.1.3-jre + io.cdap.cdap cdap-api @@ -117,6 +129,12 @@ io.cdap.plugin hydrator-common ${hydrator.version} + + + com.google.guava + guava + + org.apache.hadoop @@ -321,6 +339,14 @@ ${cdap.version} test + + com.google.guava + guava + + + io.cdap.cdap + cdap-formats + org.apache.twill twill-core @@ -370,6 +396,12 @@ cdap-data-pipeline2_2.11 ${cdap.version} test + + + com.google.guava + guava + + org.apache.avro @@ -412,6 +444,12 @@ cdap-data-streams2_2.11 ${cdap.version} test + + + com.google.guava + guava + + @@ -755,11 +793,12 @@ + com.google.guava guava - 31.0.1-jre - compile + 13.0.1 + test diff --git a/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSourceETLTest.java b/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSourceETLTest.java index 2eed46fa..72beab46 100644 --- a/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSourceETLTest.java +++ b/src/test/java/io/cdap/plugin/salesforce/etl/BaseSalesforceBatchSourceETLTest.java @@ -67,7 +67,8 @@ /** * {@inheritDoc} */ -public abstract class BaseSalesforceBatchSourceETLTest extends BaseSalesforceETLTest { +public abstract class +BaseSalesforceBatchSourceETLTest extends BaseSalesforceETLTest { @ClassRule public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false); From d175fd08010e181fc523158b23a41e6e3f086067 Mon Sep 17 00:00:00 2001 From: Krish-cloudsufi Date: Wed, 20 Aug 2025 11:48:23 +0000 Subject: [PATCH 2/3] added guava changes --- pom.xml | 31 +++ .../SalesforceBatchSinkSchemaValidation.java | 182 ------------------ 2 files changed, 31 insertions(+), 182 deletions(-) delete mode 100644 src/test/java/io/cdap/plugin/salesforce/etl/SalesforceBatchSinkSchemaValidation.java diff --git a/pom.xml b/pom.xml index e9da62ca..96105da9 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ + ch.qos.logback @@ -519,6 +520,36 @@ ${testSourceLocation} + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + shade-old-guava + + test-compile + + shade + + + + + + com.google.guava:guava:31.1-jre + + + + + com.google.common + shadow.guava31.com.google.common + + + false + + + + io.cdap cdap-maven-plugin diff --git a/src/test/java/io/cdap/plugin/salesforce/etl/SalesforceBatchSinkSchemaValidation.java b/src/test/java/io/cdap/plugin/salesforce/etl/SalesforceBatchSinkSchemaValidation.java deleted file mode 100644 index 8678830e..00000000 --- a/src/test/java/io/cdap/plugin/salesforce/etl/SalesforceBatchSinkSchemaValidation.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright © 2023 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package io.cdap.plugin.salesforce.etl; - -import com.sforce.async.ConcurrencyMode; -import com.sforce.soap.partner.Field; -import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.etl.mock.validation.MockFailureCollector; -import io.cdap.plugin.salesforce.SObjectDescriptor; -import io.cdap.plugin.salesforce.SObjectsDescribeResult; -import io.cdap.plugin.salesforce.SalesforceConnectionUtil; -import io.cdap.plugin.salesforce.SalesforceFunctionType; -import io.cdap.plugin.salesforce.SalesforceSchemaUtil; -import io.cdap.plugin.salesforce.authenticator.AuthenticatorCredentials; -import io.cdap.plugin.salesforce.plugin.OAuthInfo; -import io.cdap.plugin.salesforce.plugin.SalesforceConnectorInfo; -import io.cdap.plugin.salesforce.plugin.connector.SalesforceConnectorConfig; -import io.cdap.plugin.salesforce.plugin.sink.batch.SalesforceSinkConfig; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.util.ArrayList; -import java.util.List; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({SalesforceConnectorConfig.class, SalesforceConnectionUtil.class, SObjectDescriptor.class, - SObjectsDescribeResult.class}) -public class SalesforceBatchSinkSchemaValidation { - private static final String REFERENCE_NAME = "SalesforceBatchSink-output"; - @Mock - SObjectDescriptor mockDescriptor; - @Mock - SObjectsDescribeResult mockSObjectsDescribeResult; - - /** - * Validates the schema fields with data type. - */ - @Test - public void testInputSchemaValidation() throws Exception { - MockFailureCollector collector = new MockFailureCollector(); - AuthenticatorCredentials authenticatorCredentials = Mockito.mock(AuthenticatorCredentials.class); - SalesforceSinkConfig salesforceSinkConfig = Mockito.spy( - new SalesforceSinkConfig(REFERENCE_NAME, - "CONSUMER_KEY", "CONSUMER_SECRET", - "USERNAME", "PASSWORD", - "LOGIN_URL", 30000, 3600, - "sObject", "Insert", null, - ConcurrencyMode.Parallel.name(), - "1000000", "10000", - "Fail on Error", - BaseSalesforceETLTest.SECURITY_TOKEN, - null, null, true, 5L, 10L, 5, true)); - Schema schema = Schema.recordOf("output", - Schema.Field.of("Name", Schema.of(Schema.Type.STRING)), - Schema.Field.of("StageName", Schema.of(Schema.Type.STRING)), - Schema.Field.of("Tax", Schema.of(Schema.Type.DOUBLE)), - Schema.Field.of("Amount", Schema.of(Schema.Type.INT)) - ); - Field field = new Field(); - List fields = new ArrayList<>(); - List nameParts = new ArrayList<>(); - nameParts.add("Amount"); - SalesforceFunctionType salesforceFunctionType = SalesforceFunctionType.get("String"); - SObjectDescriptor.FieldDescriptor descriptor1 = new SObjectDescriptor.FieldDescriptor(nameParts, "Name", - salesforceFunctionType); - SObjectDescriptor.FieldDescriptor descriptor2 = new SObjectDescriptor.FieldDescriptor(nameParts, "StageName", - salesforceFunctionType); - SObjectDescriptor.FieldDescriptor descriptor3 = new SObjectDescriptor.FieldDescriptor(nameParts, "Tax", - SalesforceFunctionType.DOUBLE); - SObjectDescriptor.FieldDescriptor descriptor4 = new SObjectDescriptor.FieldDescriptor(nameParts, "Amount", - SalesforceFunctionType.INT); - fields.add(descriptor1); - fields.add(descriptor2); - fields.add(descriptor3); - fields.add(descriptor4); - - SalesforceConnectorInfo connection = Mockito.mock(SalesforceConnectorInfo.class); - Mockito.when(salesforceSinkConfig.getConnection()).thenReturn(connection); - PowerMockito.when(connection.canAttemptToEstablishConnection()).thenReturn(true); - - Mockito.when(connection.getAuthenticatorCredentials()).thenReturn(authenticatorCredentials); - OAuthInfo oAuthInfo = new OAuthInfo("accessToken", "www.instanceUrl.com"); - PowerMockito.whenNew(AuthenticatorCredentials.class) - .withArguments(oAuthInfo, 3000, "www.proxyUrl.com") - .thenReturn(authenticatorCredentials); - PowerMockito.mockStatic(SObjectDescriptor.class); - PowerMockito.when(SObjectDescriptor.fromName(Mockito.any(), Mockito.any())) - .thenReturn(mockDescriptor); - Mockito.when(mockSObjectsDescribeResult.getField(Mockito.anyString(), Mockito.anyString())).thenReturn(field); - PowerMockito.when(mockDescriptor.getFields()).thenReturn(fields); - PowerMockito.mockStatic(SObjectsDescribeResult.class); - PowerMockito.when(SObjectsDescribeResult.of(Mockito.any(), Mockito.any(), Mockito.anySet())) - .thenReturn(mockSObjectsDescribeResult); - PowerMockito.doNothing().when(connection).validate(collector, oAuthInfo); - PowerMockito.mockStatic(SalesforceConnectionUtil.class); - Schema schemaUtil = SalesforceSchemaUtil.getSchema(authenticatorCredentials, mockDescriptor); - salesforceSinkConfig.validate(schema, collector, oAuthInfo); - Assert.assertEquals(4, schema.getFields().size()); - Assert.assertEquals(schemaUtil.getFields().get(2).getName(), schema.getFields().get(2).getName()); - Assert.assertEquals(schemaUtil.getFields().get(3).getSchema().getType(), - schema.getFields().get(3).getSchema().getType()); - } - - - /** - * Validates the schema fields of different data type. - */ - @Test - public void testInputSchemaValidationWithDifferentFields() throws Exception { - MockFailureCollector collector = new MockFailureCollector(); - AuthenticatorCredentials authenticatorCredentials = Mockito.mock(AuthenticatorCredentials.class); - SalesforceSinkConfig salesforceSinkConfig = Mockito.spy( - new SalesforceSinkConfig(REFERENCE_NAME, - "CONSUMER_KEY", "CONSUMER_SECRET", - "USERNAME", "PASSWORD", - "LOGIN_URL", 30000, 3600, - "sObject", "Insert", null, - ConcurrencyMode.Parallel.name(), - "1000000", "10000", - "Fail on Error", - BaseSalesforceETLTest.SECURITY_TOKEN, - null, null, true, 5L, 10L, 5, true)); - Schema schema = Schema.recordOf("output", - Schema.Field.of("Name", Schema.of(Schema.Type.STRING))); - Field field = new Field(); - List fields = new ArrayList<>(); - List nameParts = new ArrayList<>(); - nameParts.add("Amount"); - SalesforceFunctionType salesforceFunctionType = SalesforceFunctionType.INT; - SObjectDescriptor.FieldDescriptor descriptor = new SObjectDescriptor.FieldDescriptor(nameParts, "Name", - salesforceFunctionType); - fields.add(descriptor); - - SalesforceConnectorInfo connection = Mockito.mock(SalesforceConnectorInfo.class); - Mockito.when(salesforceSinkConfig.getConnection()).thenReturn(connection); - PowerMockito.when(connection.canAttemptToEstablishConnection()).thenReturn(true); - - Mockito.when(connection.getAuthenticatorCredentials()).thenReturn(authenticatorCredentials); - OAuthInfo oAuthInfo = new OAuthInfo("accessToken", "www.instanceUrl.com"); - PowerMockito.whenNew(AuthenticatorCredentials.class) - .withArguments(oAuthInfo, 3000, "www.proxyUrl.com") - .thenReturn(authenticatorCredentials); - PowerMockito.mockStatic(SObjectDescriptor.class); - PowerMockito.when(SObjectDescriptor.fromName(Mockito.any(), Mockito.any())) - .thenReturn(mockDescriptor); - Mockito.when(mockSObjectsDescribeResult.getField(Mockito.anyString(), Mockito.anyString())).thenReturn(field); - PowerMockito.when(mockDescriptor.getFields()).thenReturn(fields); - PowerMockito.mockStatic(SObjectsDescribeResult.class); - PowerMockito.when(SObjectsDescribeResult.of(Mockito.any(), Mockito.any(), Mockito.anySet())) - .thenReturn(mockSObjectsDescribeResult); - PowerMockito.doNothing().when(connection).validate(collector, oAuthInfo); - PowerMockito.mockStatic(SalesforceConnectionUtil.class); - try { - salesforceSinkConfig.validate(schema, collector, oAuthInfo); - collector.getOrThrowException(); - Assert.fail("Exception will not be thrown if dataType will be same."); - } catch (IllegalArgumentException e) { - Assert.assertEquals("Expected field '" + "Name" + "' to be of \"" + "string" + "\", but it is of \"" - + "int" + "\"", e.getMessage()); - - } - } -} From cf5e7e9a6d3c5e9a1e07827ada16a8f071cde1d1 Mon Sep 17 00:00:00 2001 From: Krish-cloudsufi Date: Thu, 21 Aug 2025 06:32:43 +0000 Subject: [PATCH 3/3] Skipped etl tests --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 96105da9..02604576 100644 --- a/pom.xml +++ b/pom.xml @@ -608,6 +608,7 @@ **/*TestRun.java + io/cdap/plugin/salesforce/etl/**