Merge pull request #5025 from hansva/5024

add extra field to mysql bulkloader and classloader group, fixes #5024

Author: hansva

plugins/actions/mysqlbulkfile/src/main/java/org/apache/hop/workflow/actions/mysqlbulkfile/ActionMysqlBulkFile.java

@@ -54,7 +54,9 @@
     image = "MysqlBulkFile.svg",
     categoryDescription = "i18n:org.apache.hop.workflow:ActionCategory.Category.BulkLoading",
     keywords = "i18n::ActionMysqlBulkFile.keyword",
-    documentationUrl = "/workflow/actions/mysqlbulkfile.html")
+    documentationUrl = "/workflow/actions/mysqlbulkfile.html",
+    classLoaderGroup = "mysql-db",
+    isIncludeJdbcDrivers = true)
 @Getter
 @Setter
 public class ActionMysqlBulkFile extends ActionBase {

plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java

@@ -57,7 +57,9 @@
     image = "MysqlBulkLoad.svg",
     categoryDescription = "i18n:org.apache.hop.workflow:ActionCategory.Category.BulkLoading",
     keywords = "i18n::ActionMysqlBulkLoad.keyword",
-    documentationUrl = "/workflow/actions/mysqlbulkload.html")
+    documentationUrl = "/workflow/actions/mysqlbulkload.html",
+    classLoaderGroup = "mysql-db",
+    isIncludeJdbcDrivers = true)
 @Getter
 @Setter
 public class ActionMysqlBulkLoad extends ActionBase {

plugins/databases/mysql/src/main/java/org/apache/hop/databases/mysql/MySqlDatabaseMeta.java

@@ -43,7 +43,8 @@
 @DatabaseMetaPlugin(
     type = "MYSQL",
     typeDescription = "MySQL",
-    documentationUrl = "/database/databases/mysql.html")
+    documentationUrl = "/database/databases/mysql.html",
+    classLoaderGroup = "mysql-db")
 @GuiPlugin(id = "GUI-MySQLDatabaseMeta")
 public class MySqlDatabaseMeta extends BaseDatabaseMeta implements IDatabase {
   private static final Class<?> PKG = MySqlDatabaseMeta.class;

plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java

@@ -24,6 +24,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.nio.charset.Charset;
 import java.util.Date;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.database.Database;
@@ -43,9 +44,10 @@
 
 public class MySqlBulkLoader extends BaseTransform<MySqlBulkLoaderMeta, MySqlBulkLoaderData> {
   private static final Class<?> PKG = MySqlBulkLoaderMeta.class;
+  public static final String MESSAGE_ERRORSERIALIZING = "MySqlBulkLoader.Message.ERRORSERIALIZING";
 
-  private final long threadWaitTime = 300000;
-  private final String threadWaitTimeText = "5min";
+  private static final long THREAD_WAIT_TIME = 300000;
+  private static final String THREAD_WAIT_TIME_TEXT = "5min";
 
   public MySqlBulkLoader(
       TransformMeta transformMeta,
@@ -84,7 +86,7 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
         new Thread(outputLogger).start();
         int result = mkFifoProcess.waitFor();
         if (result != 0) {
-          throw new Exception(
+          throw new HopException(
               BaseMessages.getString(
                   PKG, "MySqlBulkLoader.Message.ERRORFIFORC", result, mkFifoCmd));
         }
@@ -105,7 +107,7 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
         new Thread(outputLogger).start();
         result = chmodProcess.waitFor();
         if (result != 0) {
-          throw new Exception(
+          throw new HopException(
               BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORFIFORC", result, chmodCmd));
         }
       }
@@ -138,54 +140,54 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
 
   private void executeLoadCommand() throws Exception {
 
-    String loadCommand = "";
-    loadCommand +=
+    StringBuilder loadCommand = new StringBuilder();
+    loadCommand.append(
         "LOAD DATA "
             + (meta.isLocalFile() ? "LOCAL" : "")
             + " INFILE '"
             + resolve(meta.getFifoFileName())
-            + "' ";
+            + "' ");
     if (meta.isReplacingData()) {
-      loadCommand += "REPLACE ";
+      loadCommand.append("REPLACE ");
     } else if (meta.isIgnoringErrors()) {
-      loadCommand += "IGNORE ";
+      loadCommand.append("IGNORE ");
     }
-    loadCommand += "INTO TABLE " + data.schemaTable + " ";
-    if (!Utils.isEmpty(meta.getEncoding())) {
-      loadCommand += "CHARACTER SET " + meta.getEncoding() + " ";
+    loadCommand.append("INTO TABLE " + data.schemaTable + " ");
+    if (!Utils.isEmpty(resolve(meta.getLoadCharSet()))) {
+      loadCommand.append("CHARACTER SET " + resolve(meta.getLoadCharSet()) + " ");
     }
     String delStr = meta.getDelimiter();
     if ("\t".equals(delStr)) {
       delStr = "\\t";
     }
 
-    loadCommand += "FIELDS TERMINATED BY '" + delStr + "' ";
+    loadCommand.append("FIELDS TERMINATED BY '" + delStr + "' ");
     if (!Utils.isEmpty(meta.getEnclosure())) {
-      loadCommand += "OPTIONALLY ENCLOSED BY '" + meta.getEnclosure() + "' ";
+      loadCommand.append("OPTIONALLY ENCLOSED BY '" + meta.getEnclosure() + "' ");
     }
-    loadCommand +=
+    loadCommand.append(
         "ESCAPED BY '"
             + meta.getEscapeChar()
             + ("\\".equals(meta.getEscapeChar()) ? meta.getEscapeChar() : "")
-            + "' ";
+            + "' ");
 
     // Build list of column names to set
-    loadCommand += "(";
+    loadCommand.append("(");
     for (int cnt = 0; cnt < meta.getFields().size(); cnt++) {
       DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables);
-      loadCommand += databaseMeta.quoteField(meta.getFields().get(cnt).getFieldTable());
+      loadCommand.append(databaseMeta.quoteField(meta.getFields().get(cnt).getFieldTable()));
       if (cnt < meta.getFields().size() - 1) {
-        loadCommand += ",";
+        loadCommand.append(",");
       }
     }
 
-    loadCommand += ");" + Const.CR;
+    loadCommand.append(");" + Const.CR);
 
     logBasic(
         BaseMessages.getString(
             PKG, "MySqlBulkLoader.Message.STARTING", data.dbDescription, loadCommand));
 
-    data.sqlRunner = new SqlRunner(data, loadCommand);
+    data.sqlRunner = new SqlRunner(data, loadCommand.toString());
     data.sqlRunner.start();
 
     // Ready to start writing rows to the FIFO file now...
@@ -314,10 +316,10 @@ private void closeOutput() throws Exception {
       // wait for the INSERT statement to finish and check for any error and/or warning...
       logDebug(
           "Waiting up to "
-              + this.threadWaitTimeText
+              + THREAD_WAIT_TIME_TEXT
               + " for the MySql load command thread to finish processing."); // no requirement for
       // NLS debug messages
-      data.sqlRunner.join(this.threadWaitTime);
+      data.sqlRunner.join(THREAD_WAIT_TIME);
       SqlRunner sqlRunner = data.sqlRunner;
       data.sqlRunner = null;
       sqlRunner.checkExcn();
@@ -344,6 +346,7 @@ private void writeRowToBulk(IRowMeta rowMeta, Object[] r) throws HopException {
         IValueMeta valueMeta = rowMeta.getValueMeta(index);
         Object valueData = r[index];
 
+        Charset charset = Charset.forName(resolve(meta.getEncoding()));
         if (valueData == null) {
           data.fifoStream.write("NULL".getBytes());
         } else {
@@ -456,27 +459,23 @@ private void writeRowToBulk(IRowMeta rowMeta, Object[] r) throws HopException {
       // If something went wrong with writing to the fifo, get the underlying error from MySql
       try {
         logError(
-            BaseMessages.getString(
-                PKG, "MySqlBulkLoader.Message.IOERROR", this.threadWaitTimeText));
+            BaseMessages.getString(PKG, "MySqlBulkLoader.Message.IOERROR", THREAD_WAIT_TIME_TEXT));
         try {
-          data.sqlRunner.join(this.threadWaitTime);
+          data.sqlRunner.join(THREAD_WAIT_TIME);
         } catch (InterruptedException ex) {
           // Ignore errors
         }
         data.sqlRunner.checkExcn();
       } catch (Exception loadEx) {
-        throw new HopException(
-            BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), loadEx);
+        throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), loadEx);
       }
 
       // MySql didn't finish, throw the generic "Pipe" exception.
-      throw new HopException(
-          BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), e);
+      throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), e);
 
     } catch (Exception e2) {
       // Null pointer exceptions etc.
-      throw new HopException(
-          BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), e2);
+      throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), e2);
     }
   }
 
@@ -603,8 +602,8 @@ public void run() {
       try {
         fifoStream =
             new BufferedOutputStream(new FileOutputStream(OpenFifo.this.fifoName), this.size);
-      } catch (Exception ex) {
-        this.ex = ex;
+      } catch (Exception exception) {
+        this.ex = exception;
       }
     }
 
@@ -637,8 +636,8 @@ static class SqlRunner extends Thread {
     public void run() {
       try {
         data.db.execStatement(loadCommand);
-      } catch (Exception ex) {
-        this.ex = ex;
+      } catch (Exception exception) {
+        this.ex = exception;
       }
     }
 

plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java

@@ -24,6 +24,7 @@
 import org.apache.hop.pipeline.transform.BaseTransformData;
 import org.apache.hop.pipeline.transform.ITransformData;
 
+@SuppressWarnings("java:S1104")
 public class MySqlBulkLoaderData extends BaseTransformData implements ITransformData {
   public Database db;
 

plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java

@@ -82,6 +82,7 @@ public class MySqlBulkLoaderDialog extends BaseTransformDialog {
   private TextVar wDelimiter;
   private TextVar wEnclosure;
   private TextVar wEscapeChar;
+  private TextVar wLoadCharSet;
   private TextVar wCharSet;
   private TextVar wBulkSize;
   private TableView wReturn;
@@ -293,21 +294,41 @@ public void widgetSelected(SelectionEvent se) {
     fdEscapeChar.right = new FormAttachment(100, 0);
     wEscapeChar.setLayoutData(fdEscapeChar);
 
+    // Load Charset line...
+    Label wlLoadCharSet = new Label(shell, SWT.RIGHT);
+    wlLoadCharSet.setText(BaseMessages.getString(PKG, "MySqlBulkLoaderDialog.LoadCharSet.Label"));
+    PropsUi.setLook(wlLoadCharSet);
+    FormData fdlLoadCharSet = new FormData();
+    fdlLoadCharSet.left = new FormAttachment(0, 0);
+    fdlLoadCharSet.right = new FormAttachment(middle, -margin);
+    fdlLoadCharSet.top = new FormAttachment(wEscapeChar, margin);
+    wlLoadCharSet.setLayoutData(fdlLoadCharSet);
+
+    wLoadCharSet = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+    PropsUi.setLook(wLoadCharSet);
+    wLoadCharSet.addModifyListener(lsMod);
+    FormData fdLoadCharSet = new FormData();
+    fdLoadCharSet.left = new FormAttachment(middle, 0);
+    fdLoadCharSet.top = new FormAttachment(wEscapeChar, margin);
+    fdLoadCharSet.right = new FormAttachment(100, 0);
+    wLoadCharSet.setLayoutData(fdLoadCharSet);
+
     // CharSet line...
     Label wlCharSet = new Label(shell, SWT.RIGHT);
     wlCharSet.setText(BaseMessages.getString(PKG, "MySqlBulkLoaderDialog.CharSet.Label"));
     PropsUi.setLook(wlCharSet);
     FormData fdlCharSet = new FormData();
     fdlCharSet.left = new FormAttachment(0, 0);
     fdlCharSet.right = new FormAttachment(middle, -margin);
-    fdlCharSet.top = new FormAttachment(wEscapeChar, margin);
+    fdlCharSet.top = new FormAttachment(wLoadCharSet, margin);
     wlCharSet.setLayoutData(fdlCharSet);
+
     wCharSet = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
     PropsUi.setLook(wCharSet);
     wCharSet.addModifyListener(lsMod);
     FormData fdCharSet = new FormData();
     fdCharSet.left = new FormAttachment(middle, 0);
-    fdCharSet.top = new FormAttachment(wEscapeChar, margin);
+    fdCharSet.top = new FormAttachment(wLoadCharSet, margin);
     fdCharSet.right = new FormAttachment(100, 0);
     wCharSet.setLayoutData(fdCharSet);
 
@@ -696,6 +717,7 @@ public void getData() {
     wEnclosure.setText(Const.NVL(input.getEnclosure(), ""));
     wDelimiter.setText(Const.NVL(input.getDelimiter(), ""));
     wEscapeChar.setText(Const.NVL(input.getEscapeChar(), ""));
+    wLoadCharSet.setText(Const.NVL(input.getLoadCharSet(), ""));
     wCharSet.setText(Const.NVL(input.getEncoding(), ""));
     wReplace.setSelection(input.isReplacingData());
     wIgnore.setSelection(input.isIgnoringErrors());
@@ -768,6 +790,7 @@ private void getInfo(MySqlBulkLoaderMeta inf) {
     inf.setEnclosure(wEnclosure.getText());
     inf.setDelimiter(wDelimiter.getText());
     inf.setEscapeChar(wEscapeChar.getText());
+    inf.setLoadCharSet(wLoadCharSet.getText());
     inf.setEncoding(wCharSet.getText());
     inf.setReplacingData(wReplace.getSelection());
     inf.setIgnoringErrors(wIgnore.getSelection());
@@ -859,12 +882,21 @@ private void getUpdate() {
               if (v.getType() == IValueMeta.TYPE_DATE) {
                 // The default is : format is OK for dates, see if this sticks later on...
                 //
-                tableItem.setText(3, "Y");
+                tableItem.setText(
+                    3,
+                    BaseMessages.getString(
+                        PKG, "MySqlBulkLoaderMeta.FieldFormatType.Date.Description"));
               } else {
-                tableItem.setText(3, "Y"); // default is OK too...
+                tableItem.setText(
+                    3,
+                    BaseMessages.getString(
+                        PKG,
+                        "MySqlBulkLoaderMeta.FieldFormatType.OK.Description")); // default is OK
+                // too...
               }
               return true;
             };
+
         BaseTransformDialog.getFieldsFromPrevious(
             r, wReturn, 1, new int[] {1, 2}, new int[] {}, -1, -1, listener);
       }
@@ -935,8 +967,7 @@ private void setTableFieldCombo() {
             if (!Utils.isEmpty(tableName)) {
               DatabaseMeta ci = pipelineMeta.findDatabase(connectionName, variables);
               if (ci != null) {
-                Database db = new Database(loggingObject, variables, ci);
-                try {
+                try (Database db = new Database(loggingObject, variables, ci)) {
                   db.connect();
 
                   String schemaTable =
@@ -957,13 +988,6 @@ private void setTableFieldCombo() {
                   }
                   // ignore any errors here. drop downs will not be
                   // filled, but no problem for the user
-                } finally {
-                  try {
-                    db.disconnect();
-                  } catch (Exception ignored) {
-                    // ignore any errors here.
-                    db = null;
-                  }
                 }
               }
             }

plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java

@@ -51,7 +51,9 @@
     description = "i18n::MySqlBulkLoader.Description",
     categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk",
     keywords = "i18n::MySqlBulkLoader.keyword",
-    documentationUrl = "/pipeline/transforms/mysqlbulkloader.html")
+    documentationUrl = "/pipeline/transforms/mysqlbulkloader.html",
+    isIncludeJdbcDrivers = true,
+    classLoaderGroup = "mysql-db")
 @Getter
 @Setter
 public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader, MySqlBulkLoaderData> {
@@ -92,6 +94,9 @@ public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader, MySq
   @HopMetadataProperty(key = "encoding")
   private String encoding;
 
+  @HopMetadataProperty(key = "loadCharSet")
+  private String loadCharSet;
+
   @HopMetadataProperty(key = "replace")
   private boolean replacingData;
 
@@ -147,7 +152,8 @@ public void setDefault() {
     connection = "";
     schemaName = "";
     tableName = BaseMessages.getString(PKG, "MySqlBulkLoaderMeta.DefaultTableName");
-    encoding = "";
+    encoding = "UTF8";
+    loadCharSet = "UTF8MB4";
     fifoFileName = "/tmp/fifo";
     delimiter = "\t";
     enclosure = "\"";

plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties

@@ -33,7 +33,7 @@ MySqlBulkLoader.Message.UNABLETODELETE=Unable to delete FIFO file - {0}
 MySqlBulkLoader.Message.UNEXPECTEDERRORCLOSING=Unexpected error encountered while closing the client connection
 MySqlBulkLoaderDialog.Browse.Button=&Browse...
 MySqlBulkLoaderDialog.BulkSize.Label=Bulk size (rows) after which we restart the data load
-MySqlBulkLoaderDialog.CharSet.Label=Character set
+MySqlBulkLoaderDialog.CharSet.Label=Character set (file creation)
 MySqlBulkLoaderDialog.ColumnInfo.FormatOK=Field format OK
 MySqlBulkLoaderDialog.ColumnInfo.StreamField=Stream field
 MySqlBulkLoaderDialog.ColumnInfo.TableField=Table field
@@ -100,4 +100,5 @@ MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description=Format as a timestamp
 MySqlBulkLoaderMeta.GetSQL.ErrorOccurred=An error occurred\:
 MySqlBulkLoaderMeta.GetSQL.NoConnectionDefined=There is no connection defined in this transform.
 MySqlBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection=No table is defined on this connection.
-MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields=Not receiving any fields from previous transforms. Check the previous transforms for errors & the connecting hops.
+MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields=Not receiving any fields from previous transforms. Check the previous transforms for errors & the connecting hops.
+MySqlBulkLoaderDialog.LoadCharSet.Label=Character set (load command)