Skip to content

Commit 9db4ad8

Browse files
hansvahansva
committed
add extra field to mysql bulkloader and classloader group, fixes #5024 
1 parent f24ba06 commit 9db4ad8

File tree

8 files changed

+92
-56
lines changed

8 files changed

+92
-56
lines changed

‎plugins/actions/mysqlbulkfile/src/main/java/org/apache/hop/workflow/actions/mysqlbulkfile/ActionMysqlBulkFile.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@
5454
image = "MysqlBulkFile.svg",
5555
categoryDescription = "i18n:org.apache.hop.workflow:ActionCategory.Category.BulkLoading",
5656
keywords = "i18n::ActionMysqlBulkFile.keyword",
57-
documentationUrl = "/workflow/actions/mysqlbulkfile.html")
57+
documentationUrl = "/workflow/actions/mysqlbulkfile.html",
58+
classLoaderGroup = "mysql-db",
59+
isIncludeJdbcDrivers = true)
5860
@Getter
5961
@Setter
6062
public class ActionMysqlBulkFile extends ActionBase {

‎plugins/actions/mysqlbulkload/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,9 @@
5757
image = "MysqlBulkLoad.svg",
5858
categoryDescription = "i18n:org.apache.hop.workflow:ActionCategory.Category.BulkLoading",
5959
keywords = "i18n::ActionMysqlBulkLoad.keyword",
60-
documentationUrl = "/workflow/actions/mysqlbulkload.html")
60+
documentationUrl = "/workflow/actions/mysqlbulkload.html",
61+
classLoaderGroup = "mysql-db",
62+
isIncludeJdbcDrivers = true)
6163
@Getter
6264
@Setter
6365
public class ActionMysqlBulkLoad extends ActionBase {

‎plugins/databases/mysql/src/main/java/org/apache/hop/databases/mysql/MySqlDatabaseMeta.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
@DatabaseMetaPlugin(
4444
type = "MYSQL",
4545
typeDescription = "MySQL",
46-
documentationUrl = "/database/databases/mysql.html")
46+
documentationUrl = "/database/databases/mysql.html",
47+
classLoaderGroup = "mysql-db")
4748
@GuiPlugin(id = "GUI-MySQLDatabaseMeta")
4849
public class MySqlDatabaseMeta extends BaseDatabaseMeta implements IDatabase {
4950
private static final Class<?> PKG = MySqlDatabaseMeta.class;

‎plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java‎

Lines changed: 35 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.io.FileOutputStream;
2525
import java.io.IOException;
2626
import java.math.BigDecimal;
27+
import java.nio.charset.Charset;
2728
import java.util.Date;
2829
import org.apache.hop.core.Const;
2930
import org.apache.hop.core.database.Database;
@@ -43,9 +44,10 @@
4344

4445
public class MySqlBulkLoader extends BaseTransform<MySqlBulkLoaderMeta, MySqlBulkLoaderData> {
4546
private static final Class<?> PKG = MySqlBulkLoaderMeta.class;
47+
public static final String MESSAGE_ERRORSERIALIZING = "MySqlBulkLoader.Message.ERRORSERIALIZING";
4648

47-
private final long threadWaitTime = 300000;
48-
private final String threadWaitTimeText = "5min";
49+
private static final long THREAD_WAIT_TIME = 300000;
50+
private static final String THREAD_WAIT_TIME_TEXT = "5min";
4951

5052
public MySqlBulkLoader(
5153
TransformMeta transformMeta,
@@ -84,7 +86,7 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
8486
new Thread(outputLogger).start();
8587
int result = mkFifoProcess.waitFor();
8688
if (result != 0) {
87-
throw new Exception(
89+
throw new HopException(
8890
BaseMessages.getString(
8991
PKG, "MySqlBulkLoader.Message.ERRORFIFORC", result, mkFifoCmd));
9092
}
@@ -105,7 +107,7 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
105107
new Thread(outputLogger).start();
106108
result = chmodProcess.waitFor();
107109
if (result != 0) {
108-
throw new Exception(
110+
throw new HopException(
109111
BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORFIFORC", result, chmodCmd));
110112
}
111113
}
@@ -138,54 +140,54 @@ public boolean execute(MySqlBulkLoaderMeta meta) throws HopException {
138140

139141
private void executeLoadCommand() throws Exception {
140142

141-
String loadCommand = "";
142-
loadCommand +=
143+
StringBuilder loadCommand = new StringBuilder();
144+
loadCommand.append(
143145
"LOAD DATA "
144146
+ (meta.isLocalFile() ? "LOCAL" : "")
145147
+ " INFILE '"
146148
+ resolve(meta.getFifoFileName())
147-
+ "' ";
149+
+ "' ");
148150
if (meta.isReplacingData()) {
149-
loadCommand += "REPLACE ";
151+
loadCommand.append("REPLACE ");
150152
} else if (meta.isIgnoringErrors()) {
151-
loadCommand += "IGNORE ";
153+
loadCommand.append("IGNORE ");
152154
}
153-
loadCommand += "INTO TABLE " + data.schemaTable + " ";
154-
if (!Utils.isEmpty(meta.getEncoding())) {
155-
loadCommand += "CHARACTER SET " + meta.getEncoding() + " ";
155+
loadCommand.append("INTO TABLE " + data.schemaTable + " ");
156+
if (!Utils.isEmpty(resolve(meta.getLoadCharSet()))) {
157+
loadCommand.append("CHARACTER SET " + resolve(meta.getLoadCharSet()) + " ");
156158
}
157159
String delStr = meta.getDelimiter();
158160
if ("\t".equals(delStr)) {
159161
delStr = "\\t";
160162
}
161163

162-
loadCommand += "FIELDS TERMINATED BY '" + delStr + "' ";
164+
loadCommand.append("FIELDS TERMINATED BY '" + delStr + "' ");
163165
if (!Utils.isEmpty(meta.getEnclosure())) {
164-
loadCommand += "OPTIONALLY ENCLOSED BY '" + meta.getEnclosure() + "' ";
166+
loadCommand.append("OPTIONALLY ENCLOSED BY '" + meta.getEnclosure() + "' ");
165167
}
166-
loadCommand +=
168+
loadCommand.append(
167169
"ESCAPED BY '"
168170
+ meta.getEscapeChar()
169171
+ ("\\".equals(meta.getEscapeChar()) ? meta.getEscapeChar() : "")
170-
+ "' ";
172+
+ "' ");
171173

172174
// Build list of column names to set
173-
loadCommand += "(";
175+
loadCommand.append("(");
174176
for (int cnt = 0; cnt < meta.getFields().size(); cnt++) {
175177
DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables);
176-
loadCommand += databaseMeta.quoteField(meta.getFields().get(cnt).getFieldTable());
178+
loadCommand.append(databaseMeta.quoteField(meta.getFields().get(cnt).getFieldTable()));
177179
if (cnt < meta.getFields().size() - 1) {
178-
loadCommand += ",";
180+
loadCommand.append(",");
179181
}
180182
}
181183

182-
loadCommand += ");" + Const.CR;
184+
loadCommand.append(");" + Const.CR);
183185

184186
logBasic(
185187
BaseMessages.getString(
186188
PKG, "MySqlBulkLoader.Message.STARTING", data.dbDescription, loadCommand));
187189

188-
data.sqlRunner = new SqlRunner(data, loadCommand);
190+
data.sqlRunner = new SqlRunner(data, loadCommand.toString());
189191
data.sqlRunner.start();
190192

191193
// Ready to start writing rows to the FIFO file now...
@@ -314,10 +316,10 @@ private void closeOutput() throws Exception {
314316
// wait for the INSERT statement to finish and check for any error and/or warning...
315317
logDebug(
316318
"Waiting up to "
317-
+ this.threadWaitTimeText
319+
+ THREAD_WAIT_TIME_TEXT
318320
+ " for the MySql load command thread to finish processing."); // no requirement for
319321
// NLS debug messages
320-
data.sqlRunner.join(this.threadWaitTime);
322+
data.sqlRunner.join(THREAD_WAIT_TIME);
321323
SqlRunner sqlRunner = data.sqlRunner;
322324
data.sqlRunner = null;
323325
sqlRunner.checkExcn();
@@ -344,6 +346,7 @@ private void writeRowToBulk(IRowMeta rowMeta, Object[] r) throws HopException {
344346
IValueMeta valueMeta = rowMeta.getValueMeta(index);
345347
Object valueData = r[index];
346348

349+
Charset charset = Charset.forName(resolve(meta.getEncoding()));
347350
if (valueData == null) {
348351
data.fifoStream.write("NULL".getBytes());
349352
} else {
@@ -456,27 +459,23 @@ private void writeRowToBulk(IRowMeta rowMeta, Object[] r) throws HopException {
456459
// If something went wrong with writing to the fifo, get the underlying error from MySql
457460
try {
458461
logError(
459-
BaseMessages.getString(
460-
PKG, "MySqlBulkLoader.Message.IOERROR", this.threadWaitTimeText));
462+
BaseMessages.getString(PKG, "MySqlBulkLoader.Message.IOERROR", THREAD_WAIT_TIME_TEXT));
461463
try {
462-
data.sqlRunner.join(this.threadWaitTime);
464+
data.sqlRunner.join(THREAD_WAIT_TIME);
463465
} catch (InterruptedException ex) {
464466
// Ignore errors
465467
}
466468
data.sqlRunner.checkExcn();
467469
} catch (Exception loadEx) {
468-
throw new HopException(
469-
BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), loadEx);
470+
throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), loadEx);
470471
}
471472

472473
// MySql didn't finish, throw the generic "Pipe" exception.
473-
throw new HopException(
474-
BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), e);
474+
throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), e);
475475

476476
} catch (Exception e2) {
477477
// Null pointer exceptions etc.
478-
throw new HopException(
479-
BaseMessages.getString(PKG, "MySqlBulkLoader.Message.ERRORSERIALIZING"), e2);
478+
throw new HopException(BaseMessages.getString(PKG, MESSAGE_ERRORSERIALIZING), e2);
480479
}
481480
}
482481

@@ -603,8 +602,8 @@ public void run() {
603602
try {
604603
fifoStream =
605604
new BufferedOutputStream(new FileOutputStream(OpenFifo.this.fifoName), this.size);
606-
} catch (Exception ex) {
607-
this.ex = ex;
605+
} catch (Exception exception) {
606+
this.ex = exception;
608607
}
609608
}
610609

@@ -637,8 +636,8 @@ static class SqlRunner extends Thread {
637636
public void run() {
638637
try {
639638
data.db.execStatement(loadCommand);
640-
} catch (Exception ex) {
641-
this.ex = ex;
639+
} catch (Exception exception) {
640+
this.ex = exception;
642641
}
643642
}
644643

‎plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderData.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.hop.pipeline.transform.BaseTransformData;
2525
import org.apache.hop.pipeline.transform.ITransformData;
2626

27+
@SuppressWarnings("java:S1104")
2728
public class MySqlBulkLoaderData extends BaseTransformData implements ITransformData {
2829
public Database db;
2930

‎plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ public class MySqlBulkLoaderDialog extends BaseTransformDialog {
8282
private TextVar wDelimiter;
8383
private TextVar wEnclosure;
8484
private TextVar wEscapeChar;
85+
private TextVar wLoadCharSet;
8586
private TextVar wCharSet;
8687
private TextVar wBulkSize;
8788
private TableView wReturn;
@@ -293,21 +294,41 @@ public void widgetSelected(SelectionEvent se) {
293294
fdEscapeChar.right = new FormAttachment(100, 0);
294295
wEscapeChar.setLayoutData(fdEscapeChar);
295296

297+
// Load Charset line...
298+
Label wlLoadCharSet = new Label(shell, SWT.RIGHT);
299+
wlLoadCharSet.setText(BaseMessages.getString(PKG, "MySqlBulkLoaderDialog.LoadCharSet.Label"));
300+
PropsUi.setLook(wlLoadCharSet);
301+
FormData fdlLoadCharSet = new FormData();
302+
fdlLoadCharSet.left = new FormAttachment(0, 0);
303+
fdlLoadCharSet.right = new FormAttachment(middle, -margin);
304+
fdlLoadCharSet.top = new FormAttachment(wEscapeChar, margin);
305+
wlLoadCharSet.setLayoutData(fdlLoadCharSet);
306+
307+
wLoadCharSet = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
308+
PropsUi.setLook(wLoadCharSet);
309+
wLoadCharSet.addModifyListener(lsMod);
310+
FormData fdLoadCharSet = new FormData();
311+
fdLoadCharSet.left = new FormAttachment(middle, 0);
312+
fdLoadCharSet.top = new FormAttachment(wEscapeChar, margin);
313+
fdLoadCharSet.right = new FormAttachment(100, 0);
314+
wLoadCharSet.setLayoutData(fdLoadCharSet);
315+
296316
// CharSet line...
297317
Label wlCharSet = new Label(shell, SWT.RIGHT);
298318
wlCharSet.setText(BaseMessages.getString(PKG, "MySqlBulkLoaderDialog.CharSet.Label"));
299319
PropsUi.setLook(wlCharSet);
300320
FormData fdlCharSet = new FormData();
301321
fdlCharSet.left = new FormAttachment(0, 0);
302322
fdlCharSet.right = new FormAttachment(middle, -margin);
303-
fdlCharSet.top = new FormAttachment(wEscapeChar, margin);
323+
fdlCharSet.top = new FormAttachment(wLoadCharSet, margin);
304324
wlCharSet.setLayoutData(fdlCharSet);
325+
305326
wCharSet = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
306327
PropsUi.setLook(wCharSet);
307328
wCharSet.addModifyListener(lsMod);
308329
FormData fdCharSet = new FormData();
309330
fdCharSet.left = new FormAttachment(middle, 0);
310-
fdCharSet.top = new FormAttachment(wEscapeChar, margin);
331+
fdCharSet.top = new FormAttachment(wLoadCharSet, margin);
311332
fdCharSet.right = new FormAttachment(100, 0);
312333
wCharSet.setLayoutData(fdCharSet);
313334

@@ -696,6 +717,7 @@ public void getData() {
696717
wEnclosure.setText(Const.NVL(input.getEnclosure(), ""));
697718
wDelimiter.setText(Const.NVL(input.getDelimiter(), ""));
698719
wEscapeChar.setText(Const.NVL(input.getEscapeChar(), ""));
720+
wLoadCharSet.setText(Const.NVL(input.getLoadCharSet(), ""));
699721
wCharSet.setText(Const.NVL(input.getEncoding(), ""));
700722
wReplace.setSelection(input.isReplacingData());
701723
wIgnore.setSelection(input.isIgnoringErrors());
@@ -768,6 +790,7 @@ private void getInfo(MySqlBulkLoaderMeta inf) {
768790
inf.setEnclosure(wEnclosure.getText());
769791
inf.setDelimiter(wDelimiter.getText());
770792
inf.setEscapeChar(wEscapeChar.getText());
793+
inf.setLoadCharSet(wLoadCharSet.getText());
771794
inf.setEncoding(wCharSet.getText());
772795
inf.setReplacingData(wReplace.getSelection());
773796
inf.setIgnoringErrors(wIgnore.getSelection());
@@ -859,12 +882,21 @@ private void getUpdate() {
859882
if (v.getType() == IValueMeta.TYPE_DATE) {
860883
// The default is : format is OK for dates, see if this sticks later on...
861884
//
862-
tableItem.setText(3, "Y");
885+
tableItem.setText(
886+
3,
887+
BaseMessages.getString(
888+
PKG, "MySqlBulkLoaderMeta.FieldFormatType.Date.Description"));
863889
} else {
864-
tableItem.setText(3, "Y"); // default is OK too...
890+
tableItem.setText(
891+
3,
892+
BaseMessages.getString(
893+
PKG,
894+
"MySqlBulkLoaderMeta.FieldFormatType.OK.Description")); // default is OK
895+
// too...
865896
}
866897
return true;
867898
};
899+
868900
BaseTransformDialog.getFieldsFromPrevious(
869901
r, wReturn, 1, new int[] {1, 2}, new int[] {}, -1, -1, listener);
870902
}
@@ -935,8 +967,7 @@ private void setTableFieldCombo() {
935967
if (!Utils.isEmpty(tableName)) {
936968
DatabaseMeta ci = pipelineMeta.findDatabase(connectionName, variables);
937969
if (ci != null) {
938-
Database db = new Database(loggingObject, variables, ci);
939-
try {
970+
try (Database db = new Database(loggingObject, variables, ci)) {
940971
db.connect();
941972

942973
String schemaTable =
@@ -957,13 +988,6 @@ private void setTableFieldCombo() {
957988
}
958989
// ignore any errors here. drop downs will not be
959990
// filled, but no problem for the user
960-
} finally {
961-
try {
962-
db.disconnect();
963-
} catch (Exception ignored) {
964-
// ignore any errors here.
965-
db = null;
966-
}
967991
}
968992
}
969993
}

‎plugins/transforms/mysqlbulkloader/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderMeta.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@
5151
description = "i18n::MySqlBulkLoader.Description",
5252
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Bulk",
5353
keywords = "i18n::MySqlBulkLoader.keyword",
54-
documentationUrl = "/pipeline/transforms/mysqlbulkloader.html")
54+
documentationUrl = "/pipeline/transforms/mysqlbulkloader.html",
55+
isIncludeJdbcDrivers = true,
56+
classLoaderGroup = "mysql-db")
5557
@Getter
5658
@Setter
5759
public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader, MySqlBulkLoaderData> {
@@ -92,6 +94,9 @@ public class MySqlBulkLoaderMeta extends BaseTransformMeta<MySqlBulkLoader, MySq
9294
@HopMetadataProperty(key = "encoding")
9395
private String encoding;
9496

97+
@HopMetadataProperty(key = "loadCharSet")
98+
private String loadCharSet;
99+
95100
@HopMetadataProperty(key = "replace")
96101
private boolean replacingData;
97102

@@ -147,7 +152,8 @@ public void setDefault() {
147152
connection = "";
148153
schemaName = "";
149154
tableName = BaseMessages.getString(PKG, "MySqlBulkLoaderMeta.DefaultTableName");
150-
encoding = "";
155+
encoding = "UTF8";
156+
loadCharSet = "UTF8MB4";
151157
fifoFileName = "/tmp/fifo";
152158
delimiter = "\t";
153159
enclosure = "\"";

‎plugins/transforms/mysqlbulkloader/src/main/resources/org/apache/hop/pipeline/transforms/mysqlbulkloader/messages/messages_en_US.properties‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ MySqlBulkLoader.Message.UNABLETODELETE=Unable to delete FIFO file - {0}
3333
MySqlBulkLoader.Message.UNEXPECTEDERRORCLOSING=Unexpected error encountered while closing the client connection
3434
MySqlBulkLoaderDialog.Browse.Button=&Browse...
3535
MySqlBulkLoaderDialog.BulkSize.Label=Bulk size (rows) after which we restart the data load
36-
MySqlBulkLoaderDialog.CharSet.Label=Character set
36+
MySqlBulkLoaderDialog.CharSet.Label=Character set (file creation)
3737
MySqlBulkLoaderDialog.ColumnInfo.FormatOK=Field format OK
3838
MySqlBulkLoaderDialog.ColumnInfo.StreamField=Stream field
3939
MySqlBulkLoaderDialog.ColumnInfo.TableField=Table field
@@ -100,4 +100,5 @@ MySqlBulkLoaderMeta.FieldFormatType.Timestamp.Description=Format as a timestamp
100100
MySqlBulkLoaderMeta.GetSQL.ErrorOccurred=An error occurred\:
101101
MySqlBulkLoaderMeta.GetSQL.NoConnectionDefined=There is no connection defined in this transform.
102102
MySqlBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection=No table is defined on this connection.
103-
MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields=Not receiving any fields from previous transforms. Check the previous transforms for errors & the connecting hops.
103+
MySqlBulkLoaderMeta.GetSQL.NotReceivingAnyFields=Not receiving any fields from previous transforms. Check the previous transforms for errors & the connecting hops.
104+
MySqlBulkLoaderDialog.LoadCharSet.Label=Character set (load command)

0 commit comments

Comments
 (0)