Skip to content

Commit baa3ad4

Browse files
hansvahansva
authored
Cleanup XML of transform MonetDB Bulk Loader, fixes #1954 (#6772)
1 parent 968c30c commit baa3ad4

File tree

7 files changed

+570
-437
lines changed

7 files changed

+570
-437
lines changed

‎plugins/transforms/monetdbbulkloader/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoader.java‎

Lines changed: 30 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public class MonetDbBulkLoader extends BaseTransform<MonetDbBulkLoaderMeta, Mone
5050

5151
private IRowMeta physicalTableRowMeta;
5252
private static final String ERROR_LOADING_DATA = "Error loading data: ";
53+
DatabaseMeta databaseMeta;
5354

5455
public MonetDbBulkLoader(
5556
TransformMeta transformMeta,
@@ -88,10 +89,8 @@ public boolean execute(MonetDbBulkLoaderMeta meta) throws HopException {
8889
logDetailed("Auto String Length flag: " + meta.isAutoStringWidths());
8990
}
9091

91-
DatabaseMeta dm = meta.getDatabaseMeta();
92-
93-
String user = resolve(Const.NVL(dm.getUsername(), ""));
94-
String password = Utils.resolvePassword(variables, Const.NVL(dm.getPassword(), ""));
92+
String user = resolve(Const.NVL(databaseMeta.getUsername(), ""));
93+
String password = Utils.resolvePassword(variables, Const.NVL(databaseMeta.getPassword(), ""));
9594

9695
MapiSocket mserver = getMonetDBConnection();
9796
data.mserver = mserver;
@@ -115,7 +114,7 @@ public boolean execute(MonetDbBulkLoaderMeta meta) throws HopException {
115114
// get table metadata, will be used later for date type identification (DATE, TIMESTAMP, ...)
116115
try {
117116

118-
db = new Database(meta.getParent(), variables, dm);
117+
db = new Database(meta.getParent(), variables, databaseMeta);
119118
db.connect();
120119
physicalTableRowMeta = db.getTableFields(data.schemaTable);
121120
} catch (Exception e) {
@@ -149,7 +148,7 @@ public boolean processRow() throws HopException {
149148
setOutputDone();
150149
if (!first) {
151150
try {
152-
writeBufferToMonetDB(meta.getDatabaseMeta());
151+
writeBufferToMonetDB(databaseMeta);
153152
data.out.flush();
154153
} finally {
155154
data.mserver.close();
@@ -163,17 +162,17 @@ public boolean processRow() throws HopException {
163162

164163
// Cache field indexes.
165164
//
166-
data.keynrs = new int[meta.getFieldStream().length];
165+
data.keynrs = new int[meta.getFields().size()];
167166
for (int i = 0; i < data.keynrs.length; i++) {
168-
data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
167+
data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getFields().get(i).getFieldStream());
169168
}
170169

171170
// execute the psql statement...
172171
//
173172
execute(meta);
174173
}
175174

176-
writeRowToMonetDB(getInputRowMeta(), r, meta.getDatabaseMeta());
175+
writeRowToMonetDB(getInputRowMeta(), r, databaseMeta);
177176
putRow(getInputRowMeta(), r);
178177
incrementLinesOutput();
179178

@@ -240,7 +239,7 @@ protected void addRowToBuffer(IRowMeta rowMeta, Object[] r) throws HopException
240239
line.append(data.quote);
241240
break;
242241
case IValueMeta.TYPE_INTEGER:
243-
if (valueMeta.isStorageBinaryString() && meta.getFieldFormatOk()[i]) {
242+
if (valueMeta.isStorageBinaryString() && meta.getFields().get(i).isFieldFormatOk()) {
244243
line.append(valueMeta.getString(valueData));
245244
} else {
246245
Long value = valueMeta.getInteger(valueData);
@@ -257,7 +256,7 @@ protected void addRowToBuffer(IRowMeta rowMeta, Object[] r) throws HopException
257256
//
258257
case IValueMeta.TYPE_TIMESTAMP, IValueMeta.TYPE_DATE:
259258
// Keep the data format as indicated.
260-
if (valueMeta.isStorageBinaryString() && meta.getFieldFormatOk()[i]) {
259+
if (valueMeta.isStorageBinaryString() && meta.getFields().get(i).isFieldFormatOk()) {
261260
line.append(valueMeta.getString(valueData));
262261
} else {
263262

@@ -301,7 +300,7 @@ protected void addRowToBuffer(IRowMeta rowMeta, Object[] r) throws HopException
301300
break;
302301

303302
case IValueMeta.TYPE_NUMBER:
304-
if (valueMeta.isStorageBinaryString() && meta.getFieldFormatOk()[i]) {
303+
if (valueMeta.isStorageBinaryString() && meta.getFields().get(i).isFieldFormatOk()) {
305304
line.append(valueMeta.getString(valueData));
306305
} else {
307306
Double dbl = valueMeta.getNumber(valueData);
@@ -326,7 +325,7 @@ protected void addRowToBuffer(IRowMeta rowMeta, Object[] r) throws HopException
326325
break;
327326

328327
case IValueMeta.TYPE_BIGNUMBER:
329-
if (valueMeta.isStorageBinaryString() && meta.getFieldFormatOk()[i]) {
328+
if (valueMeta.isStorageBinaryString() && meta.getFields().get(i).isFieldFormatOk()) {
330329
line.append(valueMeta.getString(valueData));
331330
} else {
332331
BigDecimal bd = valueMeta.getBigNumber(valueData);
@@ -374,8 +373,8 @@ public void truncate() throws HopException {
374373
String cmd;
375374
String table = data.schemaTable;
376375
String truncateStatement =
377-
meta.getDatabaseMeta()
378-
.getTruncateTableStatement(variables, meta.getSchemaName(), meta.getTableName());
376+
databaseMeta.getTruncateTableStatement(
377+
variables, meta.getSchemaName(), meta.getTableName());
379378
if (truncateStatement == null) {
380379
throw new HopException("Truncate table is not supported!");
381380
}
@@ -535,7 +534,7 @@ protected void writeBufferToMonetDB(DatabaseMeta dm) throws HopException {
535534

536535
protected void verifyDatabaseConnection() throws HopException {
537536
// Confirming Database Connection is defined.
538-
if (meta.getDatabaseMeta() == null) {
537+
if (databaseMeta == null) {
539538
throw new HopException(
540539
BaseMessages.getString(PKG, "MonetDbBulkLoaderMeta.GetSQL.NoConnectionDefined"));
541540
}
@@ -545,6 +544,8 @@ protected void verifyDatabaseConnection() throws HopException {
545544
public boolean init() {
546545
if (super.init()) {
547546

547+
databaseMeta = getPipelineMeta().findDatabase(meta.getDbConnectionName(), variables);
548+
548549
try {
549550
verifyDatabaseConnection();
550551
} catch (HopException ex) {
@@ -592,21 +593,12 @@ public boolean init() {
592593

593594
// Make sure our database connection settings are consistent with our dialog settings by
594595
// altering the connection with an updated answer depending on the dialog setting.
595-
meta.getDatabaseMeta().setQuoteAllFields(meta.isFullyQuoteSQL());
596-
597-
// Support parameterized database connection names
598-
String connectionName = meta.getDbConnectionName();
599-
if (!Utils.isEmpty(connectionName)
600-
&& connectionName.startsWith("${")
601-
&& connectionName.endsWith("}")) {
602-
meta.setDatabaseMeta(getPipelineMeta().findDatabase(resolve(connectionName), variables));
603-
}
596+
databaseMeta.setQuoteAllFields(meta.isFullyQuoteSQL());
604597

605598
// Schema-table combination...
606599
data.schemaTable =
607-
meta.getDatabaseMeta(this)
608-
.getQuotedSchemaTableCombination(
609-
variables, meta.getSchemaName(), meta.getTableName());
600+
databaseMeta.getQuotedSchemaTableCombination(
601+
variables, meta.getSchemaName(), meta.getTableName());
610602

611603
return true;
612604
}
@@ -622,12 +614,11 @@ protected MapiSocket getMonetDBConnection() throws Exception {
622614
if (this.meta == null) {
623615
throw new HopException("No metadata available to determine connection information from.");
624616
}
625-
DatabaseMeta dm = meta.getDatabaseMeta();
626-
String hostname = resolve(Const.NVL(dm.getHostname(), ""));
627-
String portnum = resolve(Const.NVL(dm.getPort(), ""));
628-
String user = resolve(Const.NVL(dm.getUsername(), ""));
629-
String password = Utils.resolvePassword(variables, Const.NVL(dm.getPassword(), ""));
630-
String db = resolve(Const.NVL(dm.getDatabaseName(), ""));
617+
String hostname = resolve(Const.NVL(databaseMeta.getHostname(), ""));
618+
String portnum = resolve(Const.NVL(databaseMeta.getPort(), ""));
619+
String user = resolve(Const.NVL(databaseMeta.getUsername(), ""));
620+
String password = Utils.resolvePassword(variables, Const.NVL(databaseMeta.getPassword(), ""));
621+
String db = resolve(Const.NVL(databaseMeta.getDatabaseName(), ""));
631622

632623
return getMonetDBConnection(
633624
hostname, Integer.parseInt(portnum), user, password, db, getLogChannel());
@@ -664,12 +655,11 @@ protected void executeSql(String query) throws Exception {
664655
if (this.meta == null) {
665656
throw new HopException("No metadata available to determine connection information from.");
666657
}
667-
DatabaseMeta dm = meta.getDatabaseMeta();
668-
String hostname = resolve(Const.NVL(dm.getHostname(), ""));
669-
String portnum = resolve(Const.NVL(dm.getPort(), ""));
670-
String user = resolve(Const.NVL(dm.getUsername(), ""));
671-
String password = Utils.resolvePassword(variables, Const.NVL(dm.getPassword(), ""));
672-
String db = resolve(Const.NVL(dm.getDatabaseName(), ""));
658+
String hostname = resolve(Const.NVL(databaseMeta.getHostname(), ""));
659+
String portnum = resolve(Const.NVL(databaseMeta.getPort(), ""));
660+
String user = resolve(Const.NVL(databaseMeta.getUsername(), ""));
661+
String password = Utils.resolvePassword(variables, Const.NVL(databaseMeta.getPassword(), ""));
662+
String db = resolve(Const.NVL(databaseMeta.getDatabaseName(), ""));
673663

674664
executeSql(query, hostname, Integer.parseInt(portnum), user, password, db);
675665
}

‎plugins/transforms/monetdbbulkloader/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoaderDialog.java‎

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ public void focusLost(FocusEvent arg0) {
163163
// Connection line
164164
//
165165
// Connection line
166-
wConnection = addConnectionLine(shell, wSpacer, input.getDatabaseMeta(), lsMod);
166+
wConnection = addConnectionLine(shell, wSpacer, input.getDbConnectionName(), lsMod);
167167

168168
// //////////////////////////////////////////////
169169
// Prepare the Folder that will contain tabs. //
@@ -328,12 +328,13 @@ public void widgetSelected(SelectionEvent arg0) {
328328

329329
wFullyQuoteSQL = new Button(wGeneralSettingsComp, SWT.CHECK);
330330
PropsUi.setLook(wFullyQuoteSQL);
331+
DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables);
331332
SelectionAdapter lsFullyQuoteSQL =
332333
new SelectionAdapter() {
333334
@Override
334335
public void widgetSelected(SelectionEvent arg0) {
335336
input.setChanged();
336-
input.getDatabaseMeta().setQuoteAllFields(wFullyQuoteSQL.getSelection());
337+
databaseMeta.setQuoteAllFields(wFullyQuoteSQL.getSelection());
337338
}
338339
};
339340
wFullyQuoteSQL.addSelectionListener(lsFullyQuoteSQL);
@@ -408,7 +409,7 @@ public void widgetSelected(SelectionEvent arg0) {
408409
PropsUi.setLook(wlReturn);
409410

410411
int upInsCols = 3;
411-
int upInsRows = (input.getFieldTable() != null ? input.getFieldTable().length : 1);
412+
int upInsRows = (input.getFields() != null ? input.getFields().size() : 1);
412413

413414
ciReturn = new ColumnInfo[upInsCols];
414415
ciReturn[0] =
@@ -784,21 +785,21 @@ public void getData() {
784785
logDebug(BaseMessages.getString(PKG, "MonetDBBulkLoaderDialog.Log.GettingKeyInfo"));
785786
}
786787

787-
if (input.getFieldTable() != null) {
788-
for (int i = 0; i < input.getFieldTable().length; i++) {
788+
if (!input.getFields().isEmpty()) {
789+
for (int i = 0; i < input.getFields().size(); i++) {
789790
TableItem item = wReturn.table.getItem(i);
790-
if (input.getFieldTable()[i] != null) {
791-
item.setText(1, input.getFieldTable()[i]);
791+
if (input.getFields().get(i).getFieldTable() != null) {
792+
item.setText(1, input.getFields().get(i).getFieldTable());
792793
}
793-
if (input.getFieldStream()[i] != null) {
794-
item.setText(2, input.getFieldStream()[i]);
794+
if (input.getFields().get(i).getFieldStream() != null) {
795+
item.setText(2, input.getFields().get(i).getFieldStream());
795796
}
796-
item.setText(3, input.getFieldFormatOk()[i] ? "Y" : "N");
797+
item.setText(3, input.getFields().get(i).isFieldFormatOk() ? "Y" : "N");
797798
}
798799
}
799800

800-
if (input.getDatabaseMeta() != null) {
801-
wConnection.setText(input.getDatabaseMeta().getName());
801+
if (input.getDbConnectionName() != null) {
802+
wConnection.setText(input.getDbConnectionName());
802803
}
803804
// General Settings Tab values from transform meta-data configuration.
804805
if (input.getSchemaName() != null) {
@@ -855,24 +856,23 @@ protected void setComboBoxes() {
855856
protected void getInfo(MonetDbBulkLoaderMeta inf) {
856857
int nrfields = wReturn.nrNonEmpty();
857858

858-
inf.allocate(nrfields);
859-
860859
if (log.isDebug()) {
861860
logDebug(
862861
BaseMessages.getString(PKG, "MonetDBBulkLoaderDialog.Log.FoundFields", "" + nrfields));
863862
}
863+
inf.getFields().clear();
864864

865865
for (int i = 0; i < nrfields; i++) {
866866
TableItem item = wReturn.getNonEmpty(i);
867-
inf.getFieldTable()[i] = item.getText(1);
868-
inf.getFieldStream()[i] = item.getText(2);
869-
inf.getFieldFormatOk()[i] = "Y".equalsIgnoreCase(item.getText(3));
867+
MonetDbBulkLoaderMeta.MonetDbField mf =
868+
new MonetDbBulkLoaderMeta.MonetDbField(
869+
item.getText(1), item.getText(2), "Y".equalsIgnoreCase(item.getText(3)));
870+
inf.getFields().add(mf);
870871
}
871872
// General Settings Tab values from transform meta-data configuration.
872873
inf.setDbConnectionName(wConnection.getText());
873874
inf.setSchemaName(wSchema.getText());
874875
inf.setTableName(wTable.getText());
875-
inf.setDatabaseMeta(pipelineMeta.findDatabase(wConnection.getText(), variables));
876876
inf.setBufferSize(wBufferSize.getText());
877877
inf.setLogFile(wLogFile.getText());
878878
inf.setTruncate(wTruncate.getSelection());
@@ -954,7 +954,6 @@ private void generateMappings() {
954954
return;
955955
}
956956
// refresh data
957-
input.setDatabaseMeta(pipelineMeta.findDatabase(wConnection.getText(), variables));
958957
input.setTableName(variables.resolve(wTable.getText()));
959958
ITransformMeta transformMetaInterface = transformMeta.getTransform();
960959
try {
@@ -1118,17 +1117,13 @@ private void create() {
11181117

11191118
String name = transformName; // new name might not yet be linked to other transforms!
11201119

1120+
DatabaseMeta databaseMeta = pipelineMeta.findDatabase(wConnection.getText(), variables);
11211121
SqlStatement sql = info.getTableDdl(variables, pipelineMeta, name, false, null, false);
11221122
if (!sql.hasError()) {
11231123
if (sql.hasSql()) {
11241124
SqlEditor sqledit =
11251125
new SqlEditor(
1126-
shell,
1127-
SWT.NONE,
1128-
variables,
1129-
info.getDatabaseMeta(),
1130-
DbCache.getInstance(),
1131-
sql.getSql());
1126+
shell, SWT.NONE, variables, databaseMeta, DbCache.getInstance(), sql.getSql());
11321127
sqledit.open();
11331128
} else {
11341129
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION);

0 commit comments

Comments
 (0)