Merge pull request #8 from zazi/reutilize_input_schema

[DD-1169] reutilize input schema

Author: fniederl

README.md

@@ -71,6 +71,9 @@ prototype.projectID=819f2f6e-98ed-90e2-372e-71a0a1eec786
 # if multiple projects deliver the mappings for the task
 prototype.projectIDs=9d6ec288-f1bf-4f96-78f6-5399e3050125,69664ba5-bbe5-6f35-7a77-47bacf9d3731
 
+# if an existing input should be utilised at input data model creation
+prototype.inputSchemaID=Schema-cb8f4b96-9ab2-4972-88f8-143656199518
+
 # the output data model refers to the output schema as well
 prototype.outputDataModelID=DataModel-cf998267-392a-4d87-a33a-88dd1bffb016
 

conf/config.properties.template

@@ -26,6 +26,8 @@ prototype.dataModelID=bbd368e8-b75c-0e64-b96a-ab812a700b4f
 prototype.projectID=819f2f6e-98ed-90e2-372e-71a0a1eec786
 # if multiple projects deliver the mappings for the task
 prototype.projectIDs=9d6ec288-f1bf-4f96-78f6-5399e3050125,69664ba5-bbe5-6f35-7a77-47bacf9d3731
+# if an existing input should be utilised at input data model creation
+prototype.inputSchemaID=Schema-cb8f4b96-9ab2-4972-88f8-143656199518
 prototype.outputDataModelID=DataModel-cf998267-392a-4d87-a33a-88dd1bffb016
 
 # enables init part (i.e. resource + data model creation)

src/main/java/de/tu_dortmund/ub/data/dswarm/DswarmBackendStatics.java

@@ -9,11 +9,13 @@ public final class DswarmBackendStatics {
 	public static final String RESOURCES_ENDPOINT      = "resources";
 	public static final String CONFIGURATIONS_ENDPOINT = "configurations";
 	public static final String PROJECTS_ENDPOINT       = "projects";
+	public static final String SCHEMAS_ENDPOINT        = "schemas";
 	public static final String TASKS_ENDPOINT          = "tasks";
 
 	public static final String UUID_IDENTIFIER              = "uuid";
 	public static final String MAPPINGS_IDENTIFIER          = "mappings";
 	public static final String DATA_RESOURCE_IDENTIFIER     = "data_resource";
+	public static final String SCHEMA_IDENTIFIER            = "schema";
 	public static final String NAME_IDENTIFIER              = "name";
 	public static final String DESCRIPTION_IDENTIFIER       = "description";
 	public static final String TASK_IDENTIFIER              = "task";
@@ -28,5 +30,6 @@ public final class DswarmBackendStatics {
 	public static final String DO_INGEST_ON_THE_FLY               = "do_ingest_on_the_fly";
 	public static final String DO_EXPORT_ON_THE_FLY               = "do_export_on_the_fly";
 	public static final String DO_VERSIONING_ON_RESULT_IDENTIFIER = "do_versioning_on_result";
+	public static final String UTILISE_EXISTING_INPUT_IDENTIFIER  = "utilise_existing_input_schema";
 	public static final String DO_DATA_MODEL_INGEST_IDENTIFIER    = "doIngest";
 }

src/main/java/de/tu_dortmund/ub/data/dswarm/Export.java

@@ -63,23 +63,23 @@ public String call() {
 
 		LOG.info(String.format("[%s] Starting 'XML-Export (Task)' ...", serviceName));
 
-		// init process values
-		String message = null;
-
 		try {
 
 			TPUUtil.initSchemaIndices(serviceName, config);
 
 			// export and save to results folder
 			exportDataModel(exportDataModelID, serviceName);
 
+			return null;
 		} catch (final Exception e) {
 
-			LOG.error(String.format("[%s] Exporting and saving datamodel '%s' failed with a %s", serviceName, exportDataModelID, e.getClass()
-					.getSimpleName()), e);
-		}
+			final String message = String
+					.format("[%s] Exporting and saving datamodel '%s' failed with a %s", serviceName, exportDataModelID, e.getClass()
+							.getSimpleName());
+			LOG.error(message, e);
 
-		return message;
+			throw new RuntimeException(message, e);
+		}
 	}
 
 	/**
@@ -121,6 +121,10 @@ private void exportDataModel(final String dataModelID, final String serviceName)
 					default: {
 
 						LOG.error(String.format("[%s] %d : %s", serviceName, statusCode, httpResponse.getStatusLine().getReasonPhrase()));
+
+						final String response = TPUUtil.getResponseMessage(httpResponse);
+
+						throw new Exception("something went wrong at data model export: " + response);
 					}
 				}
 

src/main/java/de/tu_dortmund/ub/data/dswarm/Ingest.java

@@ -25,13 +25,13 @@ of this software and associated documentation files (the "Software"), to deal
 package de.tu_dortmund.ub.data.dswarm;
 
 import java.io.File;
-import java.io.StringWriter;
 import java.util.Properties;
 import java.util.concurrent.Callable;
 
 import javax.json.Json;
 import javax.json.JsonReader;
 
+import de.tu_dortmund.ub.data.util.TPUUtil;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -43,7 +43,6 @@ of this software and associated documentation files (the "Software"), to deal
 import org.apache.http.entity.mime.content.StringBody;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -95,9 +94,6 @@ public String call() {
 		final String dataModelID = this.dataModelID;
 		final String updateResourceID = resourceID;
 
-		// init process values
-		final String message = null;
-
 		try {
 			// build a InputDataModel for the resource
 			final String name = String.format("resource for project '%s'", resource);
@@ -129,14 +125,18 @@ public String call() {
 
 			// no need to clean up resources or datamodels anymore
 
+			LOG.info(String.format("[%s] Finished 'Ingest (Task)' no. '%d' ...", serviceName, cnt));
+
+			return null;
 		} catch (final Exception e) {
 
-			LOG.error(String.format("[%s] Processing resource '%s' failed with a %s", serviceName, resource, e.getClass().getSimpleName()), e);
-		}
+			final String message = String.format("[%s] Processing resource '%s' failed with a %s", serviceName, resource,
+					e.getClass().getSimpleName());
 
-		LOG.info(String.format("[%s] Finished 'Ingest (Task)' no. '%d' ...", serviceName, cnt));
+			LOG.error(message, e);
 
-		return message;
+			throw new RuntimeException(message, e);
+		}
 	}
 
 	/**
@@ -178,6 +178,10 @@ private String updateDataModelContent(final String inputDataModelID, final Strin
 					default: {
 
 						LOG.error(message);
+
+						final String response = TPUUtil.getResponseMessage(httpResponse);
+
+						throw new Exception("something went wrong at data model export: " + message + " " + response);
 					}
 				}
 			}
@@ -227,36 +231,30 @@ private String uploadFileAndUpdateResource(final String resourceUUID, final Stri
 			try (final CloseableHttpResponse httpResponse = httpclient.execute(httpPut)) {
 
 				final int statusCode = httpResponse.getStatusLine().getStatusCode();
-				final HttpEntity httpEntity = httpResponse.getEntity();
 
 				final String message = String.format("[%s] %d : %s", serviceName, statusCode, httpResponse.getStatusLine()
 						.getReasonPhrase());
 
+				final String response = TPUUtil.getResponseMessage(httpResponse);
+
 				switch (statusCode) {
 
 					case 200: {
 
 						LOG.info(message);
-						final StringWriter writer = new StringWriter();
-						IOUtils.copy(httpEntity.getContent(), writer, APIStatics.UTF_8);
-						final String responseJson = writer.toString();
-						writer.flush();
-						writer.close();
 
-						LOG.debug(String.format("[%s] responseJson : %s", serviceName, responseJson));
+						LOG.debug(String.format("[%s] responseJson : %s", serviceName, response));
 
-						return responseJson;
+						return response;
 					}
 					default: {
 
 						LOG.error(message);
+
+						throw new Exception("something went wrong at data model export: " + message + " " + response);
 					}
 				}
-
-				EntityUtils.consume(httpEntity);
 			}
 		}
-
-		return null;
 	}
 }