Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
a60a49a
Implemented SQLSink
jo-bao Oct 2, 2023
b4a04ae
New quoting
jo-bao Oct 4, 2023
d5f1c9f
documentations
jo-bao Oct 12, 2023
31ff1c8
implementend createTable
jo-bao Nov 21, 2023
542f5d4
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao Nov 21, 2023
0f46762
spotless
jo-bao Nov 21, 2023
b712f17
Improve SqlUtils
jo-bao Nov 22, 2023
c963923
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao Nov 22, 2023
75c57f9
Temprorary
jo-bao Nov 22, 2023
4955b93
temp
jo-bao Nov 24, 2023
a0d8bbd
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao Nov 24, 2023
31da1fd
Merge remote-tracking branch 'origin/dev' into jb/#762-sql-grid-datas…
jo-bao Feb 26, 2024
4243e57
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao Jun 14, 2024
d85e234
fixing tests
Jun 18, 2024
15d97cc
replace UniqueEntity
jo-bao Jul 1, 2024
d77e478
Main Method
jo-bao Jul 1, 2024
a079b8c
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao Jul 2, 2024
6667c3d
Code Cleaning
jo-bao Jul 2, 2024
6bb4abe
Code Cleaning
jo-bao Jul 2, 2024
f62a10d
SQL Caps
jo-bao Jul 2, 2024
5d0a594
Fixing sonarqube
Jul 3, 2024
4da951e
optimize tests
Jul 3, 2024
97f2a4f
Merge branch 'dev' into jb/#762-sql-grid-datasources
t-ober Aug 9, 2024
16f65e8
Merge branch 'dev' into jb/#762-sql-grid-datasources
Aug 29, 2024
62564e2
changes after review
jo-bao Aug 30, 2024
802d4d2
fixing Sonarqube
jo-bao Aug 30, 2024
0072c43
Merge branch 'dev' into jb/#762-sql-grid-datasources
t-ober Sep 25, 2024
8e7ee6d
Merge remote-tracking branch 'origin/dev' into jb/#762-sql-grid-datas…
jo-bao Oct 29, 2024
07eed44
simplify persist joint grid
jo-bao Oct 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package edu.ie3.datamodel.io;

import java.util.UUID;
import java.util.stream.Stream;

import static edu.ie3.datamodel.io.IoUtil.quote;

/**
* Class for identification of grids and results in SQL databases.
*/
public class DatabaseIdentifier {

private final String identifier;
private final UUID uuid;

public DatabaseIdentifier(
String identifier,
UUID uuid
) {
this.identifier = identifier;
this.uuid = uuid;
}

public String getIdentifier() {
return identifier;
}

public UUID getUuid() {
return uuid;
}

public String toString() {
return "identifier=" + identifier + ", uuid=" + uuid.toString();
}

public String[] getQueryString() {
return new String[]{quote(identifier, "'"), quote(uuid.toString(), "'")};
}

public Stream<String> getStreamForQuery() {
return Stream.concat(Stream.of(quote(identifier, "'")), Stream.of(quote(uuid.toString(), "'")));
}
}
15 changes: 15 additions & 0 deletions src/main/java/edu/ie3/datamodel/io/IoUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

import java.io.File;
import java.nio.file.Path;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Optional;

public class IoUtil {
Expand Down Expand Up @@ -49,4 +51,17 @@ public static Path harmonizeFileSeparator(Path path) {
public static Optional<Path> pathOption(String in) {
return Optional.of(Path.of(in));
}

public String timeFormatter(ZonedDateTime time, String timePattern) {
return time.format(DateTimeFormatter.ofPattern(timePattern));
}

public static String quote(String input, String quoteSymbol) {
if (input == "") {
return "NULL";
} else {
return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol;
}
}

}
192 changes: 192 additions & 0 deletions src/main/java/edu/ie3/datamodel/io/SqlUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
package edu.ie3.datamodel.io;

import edu.ie3.datamodel.exceptions.EntityProcessorException;
import edu.ie3.datamodel.exceptions.ProcessorProviderException;
import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy;
import edu.ie3.datamodel.io.processor.ProcessorProvider;
import edu.ie3.datamodel.models.UniqueEntity;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static edu.ie3.util.StringUtils.camelCaseToSnakeCase;

public class SqlUtils {
private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;";

private SqlUtils() {
throw new IllegalStateException("Utility classes cannot be instantiated");
}


public static String queryForCreation(
String schemaName,
String tableName,
Stream<Map<String, String>> columnsWithDataTypes
) {
return beginQueryCreateTable(schemaName, tableName) +
" "
+ endQueryCreateTable;
}

public static String getEndQueryCreateTable() {
return endQueryCreateTable;
}

private static String beginQueryCreateTable(
String schemaName,
String tableName
) {
return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n";
}

public static String getDataTypes(Class<? extends UniqueEntity> cls) {
try {
ProcessorProvider processorProvider = new ProcessorProvider();
DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy();
String body = "";
Stream<String> dataTypes;
String[] headerElements = processorProvider.getHeaderElements(cls);
Stream<String> strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_name", "grid_uuid")) ;
Stream<String> dtHeader = strHeader.map(
element -> {
return camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element));
}
);
return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" +
"WITHOUT OIDS\n" +
"\t" + "TABLESPACE pg_default;\n";
} catch (EntityProcessorException e) {
return "";
} catch (ProcessorProviderException e) {
return "";
}
}

public static Map<String, String> classToDataType() {
HashMap map = new HashMap();
map.put("uuid", "uuid PRIMARY KEY");
map.put("time_series", "uuid NOT NULL");
map.put("time", "timestamp with time zone NOT NULL");
map.put("p", "double precision NOT NULL");
map.put("q", "double precision NOT NULL");
map.put("c", "double precision NOT NULL");
map.put("s_rated", "double precision NOT NULL");

map.put("cost_controlled", "bool NOT NULL");
map.put("feed_in_tariff", "int NOT NULL");
map.put("id", "TEXT NOT NULL");
map.put("market_reaction", "bool NOT NULL");
map.put("node", "uuid NOT NULL");
//map.put("operatesFrom", "timestamp with time zone");
map.put("operates_from", "timestamp with time zone");
//map.put("operatesUntil", "timestamp with time zone");
map.put("operates_until", "timestamp with time zone");
map.put("operator", "uuid");
map.put("q_characteristics", "TEXT NOT NULL");
map.put("geo_position", "TEXT NOT NULL");
map.put("length", "double precision NOT NULL");
map.put("node_a", "uuid NOT NULL");
map.put("node_b", "uuid NOT NULL");
map.put("type", "uuid NOT NULL"); //EVCS
map.put("olm_characteristic", "TEXT NOT NULL");
map.put("parallel_devices", "int NOT NULL");
//map.put("parallelDevices", "int NOT NULL");
map.put("cos_phi_rated", "TEXT NOT NULL");
map.put("dsm", "bool NOT NULL");
map.put("e_cons_annual", "double precision NOT NULL");
map.put("load_profile", "TEXT NOT NULL");

map.put("auto_tap", "bool NOT NULL");
map.put("tap_pos", "int NOT NULL");
map.put("type", "uuid NOT NULL");

map.put("v_ang", "bool NOT NULL");
map.put("v_mag", "bool NOT NULL");
map.put("slack", "bool NOT NULL");
map.put("subnet", "int NOT NULL");
//map.put("vRated", "double precision NOT NULL");
map.put("v_rated", "double precision NOT NULL");
map.put("v_target", "double precision NOT NULL");
//map.put("vTarget", "double precision NOT NULL");
map.put("volt_lvl", "TEXT NOT NULL");
map.put("charging_points", "int NOT NULL");
map.put("location_type", "TEXT NOT NULL");
map.put("v_2g_support", "bool NOT NULL");
//map.put("voltLvl", "TEXT NOT NULL");

map.put("albedo", "double precision NOT NULL");
map.put("azimuth", "double precision NOT NULL");
map.put("elevation_angle", "double precision NOT NULL");
map.put("eta_conv", "double precision NOT NULL");
map.put("k_g", "double precision NOT NULL");
map.put("k_t", "double precision NOT NULL");

map.put("grid_name", "TEXT NOT NULL");
map.put("grid_uuid", "uuid NOT NULL");


map.put("b_m", "double precision NOT NULL");
map.put("d_phi", "double precision NOT NULL");
map.put("d_v", "double precision NOT NULL");
map.put("g_m", "double precision NOT NULL");
map.put("r_sc", "double precision NOT NULL");
map.put("tap_max", "int NOT NULL");
map.put("tap_min", "int NOT NULL");
map.put("tap_neutr", "int NOT NULL");
map.put("tap_side", "bool NOT NULL");
map.put("v_rated_a", "int NOT NULL");
map.put("v_rated_b", "int NOT NULL");
map.put("x_sc", "int NOT NULL");
map.put("graphic_layer", "TEXT NOT NULL");
map.put("line", "uuid NOT NULL");
map.put("path", "TEXT NOT NULL");
map.put("point", "TEXT NOT NULL");
map.put("inlet_temp", "double precision NOT NULL");
map.put("return_temp", "double precision NOT NULL");
map.put("storage_volume_lvl", "double precision NOT NULL");
map.put("storage_volume_lvl_min", "double precision NOT NULL");
map.put("thermal_bus", "uuid NOT NULL");
map.put("eth_capa", "double precision NOT NULL");
map.put("eth_losses", "double precision NOT NULL");
map.put("lower_temperature_limit", "double precision NOT NULL");
map.put("target_temperature", "double precision NOT NULL");
map.put("upper_temperature_limit", "double precision NOT NULL");
map.put("b", "double precision NOT NULL");
map.put("g", "double precision NOT NULL");
map.put("i_max", "double precision NOT NULL");
map.put("r", "double precision NOT NULL");
map.put("x", "double precision NOT NULL");

map.put("connected_assets", "TEXT NOT NULL");
map.put("capex", "double precision NOT NULL");
map.put("control_strategy", "TEXT NOT NULL");

map.put("input_model", "uuid NOT NULL");
map.put("soc", "double precision NOT NULL");
map.put("p_max", "double precision NOT NULL");
map.put("p_min", "double precision NOT NULL");
map.put("p_ref", "double precision NOT NULL");

map.put("dod", "double precision NOT NULL");
map.put("e_storage", "double precision NOT NULL");
map.put("eta", "double precision NOT NULL");
map.put("life_cycle", "double precision NOT NULL");
map.put("life_time", "double precision NOT NULL");
map.put("opex", "double precision NOT NULL");
map.put("active_power_gradient", "double precision NOT NULL");

return map;
}

public static String quote(String input, String quoteSymbol) {
if (input == "") {
return "NULL";
} else {
return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol;
}
}
}
16 changes: 9 additions & 7 deletions src/main/java/edu/ie3/datamodel/io/connectors/SqlConnector.java
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,8 @@ public ResultSet executeQuery(Statement stmt, String query) throws SQLException
* @param updateQuery the query to execute
* @return The number of updates or a negative number if the execution failed
*/
public int executeUpdate(String updateQuery) {
try (Statement stmt = getConnection().createStatement()) {
return stmt.executeUpdate(updateQuery);
} catch (SQLException e) {
log.error(String.format("Error at execution of query \"%1.127s\": ", updateQuery), e);
return -1;
}
public int executeUpdate(String updateQuery) throws SQLException {
return getConnection().createStatement().executeUpdate(updateQuery);
}

/**
Expand Down Expand Up @@ -160,4 +155,11 @@ public Map<String, String> extractFieldMap(ResultSet rs) {
}
return insensitiveFieldsToAttributes;
}

public boolean tableExists(Connection connection, String tableName) throws SQLException {
DatabaseMetaData meta = connection.getMetaData();
ResultSet resultSet = meta.getTables(null, null, tableName, new String[] {"TABLE"});

return resultSet.next();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,23 @@

import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.models.UniqueEntity;
import edu.ie3.datamodel.models.timeseries.TimeSeries;
import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries;
import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileInput;
import edu.ie3.datamodel.models.value.Value;

import java.util.Optional;

import static edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy.logger;

/** A naming strategy for database entities */
public class DatabaseNamingStrategy {

private static final String TIME_SERIES_PREFIX = "time_series_";

private static final String LOAD_PROFILE_PREFIX = "load_profile_";

private final EntityPersistenceNamingStrategy entityPersistenceNamingStrategy;

public DatabaseNamingStrategy(EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) {
Expand Down Expand Up @@ -42,7 +53,47 @@ public String getTimeSeriesEntityName(ColumnScheme columnScheme) {
return TIME_SERIES_PREFIX + columnScheme.getScheme();
}

/**
* Provides the name of a load profile given by the load profile key
* @param lpKey Load profile key
* @return the table name
*/
private String getLoadProfileEntityName(String lpKey) {
return LOAD_PROFILE_PREFIX + lpKey;
}

/**
* Provides the name of a unique entity class.
* @param cls Class extends UniqueEntity
* @return the table name
*/
public Optional<String> getEntityName(Class<? extends UniqueEntity> cls) {
return entityPersistenceNamingStrategy.getEntityName(cls);
}

/**
* Provides the name of a time series. Used to determine the table name in SQL database.
* @param timeSeries
* @return the table name
*/
public <T extends TimeSeries<E, V>, E extends TimeSeriesEntry<V>, V extends Value>
Optional<String> getEntityName(T timeSeries) {
if (timeSeries instanceof IndividualTimeSeries individualTimeSeries) {
Optional<E> maybeFirstElement = individualTimeSeries.getEntries().stream().findFirst();
if (maybeFirstElement.isPresent()) {
Class<? extends Value> valueClass = maybeFirstElement.get().getValue().getClass();
return Optional.of(getTimeSeriesEntityName(ColumnScheme.parse(valueClass).orElseThrow()));
} else {
logger.error("Unable to determine content of time series {}", timeSeries);
return Optional.empty();
}
} else if (timeSeries instanceof LoadProfileInput loadProfileInput) {
return Optional.of(
getLoadProfileEntityName(loadProfileInput.getType().getKey())
);
} else {
logger.error("There is no naming strategy defined for {}", timeSeries);
return Optional.empty();
}
}
}
3 changes: 2 additions & 1 deletion src/main/java/edu/ie3/datamodel/io/processor/Processor.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
import javax.measure.Quantity;
Expand Down Expand Up @@ -411,7 +412,7 @@ protected String processOperationTime(OperationTime operationTime, String fieldN
* @return string representation of the ZonedDateTime
*/
protected String processZonedDateTime(ZonedDateTime zonedDateTime) {
return zonedDateTime.toString();
return zonedDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss Z"));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,18 @@ Try<LinkedHashMap<String, String>, ProcessorProviderException> handleEntity(T en
.transformF(ProcessorProviderException::new));
}

public <T extends UniqueEntity> Set<LinkedHashMap<String, String>> handleEntities(List<T> entities) throws ProcessorProviderException {
Set<T> setOfEntities = new HashSet<>(entities);
Set<LinkedHashMap<String, String>> setOfMaps = new HashSet<>();
for (T entity : setOfEntities) {
LinkedHashMap<String, String> entryResult = handleEntity(entity).getOrThrow();

/* Prepare the actual result and add them to the set of all results */
setOfMaps.add(new LinkedHashMap<>(entryResult));
}
return setOfMaps;
}

/**
* Get the correct entity processor
*
Expand Down
Loading