-
Notifications
You must be signed in to change notification settings - Fork 7
Implement SQL sinks #903
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Implement SQL sinks #903
Changes from 3 commits
Commits
Show all changes
29 commits
Select commit
Hold shift + click to select a range
a60a49a
Implemented SQLSink
jo-bao b4a04ae
New quoting
jo-bao d5f1c9f
documentations
jo-bao 31ff1c8
implementend createTable
jo-bao 542f5d4
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao 0f46762
spotless
jo-bao b712f17
Improve SqlUtils
jo-bao c963923
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao 75c57f9
Temprorary
jo-bao 4955b93
temp
jo-bao a0d8bbd
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao 31da1fd
Merge remote-tracking branch 'origin/dev' into jb/#762-sql-grid-datas…
jo-bao 4243e57
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao d85e234
fixing tests
15d97cc
replace UniqueEntity
jo-bao d77e478
Main Method
jo-bao a079b8c
Merge branch 'dev' into jb/#762-sql-grid-datasources
jo-bao 6667c3d
Code Cleaning
jo-bao 6bb4abe
Code Cleaning
jo-bao f62a10d
SQL Caps
jo-bao 5d0a594
Fixing sonarqube
4da951e
optimize tests
97f2a4f
Merge branch 'dev' into jb/#762-sql-grid-datasources
t-ober 16f65e8
Merge branch 'dev' into jb/#762-sql-grid-datasources
62564e2
changes after review
jo-bao 802d4d2
fixing Sonarqube
jo-bao 0072c43
Merge branch 'dev' into jb/#762-sql-grid-datasources
t-ober 8e7ee6d
Merge remote-tracking branch 'origin/dev' into jb/#762-sql-grid-datas…
jo-bao 07eed44
simplify persist joint grid
jo-bao File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
43 changes: 43 additions & 0 deletions
43
src/main/java/edu/ie3/datamodel/io/DatabaseIdentifier.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
package edu.ie3.datamodel.io; | ||
|
||
import java.util.UUID; | ||
import java.util.stream.Stream; | ||
|
||
import static edu.ie3.datamodel.io.IoUtil.quote; | ||
|
||
/** | ||
* Class for identification of grids and results in SQL databases. | ||
*/ | ||
public class DatabaseIdentifier { | ||
|
||
private final String identifier; | ||
private final UUID uuid; | ||
|
||
public DatabaseIdentifier( | ||
String identifier, | ||
UUID uuid | ||
) { | ||
this.identifier = identifier; | ||
this.uuid = uuid; | ||
} | ||
|
||
public String getIdentifier() { | ||
return identifier; | ||
} | ||
|
||
public UUID getUuid() { | ||
return uuid; | ||
} | ||
|
||
public String toString() { | ||
return "identifier=" + identifier + ", uuid=" + uuid.toString(); | ||
} | ||
|
||
public String[] getQueryString() { | ||
return new String[]{quote(identifier, "'"), quote(uuid.toString(), "'")}; | ||
} | ||
jo-bao marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
||
public Stream<String> getStreamForQuery() { | ||
return Stream.concat(Stream.of(quote(identifier, "'")), Stream.of(quote(uuid.toString(), "'"))); | ||
} | ||
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,192 @@ | ||
package edu.ie3.datamodel.io; | ||
|
||
import edu.ie3.datamodel.exceptions.EntityProcessorException; | ||
import edu.ie3.datamodel.exceptions.ProcessorProviderException; | ||
import edu.ie3.datamodel.io.naming.DatabaseNamingStrategy; | ||
import edu.ie3.datamodel.io.processor.ProcessorProvider; | ||
import edu.ie3.datamodel.models.UniqueEntity; | ||
|
||
import java.util.Arrays; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
import java.util.stream.Collectors; | ||
import java.util.stream.Stream; | ||
|
||
import static edu.ie3.util.StringUtils.camelCaseToSnakeCase; | ||
|
||
public class SqlUtils { | ||
private static final String endQueryCreateTable = ")\n" + "\t WITHOUT OIDS\n" + "\t TABLESPACE pg_default;"; | ||
jo-bao marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
||
private SqlUtils() { | ||
throw new IllegalStateException("Utility classes cannot be instantiated"); | ||
} | ||
|
||
|
||
public static String queryForCreation( | ||
String schemaName, | ||
String tableName, | ||
Stream<Map<String, String>> columnsWithDataTypes | ||
) { | ||
return beginQueryCreateTable(schemaName, tableName) + | ||
" " | ||
+ endQueryCreateTable; | ||
} | ||
|
||
public static String getEndQueryCreateTable() { | ||
return endQueryCreateTable; | ||
} | ||
|
||
private static String beginQueryCreateTable( | ||
String schemaName, | ||
String tableName | ||
) { | ||
return "CREATE TABLE " + schemaName + "." + tableName + "\n(\n"; | ||
} | ||
|
||
public static String getDataTypes(Class<? extends UniqueEntity> cls) { | ||
try { | ||
ProcessorProvider processorProvider = new ProcessorProvider(); | ||
DatabaseNamingStrategy namingStrategy = new DatabaseNamingStrategy(); | ||
String body = ""; | ||
Stream<String> dataTypes; | ||
String[] headerElements = processorProvider.getHeaderElements(cls); | ||
Stream<String> strHeader = Stream.concat(Arrays.stream(headerElements), Stream.of("grid_name", "grid_uuid")) ; | ||
Stream<String> dtHeader = strHeader.map( | ||
element -> { | ||
return camelCaseToSnakeCase(element) + " " + classToDataType().get(camelCaseToSnakeCase(element)); | ||
} | ||
); | ||
return "CREATE TABLE public." + namingStrategy.getEntityName(cls).orElseThrow() + "\n(\n\t" + String.valueOf(dtHeader.collect(Collectors.joining(",\n\t"))) + "\n)\n\t" + | ||
"WITHOUT OIDS\n" + | ||
"\t" + "TABLESPACE pg_default;\n"; | ||
} catch (EntityProcessorException e) { | ||
return ""; | ||
} catch (ProcessorProviderException e) { | ||
return ""; | ||
} | ||
} | ||
|
||
public static Map<String, String> classToDataType() { | ||
HashMap map = new HashMap(); | ||
map.put("uuid", "uuid PRIMARY KEY"); | ||
map.put("time_series", "uuid NOT NULL"); | ||
map.put("time", "timestamp with time zone NOT NULL"); | ||
map.put("p", "double precision NOT NULL"); | ||
map.put("q", "double precision NOT NULL"); | ||
map.put("c", "double precision NOT NULL"); | ||
map.put("s_rated", "double precision NOT NULL"); | ||
|
||
map.put("cost_controlled", "bool NOT NULL"); | ||
map.put("feed_in_tariff", "int NOT NULL"); | ||
map.put("id", "TEXT NOT NULL"); | ||
map.put("market_reaction", "bool NOT NULL"); | ||
map.put("node", "uuid NOT NULL"); | ||
//map.put("operatesFrom", "timestamp with time zone"); | ||
map.put("operates_from", "timestamp with time zone"); | ||
//map.put("operatesUntil", "timestamp with time zone"); | ||
map.put("operates_until", "timestamp with time zone"); | ||
map.put("operator", "uuid"); | ||
map.put("q_characteristics", "TEXT NOT NULL"); | ||
map.put("geo_position", "TEXT NOT NULL"); | ||
map.put("length", "double precision NOT NULL"); | ||
map.put("node_a", "uuid NOT NULL"); | ||
map.put("node_b", "uuid NOT NULL"); | ||
map.put("type", "uuid NOT NULL"); //EVCS | ||
map.put("olm_characteristic", "TEXT NOT NULL"); | ||
map.put("parallel_devices", "int NOT NULL"); | ||
//map.put("parallelDevices", "int NOT NULL"); | ||
map.put("cos_phi_rated", "TEXT NOT NULL"); | ||
map.put("dsm", "bool NOT NULL"); | ||
map.put("e_cons_annual", "double precision NOT NULL"); | ||
map.put("load_profile", "TEXT NOT NULL"); | ||
|
||
map.put("auto_tap", "bool NOT NULL"); | ||
map.put("tap_pos", "int NOT NULL"); | ||
map.put("type", "uuid NOT NULL"); | ||
|
||
map.put("v_ang", "bool NOT NULL"); | ||
map.put("v_mag", "bool NOT NULL"); | ||
map.put("slack", "bool NOT NULL"); | ||
map.put("subnet", "int NOT NULL"); | ||
//map.put("vRated", "double precision NOT NULL"); | ||
map.put("v_rated", "double precision NOT NULL"); | ||
map.put("v_target", "double precision NOT NULL"); | ||
//map.put("vTarget", "double precision NOT NULL"); | ||
map.put("volt_lvl", "TEXT NOT NULL"); | ||
map.put("charging_points", "int NOT NULL"); | ||
map.put("location_type", "TEXT NOT NULL"); | ||
map.put("v_2g_support", "bool NOT NULL"); | ||
//map.put("voltLvl", "TEXT NOT NULL"); | ||
|
||
map.put("albedo", "double precision NOT NULL"); | ||
map.put("azimuth", "double precision NOT NULL"); | ||
map.put("elevation_angle", "double precision NOT NULL"); | ||
map.put("eta_conv", "double precision NOT NULL"); | ||
map.put("k_g", "double precision NOT NULL"); | ||
map.put("k_t", "double precision NOT NULL"); | ||
|
||
map.put("grid_name", "TEXT NOT NULL"); | ||
map.put("grid_uuid", "uuid NOT NULL"); | ||
|
||
|
||
map.put("b_m", "double precision NOT NULL"); | ||
map.put("d_phi", "double precision NOT NULL"); | ||
map.put("d_v", "double precision NOT NULL"); | ||
map.put("g_m", "double precision NOT NULL"); | ||
map.put("r_sc", "double precision NOT NULL"); | ||
map.put("tap_max", "int NOT NULL"); | ||
map.put("tap_min", "int NOT NULL"); | ||
map.put("tap_neutr", "int NOT NULL"); | ||
map.put("tap_side", "bool NOT NULL"); | ||
map.put("v_rated_a", "int NOT NULL"); | ||
map.put("v_rated_b", "int NOT NULL"); | ||
map.put("x_sc", "int NOT NULL"); | ||
map.put("graphic_layer", "TEXT NOT NULL"); | ||
map.put("line", "uuid NOT NULL"); | ||
map.put("path", "TEXT NOT NULL"); | ||
map.put("point", "TEXT NOT NULL"); | ||
map.put("inlet_temp", "double precision NOT NULL"); | ||
map.put("return_temp", "double precision NOT NULL"); | ||
map.put("storage_volume_lvl", "double precision NOT NULL"); | ||
map.put("storage_volume_lvl_min", "double precision NOT NULL"); | ||
map.put("thermal_bus", "uuid NOT NULL"); | ||
map.put("eth_capa", "double precision NOT NULL"); | ||
map.put("eth_losses", "double precision NOT NULL"); | ||
map.put("lower_temperature_limit", "double precision NOT NULL"); | ||
map.put("target_temperature", "double precision NOT NULL"); | ||
map.put("upper_temperature_limit", "double precision NOT NULL"); | ||
map.put("b", "double precision NOT NULL"); | ||
map.put("g", "double precision NOT NULL"); | ||
map.put("i_max", "double precision NOT NULL"); | ||
map.put("r", "double precision NOT NULL"); | ||
map.put("x", "double precision NOT NULL"); | ||
|
||
map.put("connected_assets", "TEXT NOT NULL"); | ||
map.put("capex", "double precision NOT NULL"); | ||
map.put("control_strategy", "TEXT NOT NULL"); | ||
|
||
map.put("input_model", "uuid NOT NULL"); | ||
map.put("soc", "double precision NOT NULL"); | ||
map.put("p_max", "double precision NOT NULL"); | ||
map.put("p_min", "double precision NOT NULL"); | ||
map.put("p_ref", "double precision NOT NULL"); | ||
|
||
map.put("dod", "double precision NOT NULL"); | ||
map.put("e_storage", "double precision NOT NULL"); | ||
map.put("eta", "double precision NOT NULL"); | ||
map.put("life_cycle", "double precision NOT NULL"); | ||
map.put("life_time", "double precision NOT NULL"); | ||
map.put("opex", "double precision NOT NULL"); | ||
map.put("active_power_gradient", "double precision NOT NULL"); | ||
|
||
return map; | ||
} | ||
|
||
public static String quote(String input, String quoteSymbol) { | ||
if (input == "") { | ||
return "NULL"; | ||
} else { | ||
return input.matches("^\".*\"$") ? input : quoteSymbol + input + quoteSymbol; | ||
} | ||
} | ||
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.