From efb672f32b48262fb9095bb55eea272f8bfa23cd Mon Sep 17 00:00:00 2001 From: Mustafa Yorukoglu Date: Fri, 25 Nov 2016 11:02:19 +0300 Subject: [PATCH 1/3] An option for the encoding of output files added. --- src/main/java/com/googlecode/scheme2ddl/Main.java | 10 ++++++++++ .../com/googlecode/scheme2ddl/UserObjectWriter.java | 10 +++++++++- src/main/resources/scheme2ddl.config.xml | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/googlecode/scheme2ddl/Main.java b/src/main/java/com/googlecode/scheme2ddl/Main.java index d0cffa8..b390117 100644 --- a/src/main/java/com/googlecode/scheme2ddl/Main.java +++ b/src/main/java/com/googlecode/scheme2ddl/Main.java @@ -26,6 +26,7 @@ public class Main { private static final Log log = LogFactory.getLog(Main.class); public static String outputPath = null; + public static String outputEncoding = null; public static int parallelCount = 4; private static boolean justPrintUsage = false; private static boolean justPrintVersion = false; @@ -97,6 +98,10 @@ private static void modifyContext(ConfigurableApplicationContext context) { UserObjectWriter writer = (UserObjectWriter) context.getBean("writer"); writer.setOutputPath(outputPath); } + if (outputEncoding != null) { + UserObjectWriter writer = (UserObjectWriter) context.getBean("writer"); + writer.setFileEncoding(outputEncoding); + } if (parallelCount > 0) { SimpleAsyncTaskExecutor taskExecutor = (SimpleAsyncTaskExecutor) context.getBean("taskExecutor"); taskExecutor.setConcurrencyLimit(parallelCount); @@ -250,6 +255,8 @@ private static void printUsage() { msg.append(" example: scott/tiger@localhost:1521:ORCL" + lSep); msg.append(" -o, --output, output dir" + lSep); + msg.append(" -oe, --out-encoding, out file encoding" + lSep); + msg.append(" example: UTF-8" + lSep); msg.append(" -p, --parallel, number of parallel thread (default 4)" + lSep); msg.append(" -s, --schemas, a comma separated list of schemas for processing" + lSep); msg.append(" (works only if connected to oracle as sysdba)" + lSep); @@ -295,6 +302,9 @@ private static void collectArgs(String[] args) throws Exception { } else if (arg.equals("-o") || arg.equals("-output") || arg.equals("--output")) { outputPath = args[i + 1]; i++; + } else if (arg.equals("-oe") || arg.equals("-out-encoding") || arg.equals("--out-encoding")) { + outputEncoding = args[i + 1]; + i++; } else if (arg.equals("-s") || arg.equals("-schemas") || arg.equals("--schemas")) { schemas = args[i + 1]; i++; diff --git a/src/main/java/com/googlecode/scheme2ddl/UserObjectWriter.java b/src/main/java/com/googlecode/scheme2ddl/UserObjectWriter.java index 919d2f6..6c60bb0 100644 --- a/src/main/java/com/googlecode/scheme2ddl/UserObjectWriter.java +++ b/src/main/java/com/googlecode/scheme2ddl/UserObjectWriter.java @@ -19,6 +19,7 @@ public class UserObjectWriter implements ItemWriter { private static final Log log = LogFactory.getLog(UserObjectWriter.class); private String outputPath; + private String fileEncoding; public void write(List data) throws Exception { @@ -31,7 +32,10 @@ public void writeUserObject(UserObject userObject) throws Exception { String absoluteFileName = outputPath + "/" + userObject.getFileName(); absoluteFileName = FilenameUtils.separatorsToSystem(absoluteFileName); File file = new File(absoluteFileName); - FileUtils.writeStringToFile(file, userObject.getDdl()); + if(fileEncoding==null) + FileUtils.writeStringToFile(file, userObject.getDdl()); + else + FileUtils.writeStringToFile(file, userObject.getDdl(), fileEncoding); log.info(String.format("Saved %s %s.%s to file %s", userObject.getType().toLowerCase(), userObject.getSchema().toLowerCase(), @@ -44,6 +48,10 @@ public void writeUserObject(UserObject userObject) throws Exception { public void setOutputPath(String outputPath) { this.outputPath = outputPath; } + + public void setFileEncoding(String fileEncoding) { + this.fileEncoding = fileEncoding; + } @Deprecated public void setFileNameCase(String fileNameCase) { diff --git a/src/main/resources/scheme2ddl.config.xml b/src/main/resources/scheme2ddl.config.xml index c7c0762..d268163 100644 --- a/src/main/resources/scheme2ddl.config.xml +++ b/src/main/resources/scheme2ddl.config.xml @@ -31,6 +31,7 @@ + From 3b5cfe751533df768322c20ce55bc6574fae36ef Mon Sep 17 00:00:00 2001 From: Mustafa Yorukoglu Date: Mon, 28 Nov 2016 13:47:04 +0300 Subject: [PATCH 2/3] To speedup schema ddl exports, last ddl time related filters added. --- .../java/com/googlecode/scheme2ddl/Main.java | 107 +++++++++++++++++- .../scheme2ddl/UserObjectJobRunner.java | 5 +- .../scheme2ddl/dao/UserObjectDaoImpl.java | 18 +++ src/main/resources/scheme2ddl.config.xml | 16 +++ 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/googlecode/scheme2ddl/Main.java b/src/main/java/com/googlecode/scheme2ddl/Main.java index b390117..388d1e4 100644 --- a/src/main/java/com/googlecode/scheme2ddl/Main.java +++ b/src/main/java/com/googlecode/scheme2ddl/Main.java @@ -17,6 +17,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * @author A_Reshetnikov @@ -43,6 +50,11 @@ public class Main { private static String objectFilter = "%"; private static String typeFilter = ""; private static String typeFilterMode = "include"; + private static String ddlTimeAfter = null ; + private static String ddlTimeBefore = null ; + private static String ddlTimeIn = null ; + private static DateFormat dfDateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static DateFormat dfDate = new SimpleDateFormat("yyyy-MM-dd"); public static void main(String[] args) throws Exception { typeFilterMode = "include"; //default is to include any type filter @@ -69,7 +81,13 @@ public static void main(String[] args) throws Exception { System.out.println("DDL object filter: " + objectFilter); System.out.println("DDL type filter: " + typeFilter); System.out.println("DDL type filter mode: " + typeFilterMode); - new UserObjectJobRunner().start(context, isLaunchedByDBA, objectFilter.toLowerCase(), typeFilter.toUpperCase(), typeFilterMode.toLowerCase()); + if (ddlTimeAfter != null) + System.out.println("Last DDL time after: " + ddlTimeAfter); + if (ddlTimeBefore != null) + System.out.println("Last DDL time before: " + ddlTimeBefore); + if (ddlTimeIn != null) + System.out.println("DDL time in last : " + ddlTimeIn); + new UserObjectJobRunner().start(context, isLaunchedByDBA, objectFilter.toLowerCase(), typeFilter.toUpperCase(), typeFilterMode.toLowerCase(), ddlTimeAfter, ddlTimeBefore, ddlTimeIn); } } @@ -112,6 +130,32 @@ private static void modifyContext(ConfigurableApplicationContext context) { ConnectionDao connectionDao = (ConnectionDao) context.getBean("connectionDao"); isLaunchedByDBA = connectionDao.hasSelectCatalogRole(); //todo rename isLaunchedByDBA -> processForeignSchema } + + Map ddlTimeFilterMap = (Map)context.getBean("ddlTimeFilterMap"); + if (ddlTimeAfter == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeAfter")){ + String da = (String)ddlTimeFilterMap.get("ddlTimeAfter"); + if (da != null) + ddlTimeAfter = da; + } + if (ddlTimeBefore == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeBefore")){ + String df = (String)ddlTimeFilterMap.get("ddlTimeBefore"); + if (df != null) + ddlTimeBefore = df; + } + if (ddlTimeIn == null && ddlTimeFilterMap != null && ddlTimeFilterMap instanceof Map && ddlTimeFilterMap.containsKey("ddlTimeIn")){ + Map dim = (Map) ddlTimeFilterMap.get("ddlTimeIn"); + if (dim != null && dim instanceof Map && dim.containsKey("n") && dim.containsKey("unit")){ + String unit = dim.get("unit"); + String n = dim.get("n"); + + if ("day".equals(unit)) + ddlTimeIn = n + " /*day*/"; + if ("hour".equals(unit)) + ddlTimeIn = n + " /*hour*/ /24"; + if ("minute".equals(unit)) + ddlTimeIn = n + " /*minute*/ /(24*60)"; + } + } //process schemas processSchemas(context); @@ -265,6 +309,14 @@ private static void printUsage() { msg.append(" every LIKE wildcard can be used" + lSep); msg.append(" -tf, --type-filter, filter for specific DDL object types" + lSep); msg.append(" -tfm, --type-filtermode, mode for type filter: include(default) or exclude" + lSep); + msg.append(" -dta, --ddl-time-after, export objects with last DDL time after the time specified" + lSep); + msg.append(" example: 2016-11-27 : exports objects that modified after 27 Nov 2016" + lSep); + msg.append(" example: \"2016-11-27 01:00:00\" : exports objects that modified after 27 Nov 2016, 1am" + lSep); + msg.append(" -dtb, --ddl-time-before, export objects with last DDL time before the time specified" + lSep); + msg.append(" example: \"2016-11-27 01:00:00\"" + lSep); + msg.append(" -dti, --ddl-time-in, export objects with last DDL time in last n[minute|hour|day]" + lSep); + msg.append(" example: \"6 hour\" : exports objects that modified in last 6 hours" + lSep); + msg.append(" example: 2d : exports objects that modified in last 2 days" + lSep); msg.append(" --stop-on-warning, stop on getting DDL error (skip by default)" + lSep); msg.append(" -rsv, replace actual sequence values with 1 " + lSep); msg.append(" --replace-sequence-values, " + lSep); @@ -313,6 +365,59 @@ private static void collectArgs(String[] args) throws Exception { i++; } else if (arg.equals("-tc") || arg.equals("--test-connection")) { justTestConnection = true; + } else if (arg.equals("-dta") || arg.equals("--ddl-time-after") || arg.equals("-ddl-time-after")) { + ddlTimeAfter = args[i + 1]; + i++; + Date d = null; + try{ + d = dfDateTime.parse(ddlTimeAfter); + ddlTimeAfter = dfDateTime.format(d); + }catch(ParseException pe){ + try{ + d = dfDate.parse(ddlTimeAfter); + ddlTimeAfter = dfDateTime.format(d); + }catch(ParseException pe2){ + System.err.println("Invalid parameter format for " + arg); + printUsage(); + throw new Exception(""); + } + } + } else if (arg.equals("-dtb") || arg.equals("--ddl-time-before") || arg.equals("-ddl-time-before")) { + ddlTimeBefore = args[i + 1]; + i++; + Date d = null; + try{ + d = dfDateTime.parse(ddlTimeBefore); + ddlTimeBefore = dfDateTime.format(d); + }catch(ParseException pe){ + try{ + d = dfDate.parse(ddlTimeBefore); + ddlTimeBefore = dfDateTime.format(d); + }catch(ParseException pe2){ + System.err.println("Invalid parameter format for " + arg); + printUsage(); + throw new Exception(""); + } + } + } else if (arg.equals("-dti") || arg.equals("--ddl-time-in") || arg.equals("-ddl-time-in")) { + String param = args[i + 1]; + i++; + Pattern pattern = Pattern.compile("^(\\d+)\\s?(minute|min|m|hour|h|day|d)?"); + Matcher m = pattern.matcher(param); + if (m.find()){ + String unit = m.group(2); + String n = m.group(1); + if ("day".equals(unit) || "d".equals(unit) || unit == null) + ddlTimeIn = n + " /*day*/ "; + if ("hour".equals(unit) || "h".equals(unit)) + ddlTimeIn = n + " /*hour*/ /24"; + if ("minute".equals(unit) || "min".equals(unit) || "m".equals(unit)) + ddlTimeIn = n + " /*minute*/ /(24*60)"; + }else{ + System.err.println("Invalid parameter format for " + arg); + printUsage(); + throw new Exception(""); + } } else if (arg.equals("--stop-on-warning")) { stopOnWarning = true; } else if ((arg.equals("-rsv") || arg.equals("--replace-sequence-values"))) { diff --git a/src/main/java/com/googlecode/scheme2ddl/UserObjectJobRunner.java b/src/main/java/com/googlecode/scheme2ddl/UserObjectJobRunner.java index 39d1de1..82af624 100644 --- a/src/main/java/com/googlecode/scheme2ddl/UserObjectJobRunner.java +++ b/src/main/java/com/googlecode/scheme2ddl/UserObjectJobRunner.java @@ -20,7 +20,7 @@ public class UserObjectJobRunner { protected static final Log logger = LogFactory.getLog(UserObjectJobRunner.class); private JobLauncher launcher; - int start(ConfigurableApplicationContext context, boolean launchedByDBA, String objectFilter, String typeFilter, String typeFilterMode) throws Exception { + int start(ConfigurableApplicationContext context, boolean launchedByDBA, String objectFilter, String typeFilter, String typeFilterMode, String ddlTimeAfter, String ddlTimeBefore, String ddlTimeIn) throws Exception { try { context.getAutowireCapableBeanFactory().autowireBeanProperties(this, AutowireCapableBeanFactory.AUTOWIRE_BY_TYPE, false); @@ -40,6 +40,9 @@ int start(ConfigurableApplicationContext context, boolean launchedByDBA, String parametersBuilder.addString("objectFilter", objectFilter); parametersBuilder.addString("typeFilter", typeFilter); parametersBuilder.addString("typeFilterMode", typeFilterMode); + parametersBuilder.addString("ddlTimeAfter", ddlTimeAfter); + parametersBuilder.addString("ddlTimeBefore", ddlTimeBefore); + parametersBuilder.addString("ddlTimeIn", ddlTimeIn); JobParameters jobParameters = parametersBuilder.toJobParameters(); logger.trace(String.format("Start spring batch job with parameters %s", jobParameters)); JobExecution jobExecution = launcher.run(job, jobParameters); diff --git a/src/main/java/com/googlecode/scheme2ddl/dao/UserObjectDaoImpl.java b/src/main/java/com/googlecode/scheme2ddl/dao/UserObjectDaoImpl.java index 3ca7a7b..5e61925 100644 --- a/src/main/java/com/googlecode/scheme2ddl/dao/UserObjectDaoImpl.java +++ b/src/main/java/com/googlecode/scheme2ddl/dao/UserObjectDaoImpl.java @@ -36,6 +36,12 @@ public class UserObjectDaoImpl extends JdbcDaoSupport implements UserObjectDao { private String typeFilter; @Value("#{jobParameters['typeFilterMode']}") private String typeFilterMode = "include"; + @Value("#{jobParameters['ddlTimeAfter']}") + private String ddlTimeAfter = null; + @Value("#{jobParameters['ddlTimeBefore']}") + private String ddlTimeBefore = null; + @Value("#{jobParameters['ddlTimeIn']}") + private String ddlTimeIn = null; public List findListForProccessing() { String sql; @@ -56,6 +62,12 @@ public List findListForProccessing() { sql += " IN (" + typeFilter + ") "; } + if (ddlTimeAfter != null) + sql += " and t.last_ddl_time >= TO_DATE('" + ddlTimeAfter + "', 'YYYY-MM-DD HH24:MI:SS') "; + if (ddlTimeBefore != null) + sql += " and t.last_ddl_time <= TO_DATE('" + ddlTimeBefore + "', 'YYYY-MM-DD HH24:MI:SS') "; + if (ddlTimeIn != null) + sql += " and t.last_ddl_time >= SYSDATE-" + ddlTimeIn + " "; if (isTypeAllowed("'REFRESH GROUP'")) { sql += " UNION ALL " + " select rname as object_name, 'REFRESH_GROUP' as object_type " + @@ -79,6 +91,12 @@ public List findListForProccessing() { sql += " IN (" + typeFilter + ") "; } + if (ddlTimeAfter != null) + sql += " and t.last_ddl_time >= TO_DATE('" + ddlTimeAfter + "', 'YYYY-MM-DD HH24:MI:SS') "; + if (ddlTimeBefore != null) + sql += " and t.last_ddl_time <= TO_DATE('" + ddlTimeBefore + "', 'YYYY-MM-DD HH24:MI:SS') "; + if (ddlTimeIn != null) + sql += " and t.last_ddl_time >= SYSDATE-" + ddlTimeIn + " "; if (isTypeAllowed("'REFRESH GROUP'")) { sql += " UNION ALL " + " select rname as object_name, 'REFRESH_GROUP' as object_type " + diff --git a/src/main/resources/scheme2ddl.config.xml b/src/main/resources/scheme2ddl.config.xml index d268163..70e2bd5 100644 --- a/src/main/resources/scheme2ddl.config.xml +++ b/src/main/resources/scheme2ddl.config.xml @@ -21,6 +21,22 @@ + + + + + + + + + + From e2aef3617228befdaa21c2d41b4fb2f01b9d30e6 Mon Sep 17 00:00:00 2001 From: Mustafa Yorukoglu Date: Mon, 28 Nov 2016 13:53:04 +0300 Subject: [PATCH 3/3] README.md file updated for new parameters added. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 7217341..92e06a5 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ More command line options -url, DB connection URL example: scott/tiger@localhost:1521:ORCL -o, --output, output dir + -oe, --out-encoding, out file encoding + example: UTF-8 -p, --parallel, number of parallel thread (default 4) -s, --schemas, a comma separated list of schemas for processing (works only if connected to oracle as sysdba) @@ -51,6 +53,14 @@ More command line options every LIKE wildcard can be used -tf, --type-filter, filter for specific DDL object types -tfm, --type-filtermode, mode for type filter: include(default) or exclude + -dta, --ddl-time-after, export objects with last DDL time after the time specified + example: 2016-11-27 : exports objects that modified after 27 Nov 2016 + example: "2016-11-27 01:00:00" : exports objects that modified after 27 Nov 2016, 1am + -dtb, --ddl-time-before, export objects with last DDL time before the time specified + example: "2016-11-27 01:00:00" + -dti, --ddl-time-in, export objects with last DDL time in last n[minute|hour|day] + example: "6 hour" : exports objects that modified in last 6 hours + example: 2d : exports objects that modified in last 2 days --stop-on-warning, stop on getting DDL error (skip by default) -rsv, replace actual sequence values with 1 --replace-sequence-values,