1 package com.rapide_act;
3 import java.io.BufferedInputStream;
4 import java.io.BufferedReader;
5 import java.io.BufferedWriter;
7 import java.io.FileInputStream;
8 import java.io.FileOutputStream;
9 import java.io.FileReader;
10 import java.io.FileWriter;
11 import java.io.InputStream;
12 import java.io.InputStreamReader;
13 import java.io.IOException;
14 import java.io.OutputStreamWriter;
15 import java.io.PrintWriter;
16 import java.util.ArrayList;
17 import java.util.Date;
18 import java.util.Iterator;
19 import java.util.Properties;
22 import java.sql.Connection;
23 import java.sql.DriverManager;
24 import java.sql.PreparedStatement;
25 import java.sql.ResultSet;
26 import java.sql.ResultSetMetaData;
27 import java.sql.SQLException;
28 import java.sql.Timestamp;
30 public class RapideUnloader{
32 private String database = null;
33 private String driver = null;
34 private String user = null;
35 private String pass = null;
36 private String url = null;
37 private String sql_table_key_list;
38 private static final int DB_TYPE_ORACLE = 0;
39 private static final int DB_TYPE_SQLSERVER = 1;
40 private static final int DB_TYPE_DB2 = 2;
41 private static final int DB_TYPE_MYSQL = 3;
42 private static final int DB_TYPE_POSTGRESQL = 4;
43 private static final int DB_TYPE_UNKNOWN = -1;
44 private static final int MASK_PTN_ALL = 0;
45 private static final int MASK_PTN_ALT = 1;
46 private static final int MASK_PTN_EDGE = 2;
47 private static final int D_USER = 0;
48 private static final int D_PASS = 1;
49 private static final int D_URL = 2;
50 private static final int D_DRIVER = 3;
51 private static final int D_OUTPUT = 4;
52 private static final int D_INPUT = 5;
53 private static final int D_LINESEP = 6;
54 private static final int D_INLINESEP = 7;
55 private static final int D_FILENCODE = 8;
56 private static final int D_NULLMARK = 9;
57 private static final int D_DELIMITER = 10;
58 private static final int D_TBLSQL = 11;
59 private static final int D_TBLKEYSQL = 12;
60 private static final int D_ORDER = 13;
61 private static final int D_QUOTATION = 14;
62 private static final int D_BATCOUNT = 15;
63 private static final int D_INCLTAB = 16;
64 private static final int D_EXCLTAB = 17;
65 private static final int D_INCLCOL = 18;
66 private static final int D_EXCLCOL = 19;
67 private static final int D_INCLCOLMASK = 20;
68 private static final int D_EXCLCOLMASK = 21;
69 private static final int D_SBYTEMASKCHR = 22;
70 private static final int D_DBYTEMASKCHR = 23;
71 private static final int D_MASKPTN = 24;
72 private static final int D_PROPFILE = 25;
73 private static final int KEYS = 26;
74 private static final String [] DB_TYPE_NAME = {"ORACLE", "SQLSERVER", "DB2", "MYSQL", "POSTGRESQL"};
75 private static final String [] MASK_PTN = {"ALL", "ALT", "EDGE"};
76 private static final String [] PROP_KEY = { "us","pw","url","dr","out","in","ls","ils","fe","nm",
77 "dm","ts","tks","od","qt","bc","it","et","ic","ec",
78 "icm","ecm","smc","dmc","mp","pf"};
79 private static final String DEFAULT_FOLDER = "output";
80 private static final String DEFAULT_PROP_FILE = "RapideLoader.properties";
82 private String nullMark = "";
83 private boolean isOrder = false;
84 private boolean isQuotation = true;
85 private boolean isInclTables = false;
86 private boolean isExclTables = false;
87 private boolean isInclCols = false;
88 private boolean isExclCols = false;
89 private boolean isMask = false;
90 private boolean isInclColsMask = false;
91 private boolean isExclColsMask = false;
92 private String delimiter = "\t";
93 private String lineSeparator = System.getProperty("line.separator");
94 private String inColLineSeparator = System.getProperty("line.separator");
95 private String outFolder = null;
96 private String inFolder = null;
97 private String inclTables = null;
98 private String exclTables = null;
99 private String inclCols = null;
100 private String exclCols = null;
101 private String inclMaskCols = null;
102 private String exclMaskCols = null;
103 private String [] inclTablesArray = null;
104 private String [] exclTablesArray = null;
105 private String [] inclColsArray = null;
106 private String [] exclColsArray = null;
107 private String [] inclColsMaskArray = null;
108 private String [] exclColsMaskArray = null;
109 private char singleByteMaskChar = '*';
110 private char doubleByteMaskChar = '\u25A0';
111 private String maskPattern = "ALT";
112 private String fileEncoding = System.getProperty("file.encoding");
114 private String [] prop_val = new String[KEYS];
117 private static String [] sql_table_key_list_array = {
121 + " from user_tables a,"
122 + " (select b.table_name, "
125 + " from user_constraints b, "
126 + " user_cons_columns c "
127 + " where b.CONSTRAINT_TYPE = 'P' and "
128 + " b.TABLE_NAME = c.TABLE_NAME and "
129 + " b.CONSTRAINT_NAME = c.CONSTRAINT_NAME "
131 + "where a.table_name = d.table_name(+) "
132 + "order by a.table_name, d.position",
134 + " A.name AS table_name, "
135 + " D.name AS col_name "
136 + "FROM sys.tables AS A "
137 + "LEFT OUTER JOIN sys.key_constraints AS B "
138 + "ON A.object_id = B.parent_object_id "
139 + " AND B.type = 'PK' "
140 + "LEFT OUTER JOIN sys.index_columns AS C "
141 + "ON B.parent_object_id = C.object_id "
142 + " AND B.unique_index_id = C.index_id "
143 + "LEFT OUTER JOIN sys.columns AS D "
144 + "ON C.object_id = D.object_id "
145 + " AND C.column_id = D.column_id "
146 + "order by A.name,C.key_ordinal"
149 public static void main(String args[]){
151 if (args.length > 0){
152 RapideUnloader rapideUnloader = new RapideUnloader(args[0]);
153 rapideUnloader.unload();
155 RapideUnloader rapideUnloader = new RapideUnloader(null);
156 rapideUnloader.unload();
158 } catch (Exception e) {
163 RapideUnloader(String _database) throws Exception{
165 database = _database;
169 private void getProperties(String _propFile) throws Exception{
170 BufferedReader br = null;
171 Properties prop = null;
173 prop = new Properties();
174 br = new BufferedReader(new InputStreamReader(getClass().getResourceAsStream(_propFile), "UTF-8"));
176 for(int i=0;i<KEYS;i++){
177 prop_val[i] = prop.getProperty(database + "." +PROP_KEY[i]);
181 } catch (NullPointerException e) {
182 throw new Exception("Property File (" + _propFile +") Not Found");
183 } catch (IOException e) {
193 catch (IOException e) {
199 private void unload() throws Exception{
200 Connection conn = null;
201 PreparedStatement stmt = null;
202 ResultSet rst = null;
203 String sql_unload = null;
204 String sql_insert = null;
205 String order_columns = "";
207 int dbType = DB_TYPE_UNKNOWN;
209 PrintWriter pw = null;
211 ArrayList<String> alColName = null;
212 ArrayList<String> alColTypeName = null;
213 ArrayList<Integer> alIsNullable = null;
214 ArrayList<Boolean> alColMask = null;
215 ArrayList<Boolean> alColIncl = null;
216 ArrayList<String> alData = null;
218 int tb_col_count = 0;
222 if(database != null){
223 if(!CmnUtils.isEmpty(prop_val[D_PROPFILE])){
224 getProperties(System.getProperty(prop_val[D_PROPFILE]));
226 getProperties(DEFAULT_PROP_FILE);
230 for(int i=0;i<KEYS;i++){
231 if(System.getProperty(PROP_KEY[i])!=null)prop_val[i] = System.getProperty(PROP_KEY[i]);
232 CmnUtils.debugPrint(PROP_KEY[i] + "=" + prop_val[i]);
235 if(!CmnUtils.isEmpty(prop_val[D_USER]))user = prop_val[D_USER];
236 if(!CmnUtils.isEmpty(prop_val[D_PASS]))pass = prop_val[D_PASS];
237 if(!CmnUtils.isEmpty(prop_val[D_URL]))url = prop_val[D_URL];
238 if(!CmnUtils.isEmpty(prop_val[D_DRIVER]))driver = prop_val[D_DRIVER];
239 if(!CmnUtils.isEmpty(prop_val[D_ORDER]) && prop_val[D_ORDER].toUpperCase().equals("Y"))isOrder = true;
240 if(!CmnUtils.isEmpty(prop_val[D_QUOTATION]) && prop_val[D_QUOTATION].toUpperCase().equals("N"))isQuotation = false;
241 if(!CmnUtils.isEmpty(prop_val[D_INCLTAB]))isInclTables = true;
242 if(!CmnUtils.isEmpty(prop_val[D_EXCLTAB]))isExclTables = true;
243 if(!CmnUtils.isEmpty(prop_val[D_INCLCOL]))isInclCols = true;
244 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOL]))isExclCols = true;
245 if(!CmnUtils.isEmpty(prop_val[D_INCLCOLMASK]))isInclColsMask = true;
246 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOLMASK]))isExclColsMask = true;
247 if(isInclColsMask || isExclColsMask)isMask = true;
248 if(!CmnUtils.isEmpty(prop_val[D_DELIMITER]))delimiter = CmnUtils.getDelimiter(prop_val[D_DELIMITER]);
249 if(!CmnUtils.isEmpty(prop_val[D_LINESEP]))lineSeparator = CmnUtils.getLineSeparator(prop_val[D_LINESEP]);
250 if(!CmnUtils.isEmpty(prop_val[D_OUTPUT]))outFolder = prop_val[D_OUTPUT];
251 if(!CmnUtils.isEmpty(prop_val[D_INCLTAB]))inclTables = prop_val[D_INCLTAB].toUpperCase();
252 if(!CmnUtils.isEmpty(prop_val[D_EXCLTAB]))exclTables = prop_val[D_EXCLTAB].toUpperCase();
253 if(!CmnUtils.isEmpty(prop_val[D_INCLCOL]))inclCols = prop_val[D_INCLCOL].toUpperCase();
254 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOL]))exclCols = prop_val[D_EXCLCOL].toUpperCase();
255 if(!CmnUtils.isEmpty(prop_val[D_INCLCOLMASK]))inclMaskCols = prop_val[D_INCLCOLMASK].toUpperCase();
256 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOLMASK]))exclMaskCols = prop_val[D_EXCLCOLMASK].toUpperCase();
257 if(!CmnUtils.isEmpty(prop_val[D_INCLTAB]))inclTablesArray = CmnUtils.splitCsv(prop_val[D_INCLTAB].toUpperCase());
258 if(!CmnUtils.isEmpty(prop_val[D_EXCLTAB]))exclTablesArray = CmnUtils.splitCsv(prop_val[D_EXCLTAB].toUpperCase());
259 if(!CmnUtils.isEmpty(prop_val[D_INCLCOL]))inclColsArray = CmnUtils.splitCsv(prop_val[D_INCLCOL].toUpperCase());
260 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOL]))exclColsArray = CmnUtils.splitCsv(prop_val[D_EXCLCOL].toUpperCase());
261 if(!CmnUtils.isEmpty(prop_val[D_INCLCOLMASK]))inclColsMaskArray = CmnUtils.splitCsv(prop_val[D_INCLCOLMASK].toUpperCase());
262 if(!CmnUtils.isEmpty(prop_val[D_EXCLCOLMASK]))exclColsMaskArray = CmnUtils.splitCsv(prop_val[D_EXCLCOLMASK].toUpperCase());
263 if(!CmnUtils.isEmpty(prop_val[D_SBYTEMASKCHR]))singleByteMaskChar = prop_val[D_SBYTEMASKCHR].charAt(0);
264 if(!CmnUtils.isEmpty(prop_val[D_DBYTEMASKCHR]))doubleByteMaskChar = prop_val[D_DBYTEMASKCHR].charAt(0);
265 if(!CmnUtils.isEmpty(prop_val[D_MASKPTN]))maskPattern = prop_val[D_MASKPTN];
266 if(!CmnUtils.isEmpty(prop_val[D_FILENCODE]))fileEncoding = prop_val[D_FILENCODE];
269 throw new Exception("user is null");
270 } else if (pass == null){
271 throw new Exception("password is null");
272 } else if (url == null){
273 throw new Exception("url is null");
274 } else if (driver == null){
275 throw new Exception("driver is null");
278 for(int i = 0;i<DB_TYPE_NAME.length;i++){
279 if (url.toUpperCase().contains(DB_TYPE_NAME[i])){
285 if (dbType != DB_TYPE_UNKNOWN){
286 if(!CmnUtils.isEmpty(prop_val[D_TBLKEYSQL])){
287 sql_table_key_list = prop_val[D_TBLKEYSQL];
289 sql_table_key_list = sql_table_key_list_array[dbType];
292 throw new Exception("dbtype unknown");
295 DataAccessObjects dao = new DataAccessObjects(user, pass, url, driver);
296 dao.setPropVal(prop_val);
298 conn = dao.connect();
300 if(outFolder != null){
301 fldr = new File(outFolder);
303 if(database != null){
304 fldr = new File(DEFAULT_FOLDER + "/" + database.toUpperCase() + "_" + CmnUtils.getYmdhm());
306 fldr = new File(DEFAULT_FOLDER + "/" + DB_TYPE_NAME[dbType].toUpperCase() + "_" + CmnUtils.getYmdhm());
311 if(database!=null)CmnUtils.infoPrint("-->Database='" + database.toUpperCase() + "'");
312 CmnUtils.infoPrint("-->Output='" + fldr + "'");
313 if(!CmnUtils.isEmpty(prop_val[D_DELIMITER]))CmnUtils.infoPrint("-->Delimiter='" + prop_val[D_DELIMITER] + "'");
314 if(!CmnUtils.isEmpty(prop_val[D_LINESEP]))CmnUtils.infoPrint("-->LineSeparator='" + prop_val[D_LINESEP] + "'");
315 if(!CmnUtils.isEmpty(prop_val[D_FILENCODE]))CmnUtils.infoPrint("-->FileEncoding='" + prop_val[D_FILENCODE] + "'");
316 if(!CmnUtils.isEmpty(prop_val[D_QUOTATION]))CmnUtils.infoPrint("-->Quotation=" + CmnUtils.getYesNo(isQuotation));
317 if(!CmnUtils.isEmpty(prop_val[D_ORDER]))CmnUtils.infoPrint("-->Sort=" + CmnUtils.getYesNo(isOrder));
318 if(isInclTables)CmnUtils.infoPrint("-->Include Tables='" + inclTables.toUpperCase() + "'");
319 if(isExclTables)CmnUtils.infoPrint("-->Exclude Tables='" + exclTables.toUpperCase() + "'");
320 if(isInclCols)CmnUtils.infoPrint("-->Include Columns='" + inclCols.toUpperCase() + "'");
321 if(isExclCols)CmnUtils.infoPrint("-->Exclude Columns='" + exclCols.toUpperCase() + "'");
323 CmnUtils.infoPrint("-->Masking=" + CmnUtils.getYesNo(isMask));
324 CmnUtils.infoPrint("-->Masking Pattern=" + maskPattern.toUpperCase());
325 if(isInclColsMask)CmnUtils.infoPrint("-->Masking Columns='" + inclMaskCols.toUpperCase() + "'");
326 if(isExclColsMask)CmnUtils.infoPrint("-->Exclude Masking Columns='" + exclMaskCols.toUpperCase() + "'");
327 CmnUtils.infoPrint("-->Single Byte Mask Char='" + singleByteMaskChar + "'");
328 CmnUtils.infoPrint("-->Double Byte Mask Char='" + doubleByteMaskChar + "'");
330 CmnUtils.infoPrint("Start DataUnload ---------------------------------------------");
332 dao.select(sql_table_key_list);
333 alData = dao.getArrayList();
334 tb_count = dao.getColumnCount();
337 CmnUtils.debugPrint("tables=" + alData.size());
338 for(int i=0;i<alData.size();i++){
339 if ((i+1) % tb_count == 1){
340 if ((i + 2 <alData.size()) && (!alData.get(i).equals(alData.get(i+2))) || (i + 2 == alData.size())) {
341 if(isInclTables && !CmnUtils.isMatch(inclTablesArray,alData.get(i).toUpperCase())){
342 CmnUtils.debugPrint("inclTables=" + inclTables);
343 CmnUtils.debugPrint("table=" + alData.get(i) + "," +CmnUtils.isMatch(inclTablesArray,alData.get(i).toUpperCase()));
348 if(isExclTables && CmnUtils.isMatch(exclTablesArray,alData.get(i).toUpperCase())){
349 CmnUtils.debugPrint("exclTables=" + exclTables);
350 CmnUtils.debugPrint("table=" + alData.get(i) + "," +CmnUtils.isMatch(exclTablesArray,alData.get(i).toUpperCase()));
355 if (!alData.get(i+1).equals("")){
358 if (!alData.get(i+1).equals(""))order_columns = alData.get(i+1);
360 order_columns += "," + alData.get(i+1);
366 if(!order_columns.equals("")){
367 sql_unload = "select * from " + alData.get(i) + " order by " + order_columns;
369 sql_unload = "select * from " + alData.get(i);
372 sql_unload = "select * from " + alData.get(i);
374 CmnUtils.debugPrint("i=" + i);
375 CmnUtils.debugPrint(sql_unload);
376 stmt = dao.prepareSql(sql_unload);
377 dao.executeSql(stmt);
378 alColName = dao.getArrayColumnNameList();
379 alColTypeName = dao.getArrayColumnTypeNameList();
380 alColMask = new ArrayList<Boolean>();
381 alColIncl = new ArrayList<Boolean>();
382 tb_col_count = dao.getColumnCount();
383 for(int j=0;j<alColName.size();j++){
384 if(CmnUtils.isColString(alColTypeName.get(j))){
389 CmnUtils.isMatch(inclColsMaskArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase())
393 CmnUtils.isMatch(inclColsMaskArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase()) &&
394 !CmnUtils.isMatch(exclColsMaskArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase())
398 !CmnUtils.isMatch(exclColsMaskArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase())
401 CmnUtils.debugPrint("Mask Column=" + alColName.get(j).toUpperCase());
404 alColMask.add(false);
407 alColMask.add(false);
410 alColMask.add(false);
412 if(isInclCols || isExclCols){
416 CmnUtils.isMatch(inclColsArray,alData.get(i).toUpperCase(), alColName.get(j).toUpperCase())
420 CmnUtils.isMatch(inclColsArray,alData.get(i).toUpperCase(), alColName.get(j).toUpperCase()) &&
421 !CmnUtils.isMatch(exclColsArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase())
425 !CmnUtils.isMatch(exclColsArray,alData.get(i).toUpperCase(),alColName.get(j).toUpperCase())
428 CmnUtils.debugPrint("column=" + alColName.get(j).toUpperCase());
431 alColIncl.add(false);
437 Boolean [] colIncl = (Boolean[])alColIncl.toArray(new Boolean[0]);
440 for(firstCol=0;firstCol<colIncl.length-1;firstCol++){
441 if(colIncl[firstCol]){
445 for(lastCol=colIncl.length-1;lastCol >= 0;lastCol--){
446 if(colIncl[lastCol]){
450 CmnUtils.debugPrint("firstCol=" + firstCol);
451 CmnUtils.debugPrint("lastCol=" + lastCol);
453 dao.setTableName(alData.get(i));
454 dao.setArrayColumnInclList(alColIncl);
455 dao.setArrayColumnMaskList(alColMask);
458 pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fldr + "/" + alData.get(i) + ".csv"), fileEncoding)));
459 CmnUtils.infoPrinting(String.format("%1$-30s",alData.get(i)));
461 for(int j=0;j<alColName.size();j++){
464 pw.print("\"" + alColName.get(j));
466 pw.print(alColName.get(j));
469 if(colIncl[j])pw.print(alColName.get(j));
471 if (firstCol == lastCol){
473 pw.print("\"" + lineSeparator);
475 pw.print("" + lineSeparator);
480 pw.print("\"" + lineSeparator);
482 pw.print("" + lineSeparator);
487 pw.print("\"" + delimiter + "\"");
495 dao.getRecordToPrint(pw,firstCol,lastCol);
496 dao.closeRecordSet();
499 System.out.println(String.format("%1$10d",dao.getRecCount()) + " rows exported");
504 if (!alData.get(i+1).equals("")){
507 if (!alData.get(i+1).equals(""))order_columns = alData.get(i+1);
509 order_columns += "," + alData.get(i+1);
522 } catch (Exception e) {
530 CmnUtils.infoPrint("End DataUnload -----------------------------------------------");