SuperX
/
kern
7
0
Fork 0
Code Issues 15 Pull Requests Projects Releases Wiki Activity
SuperX-Kernmodul
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
180 Commits
10 Branches
0 Tags
1.4 GiB
 
 
 
 
 
 
Tree: 623f94f8da
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '623f94f8da'
${ noResults }
kern/src/de/superx/bin/DataProfiler.java

674 lines
34 KiB
Raw Blame History

package de.superx.bin;
import static de.superx.servlet.SxSQL_Server.DEFAULT_MANDANTEN_ID;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.JDBCType;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import javax.sql.DataSource;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.apache.poi.xssf.usermodel.XSSFDataFormat;
import org.apache.poi.xssf.usermodel.XSSFFont;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
import de.superx.servlet.SuperXManager;
import de.superx.servlet.SxPools;
import de.superx.spring.AppConfig;
import de.superx.spring.batch.His1DataSources;
import de.superx.spring.cli.config.CLIConfig;
import de.superx.spring.config.DataJdbcConfiguration;
import de.superx.spring.config.ServiceConfig;
/**
* A utility for creating data profiling statistics for tables in a database
* to be used in Data Warehouse Design.
* This can be used as a command line utility or embedded in an application.
*/
public class DataProfiler {
private final static String SQL_COUNT_NULL = "select count(*) from %s where %s is null";
private final static String SQL_PERCENT_UNIQUE = "select\n"
+ " count(distinct %s) as unique_anz,\n"
+ " count(distinct %s)::float / count(*) * 100 as unique_percentage\n"
+ "from\n"
+ " %s;";
private final static String SQL_RANKING = "select %s, count(*) as anz from %s where %s is not null group by %s order by 2 desc limit 10;";
private final static String SQL_MIN_MAX_LEN = "select min(length(%s)) as min_length, max(length(%s)) as max_length\n"
+ "from %s\n"
+ "where %s is not null";
private final static String SQL_COUNT_VALUE = "select count(*) from %s where %s = %s";
private final static String SQL_MIN_MAX_AVG = "select min(%s), max(%s), avg(%s) from %s where %s is not null;";
private final static String SQL_START_END = "select min(%s), max(%s) from %s where %s is not null";
private static GenericApplicationContext APPLICATION_CONTEXT = null;
private static String HELP_STRING = "Use this tool to profile database tables for dwh design. "
+ "It needs the config file 'his1_databases.properties' inside the classpath; "
+ "this file gets written automatically when starting the web application.";
static Logger logger = Logger.getLogger(DataProfiler.class);
private DataSource dataSource;
private String database;
private String schema;
private String[] tables;
/**
* Instantiate a new DataProfiler.
* @param dataSource The DataSource from which to read the table statistics.
* @param schema The schema from which to read. If null public is assumed.
* @param tables An Array of the names of the tables for which statistics should be created.
*/
public DataProfiler(DataSource dataSource, String schema, String[] tables) {
try (Connection con = dataSource.getConnection()) {
this.database = con.getCatalog();
} catch (SQLException e) {
logger.error("Couldn't read catalog", e);
}
this.schema = schema != null ? schema : "public";
List<String> tableList = Arrays.asList(tables);
// TODO: Make sorting configurable?
tableList.sort(null);
this.tables = tableList.toArray(new String[] {});
this.dataSource = dataSource;
}
public static void main(String[] args) {
System.setProperty(SuperXManager.SUPER_X_HISINONE_VERSION, "non-empty-value");
Options options = createOptions();
CommandLine parsedArgs = parseArgs(args, options);
if (parsedArgs.hasOption("h")) {
printHelp(options);
System.exit(0);
}
String database = null;
String schema = "public";
String[] tables = null;
if (parsedArgs.hasOption("d")) {
database = parsedArgs.getOptionValue('d');
}
if (parsedArgs.hasOption("s")) {
schema = parsedArgs.getOptionValue('s');
}
if (parsedArgs.hasOption("t")) {
tables = parsedArgs.getOptionValues('t');
}
if (!parsedArgs.hasOption('d') || !parsedArgs.hasOption('t')) {
printHelp(options);
System.exit(0);
}
try (GenericApplicationContext context = createContext()) {
initSxPools();
DataSource dataSource = context.getBean(His1DataSources.class).get(database);
DataProfiler profiler = new DataProfiler(dataSource, schema, tables);
profiler.outputExcel(profiler.createStatistics(), null);
}
}
/**
* Create the List of TableStatistics.
* @return List of TableStatistics.
*/
public List<TableStatistic> createStatistics() {
List<TableStatistic> tableStats = new ArrayList<>();
try {
JdbcTemplate jt = new JdbcTemplate(dataSource);
logger.info("Database: " + database);
logger.info("Schema: " + schema);
logger.info("Tables: " + Arrays.asList(tables));
try (Connection con = dataSource.getConnection()) {
jt.execute("set search_path to " + schema);
DatabaseMetaData meta = con.getMetaData();
for (String table : tables) {
long rowCount = jt.queryForObject("select count(*) from " + table, Long.class).longValue();
TableStatistic tableStat = new TableStatistic(table, rowCount);
logger.info("Table " + table);
try(ResultSet columns = meta.getColumns(null, schema, table, null);
ResultSet exported = meta.getExportedKeys(null, schema, table);
ResultSet imported = meta.getImportedKeys(null, schema, table);
ResultSet pks = meta.getPrimaryKeys(null, schema, table)) {
while(columns.next()) {
ColumnStatistic columnStat = new ColumnStatistic();
columnStat.name = columns.getString("COLUMN_NAME");
columnStat.size = columns.getInt("COLUMN_SIZE");
columnStat.decimalDigits = columns.getInt("DECIMAL_DIGITS");
columnStat.type = JDBCType.valueOf(columns.getInt("DATA_TYPE"));
columnStat.comment = columns.getString("REMARKS");
columnStat.isNullable = columns.getString("IS_NULLABLE").equalsIgnoreCase("yes");
columnStat.isAutoincrement = columns.getString("IS_AUTOINCREMENT").equalsIgnoreCase("yes");
columnStat.countNull = jt.queryForObject(String.format(SQL_COUNT_NULL, tableStat.name, columnStat.name), Long.class).longValue();
columnStat.percentNull = (double) columnStat.countNull / (double) tableStat.rowCount * 100.0;
jt.query(String.format(SQL_PERCENT_UNIQUE, columnStat.name, columnStat.name, tableStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.uniqueCount = rs.getLong(1);
columnStat.uniquePercent = rs.getDouble(2);
}
});
columnStat.ranking = jt.query(String.format(SQL_RANKING, columnStat.name, tableStat.name, columnStat.name, columnStat.name), new RowMapper<RankingEntry>() {
@Override
public RankingEntry mapRow(ResultSet rs, int rowNum) throws SQLException {
return new RankingEntry(rs.getString(1), rs.getInt(2));
}
});
switch (columnStat.type) {
case CHAR:
case NCHAR:
case VARCHAR:
case NVARCHAR:
case LONGVARCHAR:
case LONGNVARCHAR:
jt.query(String.format(SQL_MIN_MAX_LEN, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.minLen = Optional.of(Integer.valueOf(rs.getInt(1)));
columnStat.maxLen = Optional.of(Integer.valueOf(rs.getInt(2)));
}
});
columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name,
"length(" + columnStat.name + ")", columnStat.minLen.get()), Integer.class));
columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name,
"length(" + columnStat.name + ")", columnStat.maxLen.get()), Integer.class));
break;
case BIGINT:
case DECIMAL:
case DOUBLE:
case FLOAT:
case INTEGER:
case REAL:
case NUMERIC:
case SMALLINT:
case TINYINT:
jt.query(String.format(SQL_MIN_MAX_AVG, columnStat.name, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.min = Optional.of(Double.valueOf(rs.getDouble(1)));
columnStat.max = Optional.of(Double.valueOf(rs.getDouble(2)));
columnStat.avg = Optional.of(Double.valueOf(rs.getDouble(3)));
}
});
columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, columnStat.min.get()), Integer.class));
columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, columnStat.max.get()), Integer.class));
break;
case DATE:
jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.earliestDate = Optional.ofNullable(rs.getDate(1));
columnStat.latestDate = Optional.ofNullable(rs.getDate(2));
}
});
if (columnStat.earliestDate.isPresent()) {
columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestDate.get())), Integer.class));
}
if (columnStat.latestDate.isPresent()) {
columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestDate.get())), Integer.class));
}
break;
case TIMESTAMP:
case TIMESTAMP_WITH_TIMEZONE:
jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.earliestTimestamp = Optional.ofNullable(rs.getTimestamp(1));
columnStat.latestTimestamp = Optional.ofNullable(rs.getTimestamp(2));
}
});
if (columnStat.earliestTimestamp.isPresent()) {
columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestTimestamp.get())), Integer.class));
}
if (columnStat.latestTimestamp.isPresent()) {
columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestTimestamp.get())), Integer.class));
}
break;
case TIME:
case TIME_WITH_TIMEZONE:
jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() {
@Override
public void processRow(ResultSet rs) throws SQLException {
columnStat.earliestTime = Optional.ofNullable(rs.getTime(1));
columnStat.latestTime = Optional.ofNullable(rs.getTime(2));
}
});
if (columnStat.earliestTime.isPresent()) {
columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestTime.get())), Integer.class));
}
if (columnStat.latestTime.isPresent()) {
columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestTime.get())), Integer.class));
}
break;
default:
}
tableStat.columns.add(columnStat);
}
while (exported.next()) {
String fromColumn = exported.getString("PKCOLUMN_NAME");
String toTable = exported.getString("FKTABLE_NAME");
String toColumn = exported .getString("FKCOLUMN_NAME");
tableStat.exportedKeys.add(new ForeignKey(fromColumn, toTable, toColumn));
}
while (imported.next()) {
String fromColumn = imported.getString("FKCOLUMN_NAME");
String toTable = imported.getString("PKTABLE_NAME");
String toColumn = imported .getString("PKCOLUMN_NAME");
tableStat.importedKeys.add(new ForeignKey(fromColumn, toTable, toColumn));
}
while (pks.next()) {
String column = pks.getString("COLUMN_NAME");
tableStat.primaryKeys.add(column);
}
}
tableStats.add(tableStat);
}
}
} catch (SQLException e) {
logger.error("SQL Fehler", e);
}
return tableStats;
}
/**
* Output statistic for a list of tables to an Excel file.
* The statistics of each table are written to a separate sheet.
* @param tableStats The list of TableStats
* @param outputFile The File to output to. If null output to current dir with a default file name.
*/
public void outputExcel(List<TableStatistic> tableStats, File outputFile) {
XSSFWorkbook workbook = new XSSFWorkbook();
XSSFDataFormat dataFormat = workbook.createDataFormat();
XSSFCellStyle cellStyleDouble = workbook.createCellStyle();
cellStyleDouble.setDataFormat(dataFormat.getFormat("0.##"));
XSSFCellStyle headerStyle = workbook.createCellStyle();
XSSFFont bold = workbook.createFont();
bold.setBold(true);
headerStyle.setFont(bold);
for (TableStatistic tableStat : tableStats) {
XSSFSheet sheet = workbook.createSheet("Table " + tableStat.name);
XSSFRow header = sheet.createRow(0);
XSSFCell cell = header.createCell(0);
cell.setCellValue("Database: " + database);
cell.setCellStyle(headerStyle);
cell = header.createCell(1);
cell.setCellStyle(headerStyle);
cell.setCellValue("Schema: " + schema);
XSSFRow first = sheet.createRow(2);
first.createCell(0).setCellValue("Table");
first.getCell(0).setCellStyle(headerStyle);
first.createCell(1).setCellValue("Row Count");
first.getCell(1).setCellStyle(headerStyle);
first.createCell(2).setCellValue("Column");
first.getCell(2).setCellStyle(headerStyle);
first.createCell(3).setCellValue("Type");
first.getCell(3).setCellStyle(headerStyle);
first.createCell(4).setCellValue("Size");
first.getCell(4).setCellStyle(headerStyle);
first.createCell(5).setCellValue("Not Null");
first.getCell(5).setCellStyle(headerStyle);
first.createCell(6).setCellValue("Autoincrement");
first.getCell(6).setCellStyle(headerStyle);
first.createCell(7).setCellValue("Count NULL");
first.getCell(7).setCellStyle(headerStyle);
first.createCell(8).setCellValue("% NULL");
first.getCell(8).setCellStyle(headerStyle);
first.createCell(9).setCellValue("Count Unique");
first.getCell(9).setCellStyle(headerStyle);
first.createCell(10).setCellValue("% Unique");
first.getCell(10).setCellStyle(headerStyle);
first.createCell(11).setCellValue("Min Len");
first.getCell(11).setCellStyle(headerStyle);
first.createCell(12).setCellValue("Max Len");
first.getCell(12).setCellStyle(headerStyle);
first.createCell(13).setCellValue("Min");
first.getCell(13).setCellStyle(headerStyle);
first.createCell(14).setCellValue("Max");
first.getCell(14).setCellStyle(headerStyle);
first.createCell(15).setCellValue("Avg");
first.getCell(15).setCellStyle(headerStyle);
first.createCell(16).setCellValue("Min Count");
first.getCell(16).setCellStyle(headerStyle);
first.createCell(17).setCellValue("Max Count");
first.getCell(17).setCellStyle(headerStyle);
first.createCell(18).setCellValue("Earliest");
first.getCell(18).setCellStyle(headerStyle);
first.createCell(19).setCellValue("Latest");
first.getCell(19).setCellStyle(headerStyle);
first.createCell(20).setCellValue("Comment");
first.getCell(20).setCellStyle(headerStyle);
int row = 3;
XSSFRow tableRow = sheet.createRow(row);
tableRow.createCell(0).setCellValue(tableStat.name);
tableRow.getCell(0).setCellStyle(headerStyle);
tableRow.createCell(1).setCellValue(tableStat.rowCount);
tableRow.getCell(1).setCellStyle(headerStyle);
for (ColumnStatistic columnStat : tableStat.columns) {
row += 1;
XSSFRow descRow = sheet.createRow(row);
descRow.createCell(2).setCellValue(columnStat.name);
if (tableStat.primaryKeys.contains(columnStat.name)) {
descRow.getCell(2).setCellValue(columnStat.name + " (PK)");
descRow.getCell(2).setCellStyle(headerStyle);
}
descRow.createCell(3).setCellValue(columnStat.type.getName());
descRow.createCell(4).setCellValue(columnStat.size);
if (columnStat.decimalDigits != 0) {
descRow.getCell(4).setCellValue(
Double.valueOf(columnStat.size + "." + columnStat.decimalDigits).doubleValue()
);
descRow.getCell(4).setCellStyle(cellStyleDouble);
}
descRow.createCell(5).setCellValue(!columnStat.isNullable);
descRow.createCell(6).setCellValue(columnStat.isAutoincrement);
descRow.createCell(7).setCellValue(columnStat.countNull);
descRow.createCell(8).setCellValue(columnStat.percentNull);
descRow.getCell(8).setCellStyle(cellStyleDouble);
descRow.createCell(9).setCellValue(columnStat.uniqueCount);
descRow.createCell(10).setCellValue(columnStat.uniquePercent);
descRow.getCell(10).setCellStyle(cellStyleDouble);
if (columnStat.minLen.isPresent()) {
descRow.createCell(11).setCellValue(columnStat.minLen.get().doubleValue());
}
if (columnStat.maxLen.isPresent()) {
descRow.createCell(12).setCellValue(columnStat.maxLen.get().doubleValue());
}
if (columnStat.min.isPresent()) {
descRow.createCell(13).setCellValue(columnStat.min.get().doubleValue());
descRow.getCell(13).setCellStyle(cellStyleDouble);
}
if (columnStat.max.isPresent()) {
descRow.createCell(14).setCellValue(columnStat.max.get().doubleValue());
descRow.getCell(14).setCellStyle(cellStyleDouble);
}
if (columnStat.avg.isPresent()) {
descRow.createCell(15).setCellValue(columnStat.avg.get().doubleValue());
descRow.getCell(15).setCellStyle(cellStyleDouble);
}
if (columnStat.min_count.isPresent()) {
descRow.createCell(16).setCellValue(columnStat.min_count.get().doubleValue());
descRow.getCell(16).setCellStyle(cellStyleDouble);
}
if (columnStat.max_count.isPresent()) {
descRow.createCell(17).setCellValue(columnStat.max_count.get().doubleValue());
descRow.getCell(17).setCellStyle(cellStyleDouble);
}
if (columnStat.earliestDate.isPresent()) {
descRow.createCell(18).setCellValue(columnStat.earliestDate.get().toString());
}
if (columnStat.latestDate.isPresent()) {
descRow.createCell(19).setCellValue(columnStat.latestDate.get().toString());
}
if (columnStat.earliestTime.isPresent()) {
descRow.createCell(18).setCellValue(columnStat.earliestTime.get().toString());
}
if (columnStat.latestTime.isPresent()) {
descRow.createCell(19).setCellValue(columnStat.latestTime.get().toString());
}
if (columnStat.earliestTimestamp.isPresent()) {
descRow.createCell(18).setCellValue(columnStat.earliestTimestamp.get().toString());
}
if (columnStat.latestTimestamp.isPresent()) {
descRow.createCell(19).setCellValue(columnStat.latestTimestamp.get().toString());
}
descRow.createCell(20).setCellValue(columnStat.comment);
}
for (int i = 0; i < 20; i++) {
sheet.autoSizeColumn(i);
}
XSSFRow frequHeader1 = sheet.createRow(row + 2);
XSSFRow frequHeader2 = sheet.createRow(row + 3);
frequHeader1.createCell(0).setCellValue("Frequency");
frequHeader1.getCell(0).setCellStyle(headerStyle);
frequHeader2.createCell(0).setCellValue("Column");
frequHeader2.getCell(0).setCellStyle(headerStyle);
for (int n = 1; n <= 10; n++) {
frequHeader1.createCell(n).setCellValue(n);
frequHeader1.getCell(n).setCellStyle(headerStyle);
}
for (int colNr = 0; colNr < tableStat.columns.size(); colNr++) {
XSSFRow frequRowLabel = sheet.createRow(row + 4 + 2 * colNr);
XSSFRow frequRowCount = sheet.createRow(row + 5 + 2 * colNr);
frequRowLabel.createCell(0).setCellValue(tableStat.columns.get(colNr).name);
for (int rankNr = 0; rankNr < tableStat.columns.get(colNr).ranking.size(); rankNr++) {
frequRowLabel.createCell(rankNr + 1).setCellValue(tableStat.columns.get(colNr).ranking.get(rankNr).label);
frequRowCount.createCell(rankNr + 1).setCellValue(tableStat.columns.get(colNr).ranking.get(rankNr).count);
}
}
row = row + 2 * tableStat.columns.size() + 5;
XSSFRow exHeader1 = sheet.createRow(row);
exHeader1.createCell(0).setCellValue("Exported Keys");
exHeader1.getCell(0).setCellStyle(headerStyle);
XSSFRow exHeader2 = sheet.createRow(row + 1);
exHeader2.createCell(0).setCellValue("From Column");
exHeader2.getCell(0).setCellStyle(headerStyle);
exHeader2.createCell(1).setCellValue("To Table");
exHeader2.getCell(1).setCellStyle(headerStyle);
exHeader2.createCell(2).setCellValue("To Column");
exHeader2.getCell(2).setCellStyle(headerStyle);
for (int fkNr = 0; fkNr < tableStat.exportedKeys.size(); fkNr++) {
XSSFRow fkRow = sheet.createRow(row + 2 + fkNr);
fkRow.createCell(0).setCellValue(tableStat.exportedKeys.get(fkNr).fromColumn);
fkRow.createCell(1).setCellValue(tableStat.exportedKeys.get(fkNr).toTable);
fkRow.createCell(2).setCellValue(tableStat.exportedKeys.get(fkNr).toColumn);
}
row = row + 3 + tableStat.exportedKeys.size();
XSSFRow imHeader1 = sheet.createRow(row);
imHeader1.createCell(0).setCellValue("Imported Keys");
imHeader1.getCell(0).setCellStyle(headerStyle);
XSSFRow imHeader2 = sheet.createRow(row + 1);
imHeader2.createCell(0).setCellValue("From Table");
imHeader2.getCell(0).setCellStyle(headerStyle);
imHeader2.createCell(1).setCellValue("From Column");
imHeader2.getCell(1).setCellStyle(headerStyle);
imHeader2.createCell(2).setCellValue("To Column");
imHeader2.getCell(2).setCellStyle(headerStyle);
for (int fkNr = 0; fkNr < tableStat.importedKeys.size(); fkNr++) {
XSSFRow fkRow = sheet.createRow(row + 2 + fkNr);
fkRow.createCell(0).setCellValue(tableStat.importedKeys.get(fkNr).toTable);
fkRow.createCell(1).setCellValue(tableStat.importedKeys.get(fkNr).toColumn);
fkRow.createCell(2).setCellValue(tableStat.importedKeys.get(fkNr).fromColumn);
}
}
File currDir = new File(".");
String path = currDir.getAbsolutePath();
String fileLocation = path.substring(0, path.length() - 1) + "db_profile_" + database + ".xlsx";
if (outputFile != null) {
fileLocation = outputFile.getAbsolutePath();
}
logger.info("Writing to " + fileLocation);
FileOutputStream outputStream;
try {
outputStream = new FileOutputStream(fileLocation);
workbook.write(outputStream);
workbook.close();
} catch (IOException e) {
logger.error("Couldn't write excel file", e);
}
}
private static Options createOptions() {
Options options = new Options();
Option opt;
opt = new Option("h", "help", false, "get help");
options.addOption(opt);
opt = new Option("t", "tables", true, "tables");
opt.setArgs(Option.UNLIMITED_VALUES);
// opt.setRequired(true);
options.addOption(opt);
opt = new Option("s", "schema", true, "schema");
options.addOption(opt);
opt = new Option("d", "database", true, "database");
// opt.setRequired(true);
options.addOption(opt);
return options;
}
private static CommandLine parseArgs(String[] args, Options options) {
CommandLineParser parser = new GnuParser();
try {
return parser.parse(options, args, false);
} catch (ParseException e) {
System.out.println("error while reading the command line parameters:");
e.printStackTrace();
System.exit(1);
}
return null;
}
private static void initSxPools() {
try {
List<String> mandantenNamen = new LinkedList<String>();
mandantenNamen.add(DEFAULT_MANDANTEN_ID);
SxPools.closeAll();
SxPools.init(mandantenNamen);
SxPools.get(DEFAULT_MANDANTEN_ID).init();
SxPools.get(DEFAULT_MANDANTEN_ID).initLogging(true, Level.DEBUG);
// also init kettle env, set plugin dir
SuperXManager.initKettleEnv(APPLICATION_CONTEXT);
} catch (Exception e) {
System.out.println("error while initialising the SuperX pools:");
e.printStackTrace();
System.exit(1);
}
}
private static void printHelp(Options options) {
HelpFormatter help = new HelpFormatter();
help.printHelp(HELP_STRING, options);
}
private static GenericApplicationContext createContext() {
/*
* https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/context/annotation/AnnotationConfigApplicationContext.html
* quote:
* "In case of multiple @Configuration classes, @Bean methods defined in later classes will override those defined in earlier classes.
* This can be leveraged to deliberately override certain bean definitions via an extra @Configuration class."
* - so it's alright to override some beans via "CLIConfig"
*/
if (APPLICATION_CONTEXT == null) {
APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(AppConfig.class, DataJdbcConfiguration.class, CLIConfig.class, ServiceConfig.class);
}
return APPLICATION_CONTEXT;
}
private static String quote(Object o) {
return "'" + o + "'";
}
}
class TableStatistic {
public String name;
public long rowCount;
public List<ColumnStatistic> columns;
public List<ForeignKey> exportedKeys;
public List<ForeignKey> importedKeys;
public List<String> primaryKeys;
public TableStatistic(String name, long rowCount) {
this.name = name;
this.rowCount = rowCount;
this.columns = new ArrayList<>();
this.exportedKeys = new ArrayList<>();
this.importedKeys = new ArrayList<>();
this.primaryKeys = new ArrayList<>();
}
}
class ColumnStatistic {
public String name;
public JDBCType type;
public int size;
public int decimalDigits;
public boolean isNullable;
public boolean isAutoincrement;
public String comment;
public long countNull;
public double percentNull;
public long uniqueCount;
public double uniquePercent;
public Optional<Integer> max_count = Optional.empty();
public Optional<Integer> min_count = Optional.empty();
public List<RankingEntry> ranking;
public Optional<Integer> minLen = Optional.empty();
public Optional<Integer> maxLen = Optional.empty();
public Optional<Double> min = Optional.empty();
public Optional<Double> max = Optional.empty();
public Optional<Double> avg = Optional.empty();
public Optional<Date> earliestDate = Optional.empty();
public Optional<Date> latestDate = Optional.empty();
public Optional<Time> earliestTime = Optional.empty();
public Optional<Time> latestTime = Optional.empty();
public Optional<Timestamp> earliestTimestamp = Optional.empty();
public Optional<Timestamp> latestTimestamp = Optional.empty();
}
class RankingEntry {
public RankingEntry(String label, int count) {
this.label = label;
this.count = count;
}
public String label;
public int count;
}
class ForeignKey {
public ForeignKey(String fromColumn, String toTable, String toColumn) {
this.fromColumn = fromColumn;
this.toTable = toTable;
this.toColumn = toColumn;
}
public String fromColumn;
public String toTable;
public String toColumn;
}