diff --git a/src/de/memtext/util/ServletHelper.java b/src/de/memtext/util/ServletHelper.java index c3d53db..0058f00 100644 --- a/src/de/memtext/util/ServletHelper.java +++ b/src/de/memtext/util/ServletHelper.java @@ -1,78 +1,75 @@ -package de.memtext.util; - -import java.io.IOException; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.xml.transform.ErrorListener; -import javax.xml.transform.TransformerException; - -import de.superx.servlet.ServletBasics; - -/** - * Abstrakte Basisklasse für ServletHelper-Objekt, nimmt request und response - * und den Logger auf. Objekte, die erweitern müssen (wie bei Thread) run() - * aufrufen. Dann wird die Authentifizierung geprüft und wenn die OK ist, wird - * die perform()-Methode des eigentlichen Objekts aufgerufen (die z.B. Maske - * aufbaut oder Tabelle holt). wenn eine Exception auftritt sorgt run() dafür, - * dass der Fehler gemeldet wird. Als Variablen enthält diese Klasse bereits den - * StringBuffer returnText mit dem geplanten Rückgabetext. Und userid mit der - * UserId (wenn Authentifizierung OK) - */ -public abstract class ServletHelper extends ServletBasics{ - - - public ServletHelper(HttpServletRequest request, - HttpServletResponse response, String sessiontype) - throws IOException { - super(request,response,sessiontype); - } - - /** - * Prüft falls gewünscht die Authentifizierung (existiert eine - * superx-Session) danach wird die abstrakte Methode perform aufgerufen. Die - * meisten Exceptions werden also Infomeldung an den User weitergegeben - * - * @param isAuthentificationCheckWanted - * @throws IOException - */ - public void run(boolean isAuthentificationCheckWanted) throws IOException, - ServletException { - try { - if (isAuthentificationCheckWanted) { - checkSessionType(); - } - - perform(); - - } catch (Exception e) { - - } - } - - abstract protected void perform() throws Exception; - - class DummyErrorListener implements ErrorListener { - - public void warning(TransformerException exception) - throws TransformerException { - } - - public void error(TransformerException exception) - throws TransformerException { - } - - public void fatalError(TransformerException exception) - throws TransformerException { - System.out.println(exception); - } - - } -} - -//Created on 30.09.2004 at 08:59:42 - -//Created on 27.02.2006 at 18:50:31 - -//refactored to servletBasis 10.8.2011 +package de.memtext.util; + +import java.io.IOException; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import javax.xml.transform.ErrorListener; +import javax.xml.transform.TransformerException; + +import de.superx.servlet.ServletBasics; + +/** + * Abstrakte Basisklasse für ServletHelper-Objekt, nimmt request und response + * und den Logger auf. Objekte, die erweitern müssen (wie bei Thread) run() + * aufrufen. Dann wird die Authentifizierung geprüft und wenn die OK ist, wird + * die perform()-Methode des eigentlichen Objekts aufgerufen (die z.B. Maske + * aufbaut oder Tabelle holt). wenn eine Exception auftritt sorgt run() dafür, + * dass der Fehler gemeldet wird. Als Variablen enthält diese Klasse bereits den + * StringBuffer returnText mit dem geplanten Rückgabetext. Und userid mit der + * UserId (wenn Authentifizierung OK) + */ +public abstract class ServletHelper extends ServletBasics { + + + public ServletHelper(HttpServletRequest request, HttpServletResponse response, String sessiontype) throws IOException { + super(request, response, sessiontype); + } + + /** + * Prüft falls gewünscht die Authentifizierung (existiert eine + * superx-Session) danach wird die abstrakte Methode perform aufgerufen. Die + * meisten Exceptions werden also Infomeldung an den User weitergegeben + * + * @param isAuthentificationCheckWanted + * @throws IOException + */ + public void run(boolean isAuthentificationCheckWanted) throws IOException, ServletException { + try { + if (isAuthentificationCheckWanted) { + checkSessionType(); + } + + perform(); + + } catch (Exception e) { + + } + } + + abstract protected void perform() throws Exception; + + class DummyErrorListener implements ErrorListener { + + @Override + public void warning(TransformerException exception) throws TransformerException { + } + + @Override + public void error(TransformerException exception) throws TransformerException { + } + + @Override + public void fatalError(TransformerException exception) throws TransformerException { + System.out.println(exception); + } + + } +} + +//Created on 30.09.2004 at 08:59:42 + +//Created on 27.02.2006 at 18:50:31 + +//refactored to servletBasis 10.8.2011 diff --git a/src/de/superx/bianalysis/ColumnElement.java b/src/de/superx/bianalysis/ColumnElement.java new file mode 100644 index 0000000..4010411 --- /dev/null +++ b/src/de/superx/bianalysis/ColumnElement.java @@ -0,0 +1,88 @@ +package de.superx.bianalysis; + +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.Measure; + +public class ColumnElement { + + public String caption; + public String header; + public String dimensionAttributeFilter; + public Measure measure; + public int columnNumber; + + public ColumnElement(String caption, String dimensionAttributeFilter) { + this.caption = caption; + this.dimensionAttributeFilter = dimensionAttributeFilter; + } + + public ColumnElement(String caption, String dimensionAttributeFilter, Measure measure, int col) { + this.caption = caption; + this.dimensionAttributeFilter = dimensionAttributeFilter; + this.measure = measure; + this.columnNumber = col; + } + + public ColumnElement(Measure measure, int index) { + this.caption = "Kennzahl|" + measure.getId().composedId; + this.header = "Kennzahl|" + measure.getCaption(); + this.measure = measure; + this.columnNumber = index; + } + + public ColumnElement(ColumnElement currentColumnElement) { + this.caption = currentColumnElement.caption; + this.dimensionAttributeFilter = currentColumnElement.dimensionAttributeFilter; + this.measure = currentColumnElement.measure; + } + + /** + * Builds the attribute part of a columns's 'field' member. + * + * The attribute part is a crucial component of the column's identifier + * and typically consists of IDs and associated values. + * + *

Example of an attribute part: + * 

+     *     "conf:123 : conf:124 |weiblich"
+     *
+ *

+ * + *

In the context of a complete 'field' member, it might appear as: + * 

+     *     "conf:123: conf:124|weiblich || Kennzahl|res:123"
+     *
+ * where the part before "||" is the attribute part, and after is the measure.

+ * + * The 'field' member serves as a unique identifier for each column. + * + * @see ColumnElementBuilder For the complete column building process + */ + public static String buildField(DimensionAttribute attr, String value) { + // The conformed id takes precedence, so that we can merge reports + String attrId = attr.getAttrConformedId(); + if(attrId == null) { + attrId = attr.getStringId(); + } + + String dimId = attr.getDimConformedId(); + if(dimId == null) { + dimId = attr.getDimId(); + } + + return dimId + ": " + attrId + "|" + value; + } + + public static String buildHeader(DimensionAttribute attr, String value) { + return attr.getCaption() + ": " + attr.getCaption() + "|" + value; + } + + public static String buildFilter(DimensionAttribute attr, String value) { + return attr.getDimensionTableAlias() + "." + attr.getColumnname() + " = '" + value + "'"; + } + + public void setHeader(String finalHeader) { + this.header = finalHeader; + } + +} diff --git a/src/de/superx/bianalysis/ColumnElementBuilder.java b/src/de/superx/bianalysis/ColumnElementBuilder.java new file mode 100644 index 0000000..b3a5dc2 --- /dev/null +++ b/src/de/superx/bianalysis/ColumnElementBuilder.java @@ -0,0 +1,157 @@ +package de.superx.bianalysis; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; + +import org.apache.log4j.Logger; + +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.Measure; +import de.superx.common.NotYetImplementedException; + +public class ColumnElementBuilder { + + private static Logger logger = Logger.getLogger(ColumnElementBuilder.class); + + /** + * Lets assume we have the two Dimensions X and Y each with one Attribute. DA for X + * and DB for Y. Both Attributes have two possible values DA1, DA2 and DB1, DB2. There + * also exist two Measures M1, M2. + * + * If the users wants to see all attributes and measures the header of the cross table looks like this: + * + * +-----------------------+----------------------+ + * | DA1 | DA2 | + * +-----------+-----------+-----------+----------+ + * | DB1 | DB2 | DB1 | DB2 | + * +-----+-----+-----+-----+-----+-----+-----+----+ + * | M1 | M2 | M1 | M2 | M1 | M2 | M1 | M2 | + * +=====+=====+=====+=====+=====+=====+=====+====+ + * | | | | | | | | | + * +-----+-----+-----+-----+-----+-----+-----+----+ + * + * This header would be defined like follows: + * + * "X: DA | DA1 || Y: DB | DB1 || Kennzahl| M1" + * "X: DA | DA1 || Y: DB | DB1 || Kennzahl| M2" + * "X: DA | DA1 || Y: DB | DB2 || Kennzahl| M1" + * "X: DA | DA1 || Y: DB | DB2 || Kennzahl| M2" + * "X: DA | DA2 || Y: DB | DB1 || Kennzahl| M1" + * "X: DA | DA2 || Y: DB | DB1 || Kennzahl| M2" + * "X: DA | DA2 || Y: DB | DB2 || Kennzahl| M1" + * "X: DA | DA2 || Y: DB | DB2 || Kennzahl| M2" + * + * Every single line is represented by one 'ColumnElement'. + * @throws NotYetImplementedException + */ + public static List buildColumnElements(ReportMetadata metadata) { + + List filters = metadata.filters; + List measures = metadata.measures; + List dimensionAttributes = metadata.topDimensionAttributes; + + List columnElements = new ArrayList(); + final String HEADER_DIVIDER = " || "; + final String KENNZAHL_IDENTIFIER = "Kennzahl|"; + + if(measures == null || measures.isEmpty()) { + // edge case 1: no measures were selected, simply return empty columnElements list + return columnElements; + } + + // for every column there exists an offset of 'maxbridgelvl' if a hierarchy-attribute was selected + int colStartPoint = metadata.getHierarchyAttributes().size() * metadata.maxBridgeLvl; + if(dimensionAttributes == null || dimensionAttributes.isEmpty()) { + // edge case 2: no dimension attributes were selected, only display the measures + for (Measure measure : measures) { + columnElements.add(new ColumnElement(measure, colStartPoint + columnElements.size())); + } + return columnElements; + } + + // for every single column combination (one list of combined attribute values) we build one 'ColumnElement' object + List> dimAttrCombinations = cartesianProductOfDimensionAttributeValues(dimensionAttributes, filters); + for (int i = 0; i < dimAttrCombinations.size(); i++) { + StringJoiner captionJoiner = new StringJoiner(HEADER_DIVIDER); + StringJoiner headerJoiner = new StringJoiner(HEADER_DIVIDER); + StringJoiner filterJoiner = new StringJoiner(" AND "); + List comb = dimAttrCombinations.get(i); + for(int j = 0; j < comb.size(); j++) { + DimensionAttribute attr = dimensionAttributes.get(j); + String value = comb.get(j); + captionJoiner.add(ColumnElement.buildField(attr, value)); + headerJoiner.add(ColumnElement.buildHeader(attr, value)); + filterJoiner.add(ColumnElement.buildFilter(attr, value)); + } + String partialCaption = captionJoiner.toString(); + String partialHeader = headerJoiner.toString(); + String filter = filterJoiner.toString(); + for (Measure measure : measures) { + String finalCaption = partialCaption + HEADER_DIVIDER + KENNZAHL_IDENTIFIER + measure.getId().composedId; + String finalHeader = partialHeader + HEADER_DIVIDER + KENNZAHL_IDENTIFIER + measure.getCaption(); + ColumnElement colElement = new ColumnElement(finalCaption, filter, measure, colStartPoint + columnElements.size()); + colElement.setHeader(finalHeader); + columnElements.add(colElement); + } + } + + return columnElements; + } + + /** + * + * Computes all possible combination of dimension attribute values. + * Each of the individual combinations is a specific column. + * + * Example + * Input: DimensionAttributes = {DA, DB}, each with two possible values DA1, DA2, DB1, DB2, Filters = { } + * Output: {{DA1, DB1}, {DA1, DB2}, {DA2, DB1}, {DA2, DB2}} + * + * If the user choose the following Filter = {DB2} + * Output: {{DA1, DB2}, {DA2, DB2}} + * + * @param dimensionAttributes The list of choosen dimension attributes. + * @param filters The list of choosen filters. + * @return A list containing the cartesian product of all the possible combination for a set of dimension attributes and filters. + */ + private static List> cartesianProductOfDimensionAttributeValues(List dimensionAttributes, List filters){ + List> allDimAttrVals = new ArrayList<>(); + for (DimensionAttribute attr: dimensionAttributes) { + //if(attr.bridge != null) { + // continue; + //} + // did the user choose a filter for this attribute ? + Filter compoundFilter = Filter.findFilterById(filters, attr.getId()); + if(compoundFilter != null) { + // if yes only use the filter values + allDimAttrVals.add(compoundFilter.filterValues); + } else { + // if no use all possible attribute values + allDimAttrVals.add(attr.getDimensionAttributeValues()); + } + } + // compute and return all possible column combinations + return cartesian(allDimAttrVals); + } + + private static List> cartesian(List> lists) { + List> result = new ArrayList<>(); + if(lists.size() == 0) { + result.add(new ArrayList<>()); + return result; + } + List curr = lists.get(0); + List> remainingLists = cartesian(lists.subList(1, lists.size())); + for (String val : curr) { + for (List list : remainingLists) { + List resultList = new ArrayList<>(); + resultList.add(val); + resultList.addAll(list); + result.add(resultList); + } + } + return result; + } +} diff --git a/src/de/superx/bianalysis/ExcelSheetBuilder.java b/src/de/superx/bianalysis/ExcelSheetBuilder.java new file mode 100644 index 0000000..9d216bf --- /dev/null +++ b/src/de/superx/bianalysis/ExcelSheetBuilder.java @@ -0,0 +1,415 @@ +package de.superx.bianalysis; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.Footer; +import org.apache.poi.ss.usermodel.Header; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.util.CellRangeAddress; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import de.superx.bianalysis.models.InfoItem; +import de.superx.rest.model.Column; +import de.superx.rest.model.ColumnType; +import de.superx.rest.model.Result; +import de.superx.rest.model.ResultType; + +public class ExcelSheetBuilder { + + private Result result; + private XSSFWorkbook workbook; + private XSSFSheet sheet; + private String reportName; + private String reportDescription; + private String date; + private int leftDimensionAttributes; + private int topDimensionAttributes = 0; + + private List visibleColumns; + private final boolean mergeCells = true; + private final int startingRow = 1; + + private static HashMap defaultStyles = new HashMap<>(); + + public ExcelSheetBuilder(Result result) { + this.result = result; + this.visibleColumns = getVisibleColumns(result); + this.workbook = new XSSFWorkbook(); + initializeDefaultStyles(); + + leftDimensionAttributes = this.result.info.leftDimensionAttributes.size(); + if(this.result.info.topDimensionAttributes != null && this.result.info.topDimensionAttributes.size() > 0) { + topDimensionAttributes = this.result.info.topDimensionAttributes.size(); + } + } + + public XSSFWorkbook build() { + int rowNum = startingRow; + rowNum = createRowsFromGrid(createReportInfoGrid(), rowNum); + rowNum += 2; // rows between report info and header + int reportInfoEnd = rowNum; + + String[][] grid = createHeaderGrid(); + + rowNum = createRowsFromGrid(grid, rowNum); + rowNum = createDataRows(rowNum); + rowNum = createTotalRow(rowNum); + + if(mergeCells) { + mergeHeaderCells(grid, 0, reportInfoEnd); + } + + styleHeaderCells(reportInfoEnd, grid); + styleDataCells(reportInfoEnd + grid.length); + styleTotalRowCells(rowNum); + styleReportInfoCells(); + + Footer footer = sheet.getFooter(); + Header header = sheet.getHeader(); + header.setLeft(reportName); + header.setRight(this.date); + footer.setRight("Seite &P von &N"); + + return workbook; + } + + private void initializeDefaultStyles() { + //ReportInfoCells + XSSFCellStyle infoStyle = workbook.createCellStyle(); + Font infoFont = workbook.createFont(); + infoFont.setBold(true); + infoStyle.setFont(infoFont); + + defaultStyles.put("info", Integer.valueOf(infoStyle.getIndex())); + + //Header + XSSFCellStyle headerStyle = workbook.createCellStyle(); + headerStyle.setBorderBottom(BorderStyle.THIN); + headerStyle.setBorderLeft(BorderStyle.THIN); + headerStyle.setBorderRight(BorderStyle.THIN); + headerStyle.setBorderTop(BorderStyle.THIN); + Font headerFont = workbook.createFont(); + headerFont.setBold(true); + headerStyle.setFont(headerFont); + + defaultStyles.put("header", Integer.valueOf(headerStyle.getIndex())); + + //Data + XSSFCellStyle dataStyle = workbook.createCellStyle(); + dataStyle.setBorderBottom(BorderStyle.HAIR); + dataStyle.setBorderLeft(BorderStyle.HAIR); + dataStyle.setBorderRight(BorderStyle.HAIR); + dataStyle.setBorderTop(BorderStyle.HAIR); + + defaultStyles.put("data", Integer.valueOf(dataStyle.getIndex())); + + //Total + XSSFCellStyle totalStyle = workbook.createCellStyle(); + totalStyle.setBorderBottom(BorderStyle.THIN); + totalStyle.setBorderLeft(BorderStyle.THIN); + totalStyle.setBorderRight(BorderStyle.THIN); + totalStyle.setBorderTop(BorderStyle.DOUBLE); + Font totalFont = workbook.createFont(); + totalFont.setBold(true); + totalStyle.setFont(totalFont); + + defaultStyles.put("total", Integer.valueOf(totalStyle.getIndex())); + + } + + + private void styleTotalRowCells(int rowNum) { + int current = rowNum; + Row row = sheet.getRow(--current); + for (int i = 0; i < visibleColumns.size(); i++) { + Cell cell = row.getCell(i); + cell.setCellStyle(getTotalCellStyle(workbook)); + } + } + + private int createTotalRow(int startFrom) { + de.superx.rest.model.Row totalRow = result.getTotalRow(); + int rowNum = startFrom; + Row row = sheet.createRow(rowNum++); + Cell labelCell = row.createCell(0); + labelCell.setCellValue("Gesamt"); + for (int i = 1; i < visibleColumns.size(); i++) { + Column col = visibleColumns.get(i); + Cell cell = row.createCell(i); + if(col.type.equals(ColumnType.StringColumn)) { + cell.setCellValue(""); + } else { + Object obj = totalRow.cells.get(col.field); + if(obj == null) { + cell.setCellValue(""); + continue; + } + Double value = Double.valueOf(String.valueOf(obj)); + cell.setCellValue(value.doubleValue()); + } + } + return rowNum; + } + + private void styleReportInfoCells() { + Row row = sheet.getRow(startingRow); + Cell cell = row.getCell(0); + cell.setCellStyle(workbook.getCellStyleAt(defaultStyles.get("info").intValue())); + } + + private String[][] createReportInfoGrid() { + + List> gridList = new ArrayList<>(); + gridList.add(List.of("Informationen zur BI-Analyse", "")); + gridList.add(List.of("Name:", this.reportName)); + gridList.add(List.of("Beschreibung:", this.reportDescription)); + + String sachgebiet = this.result.info.sachgebiete.stream().collect(Collectors.joining(", ")); + String theme = getInfoCaptions(this.result.info.facttables); + String measures = getInfoCaptions(this.result.info.measures); + String topAttributes = getInfoCaptions(this.result.info.topDimensionAttributes); + String leftAttributes = getInfoCaptions(this.result.info.leftDimensionAttributes); + String filter = this.result.info.filter.stream().collect(Collectors.joining(", ")); + String lastUpdateBad = this.result.info.lastUpdateBiad; + + if(sachgebiet != null) { + gridList.add(List.of("Sachgebiet:", sachgebiet)); + } + if(theme != null) { + gridList.add(List.of("Thema:", theme)); + } + if(measures != null) { + gridList.add(List.of("Kennzahlen:", measures)); + } + if(leftAttributes != null) { + gridList.add(List.of("Zeilenattribute:", leftAttributes)); + } + if(topAttributes != null) { + gridList.add(List.of("Spaltenattribute:", topAttributes)); + } + if(filter != null) { + gridList.add(List.of("Filter:", filter)); + } + if(lastUpdateBad != null) { + gridList.add(List.of("Letztes Update von BI-Analyse-Daten:", lastUpdateBad)); + } + if(result.resultType.equals(ResultType.FlatTable)) { + gridList.add(List.of("Tabellentyp:", "Flache Tabelle")); + } else if(result.resultType.equals(ResultType.DrilldownTableGroupable)) { + gridList.add(List.of("Tabellentyp:", "Hierarchische Tabelle")); + } + return listToStringGrid(gridList); + } + + private static String getInfoCaptions(List infoItems) { + if(infoItems != null && infoItems.size() > 0) { + return infoItems.stream().map(f->f.caption).collect(Collectors.joining(", ")); + } + return ""; + } + + private static String[][] listToStringGrid(List> list) { + String[][] result = new String[list.size()][list.get(0).size()]; + for (int i = 0; i < result.length; i++) { + for (int j = 0; j < result[i].length; j++) { + result[i][j] = list.get(i).get(j); + } + } + return result; + } + + private static CellRangeAddress mergeCellByOffset(int firstRow, int lastRow, int firstCol, int lastCol, int xOffset, int yOffset) { + return new CellRangeAddress(firstRow + yOffset, lastRow + yOffset, firstCol + xOffset, lastCol + xOffset); + } + + private void styleHeaderCells(int start, String[][] grid) { + for (int i = 0; i < grid.length; i++) { + Row row = this.sheet.getRow(i+start); + row.setHeightInPoints((short) 25); + for (int j = 0; j < grid[i].length; j++) { + Cell cell = row.getCell(j); + cell.setCellStyle(getHeaderStyle(workbook)); + } + } + } + + private static CellStyle getHeaderStyle(XSSFWorkbook workbook) { + return workbook.getCellStyleAt(defaultStyles.get("header").intValue()); + } + + private static CellStyle getDataCellStyle(XSSFWorkbook workbook) { + return workbook.getCellStyleAt(defaultStyles.get("data").intValue()); + } + + private static CellStyle getTotalCellStyle(XSSFWorkbook workbook) { + return workbook.getCellStyleAt(defaultStyles.get("total").intValue()); + } + + private void styleDataCells(int startDataCells) { + for (int i = startDataCells; i < startDataCells + this.result.rows.size(); i++) { + Row row = this.sheet.getRow(i); + for (int j = 0; j < this.visibleColumns.size(); j++) { + Cell cell = row.getCell(j); + if(this.visibleColumns.get(j).groupable) { + cell.setCellStyle(getHeaderStyle(workbook)); + } else { + cell.setCellStyle(getDataCellStyle(workbook)); + } + } + } + } + + private void mergeHeaderCells(String grid[][], int xOffset, int yOffset) { + // merge header grid cells + if(topDimensionAttributes > 0) { + for(int i = 0; i < grid.length; i++) { + String lastCell = ""; + int cellsToMerge = 0; + for (int j = 0; j < grid[i].length; j++) { + String currentCell = grid[i][j]; + if(!currentCell.equals(lastCell) && cellsToMerge > 0) { + sheet.addMergedRegion(mergeCellByOffset(i, i, j - cellsToMerge - 1, j - 1, xOffset, yOffset)); + } + if(currentCell.equals(lastCell)) { + cellsToMerge++; + } else { + cellsToMerge = 0; + } + lastCell = currentCell; + } + if(cellsToMerge > 0) { + int j = grid[i].length - 1; + sheet.addMergedRegion(mergeCellByOffset(i, i, j - cellsToMerge, j, xOffset, yOffset)); + } + } + } + + // merge left header cols + if(grid.length > 1) { + for (int i = 0; i < leftDimensionAttributes; i++) { + sheet.addMergedRegion(mergeCellByOffset(0, grid.length - 1, i, i, xOffset, yOffset)); + } + } + } + + private int createRowsFromGrid(String[][] grid, int startFrom) { + if(grid == null) { + return startFrom; + } + int rowNum = startFrom; + for (int i = 0; i < grid.length; i++) { + Row poiRow = sheet.createRow(rowNum++); + int colNum = 0; + for (int j = 0; j < grid[i].length; j++) { + Cell cell = poiRow.createCell(colNum++); + if(grid[i][j] != null && !grid[i][j].isBlank()) { + cell.setCellValue(grid[i][j]); + } else { + cell.setBlank(); + } + } + } + return rowNum; + } + + private int createDataRows(int startFrom) { + int rowNum = startFrom; + // build cells from row data without sumrow + List resultRows = result.rows.stream().filter(r -> r.aggregated != -1).collect(Collectors.toList()); + Row[] rows = new Row[resultRows.size()]; + for (int i = 0; i < resultRows.size(); i++) { + rows[i] = sheet.createRow(rowNum++); + rows[i].setHeightInPoints((short) 20); + } + for (int i = 0; i < visibleColumns.size(); i++) { + Column col = visibleColumns.get(i); + for(int j = 0; j < resultRows.size(); j++) { + Object obj = resultRows.get(j).cells.get(col.field); + String objVal = String.valueOf(obj); + Cell cell = rows[j].createCell(i); + if(obj == null) { + cell.setBlank(); + continue; + } + if(col.type == ColumnType.IntegerColumn || col.type == ColumnType.DecimalColumn) { + Double value = Double.valueOf(objVal); + cell.setCellValue(value.doubleValue()); + } else { + cell.setCellValue(obj.toString()); + } + //if(col.groupable) { + // cell.setCellStyle(style); + //} + } + } + return rowNum; + } + + private String[][] createHeaderGrid(){ + int colSize = visibleColumns.size(); + int rowSize = topDimensionAttributes + 1; + String[][] grid = new String[rowSize][colSize]; + + for(int i = 0; i < colSize; i++) { + Column column= this.visibleColumns.get(i); + String[] columnHeader = column.header.split("\\|\\|"); + boolean isLeftDimensionAttributeColumn = (columnHeader.length == 1 && !columnHeader[0].contains("|"))? true : false; + for(int j = 0; j < rowSize; j++) { + if(isLeftDimensionAttributeColumn) { + grid[j][i] = columnHeader[0]; + }else { + String header = columnHeader[j]; + String[] headerValues = header.split("\\|"); + grid[j][i] = headerValues[1]; + } + } + } + return grid; + } + + public ExcelSheetBuilder withFileName(String name) { + this.sheet = workbook.createSheet(name); + return this; + } + + public ExcelSheetBuilder withReportName(String name) { + this.reportName = replaceEmptyString(name, "Nicht gespeicherte BI-Analyse"); + return this; + } + + public ExcelSheetBuilder withDescription(String description) { + this.reportDescription = replaceEmptyString(description, "-"); + return this; + } + + private static String replaceEmptyString(String value, String replacement) { + if(value == null || value.isBlank()) { + return replacement; + } + return value; + } + + public ExcelSheetBuilder withDate(Date currentDate) { + this.date = new SimpleDateFormat("dd.MM.yyyy HH:mm").format(currentDate); + return this; + } + + private static List getVisibleColumns(Result result) { + return result.columns + .stream() + .filter(col -> !col.hidden) + .collect(Collectors.toList()); + } + +} diff --git a/src/de/superx/bianalysis/FaultyMetadataException.java b/src/de/superx/bianalysis/FaultyMetadataException.java new file mode 100644 index 0000000..39dad05 --- /dev/null +++ b/src/de/superx/bianalysis/FaultyMetadataException.java @@ -0,0 +1,21 @@ +package de.superx.bianalysis; + +import de.superx.bianalysis.metadata.Identifier; + +public class FaultyMetadataException extends RuntimeException { + + private static final long serialVersionUID = -5959640234409065198L; + + public FaultyMetadataException(String message) { + super(message); + } + + public FaultyMetadataException(Identifier id) { + super("Metadata Object with ID: '" + id.composedId + "' does not exist."); + } + + public FaultyMetadataException(Identifier id, String metaType) { + super("Metadata " + metaType + " with ID: '" + id.composedId + "' does not exist."); + } + +} diff --git a/src/de/superx/bianalysis/ReportDefinition.java b/src/de/superx/bianalysis/ReportDefinition.java new file mode 100644 index 0000000..b5aa319 --- /dev/null +++ b/src/de/superx/bianalysis/ReportDefinition.java @@ -0,0 +1,57 @@ +package de.superx.bianalysis; + +import java.util.ArrayList; +import java.util.List; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.service.DbMetaAdapter; + +public class ReportDefinition { + + public List factTableIds; + public List leftDimensionAttributeIds; + public List topDimensionAttributeIds; + public List measureIds; + public List filters; + public boolean hideEmptyColumns; + + public ReportDefinition() { + this.factTableIds = new ArrayList<>(); + this.leftDimensionAttributeIds = new ArrayList<>(); + this.topDimensionAttributeIds = new ArrayList<>(); + this.measureIds = new ArrayList<>(); + this.filters = new ArrayList<>(); + this.hideEmptyColumns = false; + } + + public ReportDefinition(ReportDefinition definition) { + super(); + this.factTableIds = definition.factTableIds; + this.topDimensionAttributeIds = definition.topDimensionAttributeIds; + this.measureIds = definition.measureIds; + this.filters = definition.filters; + this.leftDimensionAttributeIds = new ArrayList<>(); + this.hideEmptyColumns = definition.hideEmptyColumns; + + } + + public ReportMetadata getReportMetadata(DbMetaAdapter dbAdapter, Identifier factTableId) { + ReportMetadata reportMetadata = new ReportMetadata(this, factTableId, dbAdapter); + return reportMetadata; + } + + public static List getAttributesForDefinitions(List definitions){ + List ids = new ArrayList<>(); + for (ReportDefinition def : definitions) { + for (Identifier id : def.topDimensionAttributeIds) { + ids.add(id); + } + for (Identifier id : def.leftDimensionAttributeIds) { + ids.add(id); + } + } + return ids; + } + +} \ No newline at end of file diff --git a/src/de/superx/bianalysis/ReportMetadata.java b/src/de/superx/bianalysis/ReportMetadata.java new file mode 100644 index 0000000..4e2d405 --- /dev/null +++ b/src/de/superx/bianalysis/ReportMetadata.java @@ -0,0 +1,338 @@ +package de.superx.bianalysis; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.FactTable; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.InfoItem; +import de.superx.bianalysis.models.Measure; +import de.superx.bianalysis.service.DbMetaAdapter; +import de.superx.jdbc.entity.Sachgebiet; + +public class ReportMetadata { + + public final FactTable factTable; + public final Sachgebiet sachgebiet; + public final List leftDimensionAttributes; + public final List topDimensionAttributes; + public final List measures; + public final List filters; + + public String lastBiadUpdate; + + // only used if hierarchy dimension is present in left dim attributes + public int maxBridgeLvl; + public int minBridgeLvl; + + public DbMetaAdapter dbMetaAdapter; + + public boolean hideEmptyColumns; + + public ReportMetadata(ReportDefinition reportDefinition, Identifier factTableId, DbMetaAdapter dbAdapter) { + this.dbMetaAdapter = dbAdapter; + if(factTableId == null) { // merged Report + this.factTable = new FactTable(); + this.sachgebiet = new Sachgebiet(); + } else { + this.factTable = dbAdapter.getFactTable(factTableId); + this.sachgebiet = dbAdapter.getSachgebietById(this.factTable.getSachgebiettid()); + } + List databaseOrderedLeftDimensionAttributes = dbAdapter.getDimensionAttributeMetadata(reportDefinition.leftDimensionAttributeIds, factTableId); + this.leftDimensionAttributes = reorderDimensionAttributesToReportOrder(databaseOrderedLeftDimensionAttributes, reportDefinition, false); + List databaseOrderedTopDimensionAttributes = dbAdapter.getDimensionAttributeMetadata(reportDefinition.topDimensionAttributeIds, factTableId); + this.topDimensionAttributes = reorderDimensionAttributesToReportOrder(databaseOrderedTopDimensionAttributes, reportDefinition, true); + List databaseOrderedMeasures = dbAdapter.getMeasureMetadata(reportDefinition.measureIds); + this.measures = reorderMeasuresToReportOrder(databaseOrderedMeasures, reportDefinition); + if (reportDefinition.filters != null) { + this.filters = dbAdapter.getFilterMetadata(reportDefinition.filters); + } else { + this.filters = new ArrayList(); + } + this.setTopDimensionAttributeValues(dbAdapter); + if(factTableId != null) { + this.setMaxBridgeLvl(); + } else { + // for merged report + this.setMaxBridgeLvlForConformed(reportDefinition.factTableIds); + } + this.lastBiadUpdate = dbAdapter.getLastUpdate(440); + this.hideEmptyColumns = reportDefinition.hideEmptyColumns; + } + + public ReportMetadata(ReportMetadata metadata, List leftDimensionAttributes) { + this.dbMetaAdapter = metadata.dbMetaAdapter; + this.factTable = metadata.factTable; + this.sachgebiet = metadata.sachgebiet; + this.topDimensionAttributes = metadata.topDimensionAttributes; + this.measures = metadata.measures; + this.filters = metadata.filters; + this.leftDimensionAttributes = leftDimensionAttributes; + this.hideEmptyColumns = metadata.hideEmptyColumns; + } + + public ReportMetadata() { + this.factTable = new FactTable(); + this.sachgebiet = new Sachgebiet(); + this.leftDimensionAttributes = new ArrayList<>(); + this.topDimensionAttributes = new ArrayList<>(); + this.measures = new ArrayList<>(); + this.filters = new ArrayList<>(); + } + + public List getSortOrderLeftDimensionAttributes(){ + return leftDimensionAttributes.stream().filter(d -> d.getSortOrderColumn() != null).collect(Collectors.toList()); + } + + private void setMaxBridgeLvl() { + List attrs = leftDimensionAttributes + .stream() + .filter(a -> a.isHierarchy() ) + .collect(Collectors.toList()); + if(attrs.size() > 1) { + throw new RuntimeException("NOT YET IMPLEMENTED: There can only be one hierarchy attribute."); + } + if(!attrs.isEmpty()) { + this.maxBridgeLvl = dbMetaAdapter.getBridgeMaxLevel(attrs.get(0), this); + this.minBridgeLvl = dbMetaAdapter.getBridgeMinLevel(getHierarchyFilter(), this.maxBridgeLvl, attrs.get(0).getTablename()); + } + } + + private void setMaxBridgeLvlForConformed(List factTableIds) { + List attrs = leftDimensionAttributes + .stream() + .filter(a -> a.isHierarchy()) + .collect(Collectors.toList()); + if(!attrs.isEmpty()) { + DimensionAttribute attr = attrs.get(0); + int lvl = 0; + for (Identifier fact : factTableIds) { + String name = dbMetaAdapter.getFactTableNameMaxBridgeLvl(fact, attr.getId()); + if(name == null || name.isBlank()) { + continue; + } + int value = -1; + Identifier checkedAttr = dbMetaAdapter.checkIfFactTableHasDimensionAttribute(attr.getId(), fact); + if (checkedAttr != null && !checkedAttr.equals(attr.getId())) { + DimensionAttribute rolePlayingAttribute = dbMetaAdapter.getDimensionAttributeMetadataById(checkedAttr); + value = dbMetaAdapter.getBridgeMaxLevel(rolePlayingAttribute, this, name); + } + if (value > lvl) { + lvl = value; + } + } + this.maxBridgeLvl = lvl; + } + } + + private void setTopDimensionAttributeValues(DbMetaAdapter dbAdapter) { + for(DimensionAttribute attr : this.topDimensionAttributes) { + Filter filter = getFilterForDimensionAttribute(attr.getId()); + if(filter != null) { + attr.setDimensionAttributeValues(filter.filterValues); + } else { + attr.setDimensionAttributeValues(dbAdapter.getDimensionAttributeValues(attr, null, null)); + } + } + } + + private Filter getFilterForDimensionAttribute(Identifier id) { + return this.filters + .stream() + .filter(f -> f.dimensionAttributeId.equals(id)) + .findFirst() + .orElse(null); + } + + private static List reorderMeasuresToReportOrder(List measures, ReportDefinition reportDefinition) { + List orderedMeasures = new ArrayList(); + reportDefinition.measureIds.forEach(measureId -> { + Measure nextMeasure = measures + .stream() + .filter( measure -> measure.getId().equals( measureId ) ) + .findFirst() + .orElse(null); + orderedMeasures.add(nextMeasure); + }); + return orderedMeasures; + } + + public static List reorderDimensionAttributesToReportOrder(List dimensionAttributes, ReportDefinition reportDefinition, boolean isTopAttribute) { + List orderedDimensionAttributes = new ArrayList(); + List attributeIds; + if (isTopAttribute) { + attributeIds = reportDefinition.topDimensionAttributeIds; + } else { + attributeIds = reportDefinition.leftDimensionAttributeIds; + } + attributeIds.forEach(attributeId -> { + DimensionAttribute nextAttribute = dimensionAttributes + .stream() + .filter( dimensionAttribute -> dimensionAttribute.getId().equals( attributeId )) + .findFirst() + .orElse(null); + orderedDimensionAttributes.add(nextAttribute); + }); + return orderedDimensionAttributes; + } + + + public DimensionAttribute getDimById(Identifier id) { + DimensionAttribute attr = topDimensionAttributes + .stream() + .filter(a -> a.getDimensionId().equals(id)) + .findFirst() + .orElse(null); + + if(attr != null) { + return attr; + } + + attr = leftDimensionAttributes + .stream() + .filter(a -> a.getDimensionId().equals(id)) + .findFirst() + .orElse(null); + + return attr; + } + + public DimensionAttribute getDimAttrById(Identifier id) { + DimensionAttribute attr = topDimensionAttributes + .stream() + .filter(a -> a.getId().equals(id)) + .findFirst() + .orElse(null); + + if(attr != null) { + return attr; + } + + attr = leftDimensionAttributes + .stream() + .filter(a -> a.getId().equals(id)) + .findFirst() + .orElse(null); + + return attr; + } + + /** + * We want to join dimension tables only once. There we need a list of unique ids of + * the dimensions, otherwise we duplicate our joins. + */ + public List getUniqueDimensionAttributes(){ + + Map joinTables = new HashMap(); + + for (DimensionAttribute attr : leftDimensionAttributes) { + if(!joinTables.containsKey(attr.getDimensionTableAlias())) { + joinTables.put(attr.getDimensionTableAlias(), attr); + } + } + + for (DimensionAttribute attr : topDimensionAttributes) { + if(!joinTables.containsKey(attr.getDimensionTableAlias())) { + joinTables.put(attr.getDimensionTableAlias(), attr); + } + } + + // join dimension if attribute occurs in filter + for (Filter filter : filters) { + Identifier attrId = filter.dimensionAttributeId; + DimensionAttribute attr = dbMetaAdapter.getDimensionAttributeMetadataById(attrId); + if(!joinTables.containsKey(attr.getDimensionTableAlias())) { + joinTables.put(attr.getDimensionTableAlias(), attr); + } + } + + // join dimension if measure has a filter for an attribute + for (Measure measure : measures) { + if(measure.filterAttributeId != null) { + Identifier attrId = measure.filterAttributeId; + DimensionAttribute attr = dbMetaAdapter.getDimensionAttributeMetadataById(attrId); + if(!joinTables.containsKey(attr.getDimensionTableAlias())) { + joinTables.put(attr.getDimensionTableAlias(), attr); + } + } + } + + return new ArrayList(joinTables.values()); + } + + public List getMeasureInfo() { + if(measures != null && !measures.isEmpty()) { + return measures.stream().map(m-> + new InfoItem(m.getId().composedId, m.getCaption(), m.getDescription())).collect(Collectors.toList()); + } + return null; + } + + public List getTopDimAttrAsInfo() { + if(topDimensionAttributes != null && !topDimensionAttributes.isEmpty()) { + return topDimensionAttributes.stream().map(m-> + new InfoItem(m.getStringId(), m.getCaption(), m.getDescription())).collect(Collectors.toList()); + } + return null; + } + + public List getFilterNoHierarchy() { + List filterNoBridge = new ArrayList<>(); + for (Filter filter : this.filters) { + DimensionAttribute attr = dbMetaAdapter.getDimensionAttributeById(filter.dimensionAttributeId); + if(!attr.isHierarchy()) { + filterNoBridge.add(filter); + } + } + return filterNoBridge; + } + + public List getLeftDimAttrAsInfo() { + if(leftDimensionAttributes != null && !leftDimensionAttributes.isEmpty()) { + return leftDimensionAttributes.stream().map(m-> + new InfoItem(m.getStringId(), m.getCaption(), m.getDescription())).collect(Collectors.toList()); + } + return null; + } + + public List getFilterAsInfo(){ + ArrayList filterList = new ArrayList(); + DimensionAttribute dimAttr; + for (Filter filter: filters) { + dimAttr = filter.getDimAttribute(this); + if(dimAttr == null) { + dimAttr = dbMetaAdapter.getDimensionAttributeById(filter.dimensionAttributeId); + } + filterList.add(" (" + dimAttr.getCaption() + ") " + filter.getValuesAsString()); + } + return filterList; + } + + public List getHierarchyAttributes() { + return leftDimensionAttributes + .stream() + .filter(a -> a.isHierarchy()) + .collect(Collectors.toList()); + } + + public List getHierarchyFilter(){ + List hierarchyFilter = new ArrayList<>(); + for (Filter filter : this.filters) { + if(isHierarchyFilter(filter)) { + hierarchyFilter.add(filter); + } + } + return hierarchyFilter; + } + + public boolean isHierarchyFilter(Filter filter) { + return dbMetaAdapter.isAttributeHierarchyBridge(filter.dimensionAttributeId); + } + +} \ No newline at end of file diff --git a/src/de/superx/bianalysis/ResultBuilder.java b/src/de/superx/bianalysis/ResultBuilder.java new file mode 100644 index 0000000..037ffcb --- /dev/null +++ b/src/de/superx/bianalysis/ResultBuilder.java @@ -0,0 +1,536 @@ +package de.superx.bianalysis; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import javax.sql.DataSource; + +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.InfoItem; +import de.superx.bianalysis.models.Measure; +import de.superx.rest.model.Column; +import de.superx.rest.model.ColumnType; +import de.superx.rest.model.Item; +import de.superx.rest.model.Result; +import de.superx.rest.model.ResultType; +import de.superx.rest.model.Row; + +public class ResultBuilder { + + private static final boolean IGNORE_SELF_LOOPS = true; + + private DataSource dataSource; + + private ReportMetadata reportMetadata; + private List columnElements; + + Logger logger = Logger.getLogger(ResultBuilder.class); + + public ResultBuilder() {} + + // used for testing + public ResultBuilder(ReportMetadata metadata, List columns) { + this.reportMetadata = metadata; + this.columnElements = columns; + } + + public ResultBuilder(DataSource dataSource) { + this.dataSource = dataSource; + } + + public void setReportMetadata(ReportMetadata reportMetadata) { + this.reportMetadata = reportMetadata; + } + + public void setColumnElements(List columnElements) { + this.columnElements = columnElements; + } + + private Row buildRowCells(ResultSet rs) { + Row row = new Row(); + Map cells = new TreeMap(); + if (reportMetadata.leftDimensionAttributes != null && !reportMetadata.leftDimensionAttributes.isEmpty()) { + int aggregationLvl = reportMetadata.leftDimensionAttributes.size() -1; + for (DimensionAttribute dimensionAttribute : reportMetadata.leftDimensionAttributes) { + if(dimensionAttribute.isHierarchy()) { + try { + String prevLbl = ""; + int countLvl = 0; + aggregationLvl += reportMetadata.maxBridgeLvl - reportMetadata.minBridgeLvl - 1; + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl; i++) { + Object cell = rs.getObject("col" + i); + String curLbl = (String) cell; + + if(cell == null) { + // An empty cell means a lower aggregation level because + // of how the GROUP BY ROLLUP works. + aggregationLvl--; + } + + if(IGNORE_SELF_LOOPS && + curLbl != null && + curLbl != "" && + curLbl.equals(prevLbl)) { + // If the cell label is equal to the previous cell label + // then this row contains a self loop, meaning the node is + // both its own parent and child. This happens due to the + // GROUP BY ROLLUP part of the sql statement, which groups + // columns in which the same node can appear right next to + // each other in two columns. + continue; + } + + String id = dimensionAttribute.getAttrConformedId(); + if(id == null) { + id = dimensionAttribute.getStringId(); + } + + String cellKey = id + " (Ebene " + countLvl + ")"; + + if(countLvl == 0) { + cellKey = dimensionAttribute.getAttrConformedId(); + } + + if(cell != null && cellKey != null) { + cells.put(cellKey, cell); + row.rowKey += cellKey + cell; + countLvl++; + } + + prevLbl = (String) cell; + } + } catch (SQLException e) { + e.printStackTrace(); + } + } else { + try { + Object val = rs.getObject(dimensionAttribute.getDimensionColumnAlias()); + if(val != null) { + String id = dimensionAttribute.getAttrConformedId(); + if(id == null) { + id = dimensionAttribute.getStringId(); + } + cells.put(id, val); + row.rowKey += id + val; + } else { + aggregationLvl--; + } + } catch (SQLException e) { + e.printStackTrace(); + } + if(dimensionAttribute.getSortOrderColumn() != null) { + try { + String id = dimensionAttribute.getAttrConformedId(); + if(id == null) { + id = dimensionAttribute.getStringId(); + } + cells.put(id + "_sorting", rs.getObject(dimensionAttribute.getDimensionColumnAlias()+"_"+dimensionAttribute.getSortOrderColumn())); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + } + try { + row.aggregated = aggregationLvl; + if(row.aggregated == -1) { + int colNum = this.reportMetadata.maxBridgeLvl + this.columnElements.size() - this.reportMetadata.measures.size(); + for (Measure measure : reportMetadata.measures) { + cells.put(getTotalCellHeaderPrefix(reportMetadata) + measure.getId().composedId, rs.getObject("col" + (colNum++))); + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + + } + if (!columnElements.isEmpty()) { + for (ColumnElement columnElement : columnElements) { + try { + cells.put(columnElement.caption, rs.getObject("col" + columnElement.columnNumber)); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + row.cells = cells; + return row; + } + + + public static List buildColumns(ReportMetadata reportMetadata, List columnElements) { + List columns = new ArrayList(); + if (reportMetadata.leftDimensionAttributes != null) { + reportMetadata.leftDimensionAttributes.forEach((dimensionAttribute) -> { + String id = dimensionAttribute.getAttrConformedId(); + if(id == null) { + id = dimensionAttribute.getStringId(); + } + if(dimensionAttribute.isHierarchy()) { + int cnt = 0; + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl - 1; i++) { + if(cnt == 0) { + columns.add(new Column(id, dimensionAttribute.getCaption(), ColumnType.StringColumn, true)); + } else { + String caption = id + " (Ebene " + cnt + ")"; + String header = dimensionAttribute.getCaption() + " (Ebene " + (cnt) + ")"; + columns.add(new Column(caption, header, ColumnType.HierarchyLevelColumn, true, id, false)); + } + cnt += 1; + } + } else { + if(dimensionAttribute.getSortOrderColumn() != null) { + columns.add(new Column(id, dimensionAttribute.getCaption(), ColumnType.StringColumn, true, dimensionAttribute.getStringId() + "_sorting", false)); + columns.add(new Column(id + "_sorting", dimensionAttribute.getCaption(), ColumnType.SortOrderColumn, false, dimensionAttribute.getStringId(), true)); + }else { + columns.add(new Column(id, dimensionAttribute.getCaption(), ColumnType.StringColumn, true)); + } + } + }); + } + if (!columnElements.isEmpty()) { + columnElements.forEach((columnElement) -> { + Column col = new Column(columnElement.caption, columnElement.header, columnElement.measure.getMeasureType(), false); + col.aggregation = columnElement.measure.getAggregationType(); + columns.add(col); + }); + } + return columns; + } + + public List getRowsForReport(String sqlStatement, Connection con) { + + List rows = null; + try (Statement stmt = con.createStatement(); + ResultSet rs = stmt.executeQuery(sqlStatement)) { + rows = buildRowList(rs); + } catch (SQLException e) { + System.out.println(sqlStatement); + throw new RuntimeException(e); + } + + return rows; + } + + public List buildRowList(ResultSet rs) throws SQLException { + List rows = new ArrayList<>(); + while (rs.next()) { + Row row = buildRowCells(rs); + rows.add(row); + } + List result = new ArrayList<>(); + if (reportMetadata.getHierarchyAttributes().size() > 0) { + HashMap keys = new HashMap<>(); + for (Row row : rows) { + if (keys.containsKey(row.rowKey)) { + Row currentRow = keys.get(row.rowKey ); + if (currentRow.aggregated > row.aggregated) { + keys.put(currentRow.rowKey , row); + } + // workaround: fix multiple aggregated rows, take only highest + if(currentRow.aggregated == -1 && row.aggregated == -1) { + for(String cell : currentRow.cells.keySet()) { + Number currentCellVal = (Number) currentRow.cells.get(cell); + Number candidateVal = (Number) row.cells.get(cell); + if(candidateVal.doubleValue() > currentCellVal.doubleValue()) { + keys.put(currentRow.rowKey, row); + } + } + } + } else { + keys.put(row.rowKey, row); + } + } + rows = new ArrayList(keys.values()); + } + + for (Row row : rows) { + if(row != null) { + result.add(row); + } + } + return rows; + } + + public Result buildReport(List sqlStatements, boolean isCreateRight) { + JdbcTemplate jt = new JdbcTemplate(dataSource); + Result report; + report = new Result(); + if(isCreateRight) { + report.info.setSqlStatements(sqlStatements); + } + + String sql = findByLabel(sqlStatements, "noAggregatesSQL").value; + String sqlColumnTotal = findByLabel(sqlStatements, "totalsColumnSQL").value; + + List columns = buildColumns(reportMetadata, columnElements); + List rows = null; + List totalColumns = null; + + try (Connection con = jt.getDataSource().getConnection()){ + rows = getRowsForReport(sql, con); + } catch (Exception e) { + logger.error(e); + e.printStackTrace(); + report.info.setSegmentCaption(reportMetadata.factTable.getCaption()); + report.info.setErrorMessage(e.getCause().getMessage()); + return report; + } + + // add one column for each measure to the report with the total sum + if(!reportMetadata.topDimensionAttributes.isEmpty()) { + try { + totalColumns = getTotalColumnResult(sqlColumnTotal, jt); + ResultBuilder.setTotalColumnToColumns(columns, reportMetadata); + ResultBuilder.setTotalColumnToRows(rows, totalColumns); + } catch (Exception e) { + e.printStackTrace(); + report.info.setErrorMessage("Die Gesamtspalte konnte nicht ermittelt werden."); + } + } + + setAttributesToReport(report, reportMetadata, rows, columns); + + try { + if(reportMetadata.hideEmptyColumns) { + removeEmptyColumns(columns, rows); + } + } catch(Exception e ) { + logger.error(e); + } + + return report; + } + + public static void removeEmptyColumns(List columns, List rows) { + HashMap map = new HashMap<>(); + for (Column col : columns) { + if(!map.containsKey(col.field)) { + map.put(col.field, Integer.valueOf(-1)); + } + } + + for (Row row : rows) { + for (String cellKey : row.cells.keySet()) { + Object value = row.cells.get(cellKey); + if(value instanceof Number) { + Number val = (Number) value; + if(val.intValue() != 0) { + if(map.containsKey(cellKey)) { + map.remove(cellKey); + } + } + } else { + if(value == null) { + continue; + } + if(map.containsKey(cellKey)) { + map.remove(cellKey); + } + } + } + } + + + for (Row row : rows) { + for(String key : map.keySet()) { + if(row.cells.containsKey(key)) { + row.cells.remove(key); + } + } + } + + if(rows.size() > 1) { + for(Iterator iterator = columns.iterator(); iterator.hasNext(); ) { + if(map.containsKey(iterator.next().field)) + iterator.remove(); + } + } + } + + public static void setAttributesToReport(Result report, ReportMetadata reportMetadata, List rows, List columns) { + report.setResultType(ResultType.DrilldownTableGroupable); + report.setRows(rows); + report.setColumns(columns); + if(reportMetadata.factTable.getCaption() != null) { + report.info.setSegmentCaption(reportMetadata.factTable.getCaption()); + InfoItem facttableInfo = new InfoItem (reportMetadata.factTable.getId().composedId, + reportMetadata.factTable.getCaption(), + reportMetadata.factTable.getDescription()); + report.info.addFacttable(facttableInfo); + report.info.addSachgebiet(reportMetadata.sachgebiet.name); + } + report.info.setMeasures(reportMetadata.getMeasureInfo()); + report.info.setLeftDimensionAttributes(reportMetadata.getLeftDimAttrAsInfo()); + report.info.setTopDimensionAttributes(reportMetadata.getTopDimAttrAsInfo()); + report.info.setFilter(reportMetadata.getFilterAsInfo()); + report.info.setLastUpdateBiad(reportMetadata.lastBiadUpdate); + report.info.hideEmptyColumns(reportMetadata.hideEmptyColumns); + } + + private static String getTotalCellHeaderPrefix(ReportMetadata reportMetadata) { + String totalCellHeaderPrefix = ""; + for (int i = 0; i < reportMetadata.topDimensionAttributes.size(); i++) { + DimensionAttribute attr = reportMetadata.topDimensionAttributes.get(i); + if(i == 0) { + totalCellHeaderPrefix += ColumnElement.buildField(attr, "Gesamt"); + } else { + totalCellHeaderPrefix += ColumnElement.buildField(attr, " "); + } + } + totalCellHeaderPrefix += " || Kennzahl|"; + return totalCellHeaderPrefix; + } + + private static String getTotalCellHeaderPrefixHeader(ReportMetadata reportMetadata) { + String totalCellHeaderPrefix = ""; + for (int i = 0; i < reportMetadata.topDimensionAttributes.size(); i++) { + DimensionAttribute attr = reportMetadata.topDimensionAttributes.get(i); + if(i == 0) { + totalCellHeaderPrefix += attr.getCaption() + ": " + attr.getCaption() + "| Gesamt "; + } else { + totalCellHeaderPrefix += " || " + attr.getCaption() + ": " + attr.getCaption() + "| "; + } + } + totalCellHeaderPrefix += " || Kennzahl|"; + return totalCellHeaderPrefix; + } + + public List getTotalColumnResult(String sqlStatement, JdbcTemplate jt) { + if(sqlStatement.isEmpty()) { + return null; + } + List rows = null; + rows = jt.query(sqlStatement, new Object[0], new RowMapper() { + @Override + public Row mapRow(ResultSet rs, int rowNum) { + Row row = new Row(); + Map cells = new TreeMap(); + int numCols = reportMetadata.maxBridgeLvl; + try { + for (DimensionAttribute attr : reportMetadata.leftDimensionAttributes) { + if(attr.isHierarchy()) { + String prevCell = ""; + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl; i++) { + Object cell = rs.getObject("col" + i); + if(cell == null || cell.equals(prevCell)) { + continue; + } + String cellKey = attr.getStringId() + " (Ebene " + i + ")"; + if(i == 0) { + cellKey = attr.getStringId(); + } + row.rowKey += cellKey + cell; + prevCell = (String) cell; + } + } else { + Object val = rs.getObject(attr.getDimensionColumnAlias()); + if(val != null) { + String id = attr.getAttrConformedId(); + if(id == null) { + id = attr.getStringId(); + } + cells.put(id, val); + row.rowKey += id + val; + } + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + for (Measure measure : reportMetadata.measures) { + try { + String key = getTotalCellHeaderPrefix(reportMetadata) + measure.getId().composedId; + Object val = rs.getObject("col" + numCols++); + cells.put(key, val); + } catch (SQLException e) { + e.printStackTrace(); + } + } + row.cells = cells; + return row; + } + }); + + if(reportMetadata.getHierarchyAttributes().size() > 0) { + HashMap rowKeyValue = new HashMap<>(); + for (Row row : rows) { + boolean replace = false; + if(rowKeyValue.containsKey(row.rowKey)) { + Row alreadyThere = rowKeyValue.get(row.rowKey); + for(String key : alreadyThere.cells.keySet()) { + Number candidateVal = (Number) row.cells.get(key); + if(candidateVal == null) { + continue; + } + Number alreadyVal = (Number) alreadyThere.cells.get(key); + if(alreadyVal == null) { + continue; + } + if(candidateVal.doubleValue() > alreadyVal.doubleValue()) { + replace = true; + } + } + if(replace) { + rowKeyValue.put(row.rowKey, row); + replace = false; + } + } else { + rowKeyValue.put(row.rowKey, row); + } + } + return new ArrayList<>(rowKeyValue.values()); + } + + return rows; + } + + public static void setTotalColumnToRows(List rows, List result) { + for (Row row : rows) { + for (Row r : result) { + if(r.rowKey.equals(row.rowKey)) { + row.cells.putAll(r.cells); + } + } + } + //for (int i = 0; i < rows.size(); i++) { + // Row row = rows.get(i); + // if(row.aggregated == -1) { + // continue; + // } + // row.cells.putAll(result.get(i).cells); + //} + } + + public static void setTotalColumnToColumns(List columns, ReportMetadata reportMetadata) { + for (Measure measure : reportMetadata.measures) { + String field = getTotalCellHeaderPrefix(reportMetadata) + measure.getId().composedId; + String header = getTotalCellHeaderPrefixHeader(reportMetadata) + measure.getCaption(); + Column col = new Column(field, header, measure.getMeasureType(), false); + col.setHidden(true); + col.setTotalColumn(true); + columns.add(col); + } + + } + + private static Item findByLabel(List items, String label) { + return items.stream() + .filter(s -> s.label.equals(label)) + .findAny() + .get(); + } + +} diff --git a/src/de/superx/bianalysis/ResultMerger.java b/src/de/superx/bianalysis/ResultMerger.java new file mode 100644 index 0000000..45a3990 --- /dev/null +++ b/src/de/superx/bianalysis/ResultMerger.java @@ -0,0 +1,158 @@ +package de.superx.bianalysis; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.FactTable; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.InfoItem; +import de.superx.bianalysis.service.DbMetaAdapter; +import de.superx.jdbc.entity.Sachgebiet; +import de.superx.rest.model.Column; +import de.superx.rest.model.Result; +import de.superx.rest.model.Row; + +public class ResultMerger { + + private DbMetaAdapter dbAdapter; + + public ResultMerger(DbMetaAdapter dbAdapter) { + this.dbAdapter = dbAdapter; + } + + public Result buildMergedReport(ReportDefinition definition, List reportResults) { + + Result result = new Result(); + ReportMetadata metadata = new ReportMetadata(definition, null, dbAdapter); + + List columnElements = ColumnElementBuilder.buildColumnElements(metadata); + List columns = ResultBuilder.buildColumns(metadata, columnElements); + if(!metadata.topDimensionAttributes.isEmpty()) { + ResultBuilder.setTotalColumnToColumns(columns, metadata); + } + + // create list of merged rows + List> allRows = Result.getRowsFromReports(reportResults); + List rows = mergeRows(allRows); + + if(metadata.hideEmptyColumns) { + ResultBuilder.removeEmptyColumns(columns, rows); + } + + ResultBuilder.setAttributesToReport(result, metadata, rows, columns); + + // override merge report specific attributes + result.setSubResults(reportResults); + List factTablesInfo = getFactTablesAsInfo(dbAdapter, definition.factTableIds); + result.info.setSegmentCaption(factTablesInfo.stream().map(f -> f.caption).collect(Collectors.joining(", "))); + result.info.setSachgebiete(getSachgebieteAsInfo(dbAdapter, definition.factTableIds)); + result.info.setFacttables(factTablesInfo); + + for (Result r : reportResults) { + if(r.info.error != null && !r.info.error.isBlank()) { + result.info.setErrorMessage(r.info.error); + break; + } + } + return result; + } + + private static List getSachgebieteAsInfo(DbMetaAdapter dbAdapter, List factTableIds) { + List sachgebiete = new ArrayList(); + List tids = new ArrayList<>(); + for (Identifier id : factTableIds) { + FactTable factTable = dbAdapter.getFactTable(id); + Sachgebiet sachgebiet = dbAdapter.getSachgebietById(factTable.getSachgebiettid()); + Integer tid = sachgebiet.tid; + if(!tids.contains(tid)) { + tids.add(tid); + sachgebiete.add(sachgebiet.name.trim()); + } + } + return sachgebiete; + } + + private static List getFactTablesAsInfo(DbMetaAdapter dbAdapter, List factTableIds) { + List facttables = new ArrayList(); + for (Identifier id : factTableIds) { + FactTable factTable = dbAdapter.getFactTable(id); + facttables.add(new InfoItem(factTable.getId().composedId, factTable.getCaption(), factTable.getDescription())); + } + return facttables; + } + + public ReportDefinition createFactTableSpecificReportDefinition(ReportDefinition reportDefinition, + Identifier factTableId) { + ReportDefinition definition = new ReportDefinition(); + definition.hideEmptyColumns = reportDefinition.hideEmptyColumns; + definition.factTableIds.add(factTableId); + for (int i = 0; i < reportDefinition.leftDimensionAttributeIds.size(); i++) { + Identifier attr = reportDefinition.leftDimensionAttributeIds.get(i); + Identifier checkedAttr = dbAdapter.checkIfFactTableHasDimensionAttribute(attr, factTableId); + if(checkedAttr != null) { + definition.leftDimensionAttributeIds.add(checkedAttr); + } + } + for (int i = 0; i < reportDefinition.topDimensionAttributeIds.size(); i++) { + Identifier attr = reportDefinition.topDimensionAttributeIds.get(i); + Identifier checkedAttr = dbAdapter.checkIfFactTableHasDimensionAttribute(attr, factTableId); + if(checkedAttr != null) { + definition.topDimensionAttributeIds.add(checkedAttr); + } + } + for (int i = 0; i < reportDefinition.measureIds.size(); i++) { + Identifier measure = reportDefinition.measureIds.get(i); + if(dbAdapter.checkIfFactTableHasMeasure(measure, factTableId)) { + definition.measureIds.add(measure); + } + } + for (int i = 0; i < reportDefinition.filters.size(); i++) { + Filter filter = reportDefinition.filters.get(i); + Identifier checkedAttr = dbAdapter.checkIfFactTableHasDimensionAttribute(filter.dimensionAttributeId, factTableId); + if(checkedAttr != null) { + Filter roleFilter = new Filter(filter); + roleFilter.dimensionAttributeId = checkedAttr; + definition.filters.add(roleFilter); + } + } + return definition; + } + + public static List mergeRows(List> rows) { + List result = new ArrayList<>(); + + for (List inputRows : rows) { + for (Row row : inputRows) { + + Row rowRepl = new Row(row.aggregated); + rowRepl.rowKey = row.rowKey; + + for(String key : row.cells.keySet()){ + String newKey = key; + rowRepl.cells.put(newKey, row.cells.get(key)); + } + + if(!result.contains(rowRepl)) { + result.add(rowRepl); + } else { + // row with the same rowkey exists -> add only cells + Row found = result.get(result.indexOf(rowRepl)); + for(String key : rowRepl.cells.keySet()){ + if(!found.cells.containsKey(key)) { + found.cells.put(key, rowRepl.cells.get(key)); + } + } + } + } + } + + return result; + } + +} diff --git a/src/de/superx/bianalysis/StoredReport.java b/src/de/superx/bianalysis/StoredReport.java new file mode 100644 index 0000000..c04614f --- /dev/null +++ b/src/de/superx/bianalysis/StoredReport.java @@ -0,0 +1,82 @@ +package de.superx.bianalysis; + +import java.util.ArrayList; + + +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.Transient; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.json.JsonMapper; + +import de.superx.rest.model.Result; +import de.superx.rest.model.TreeNode; + +@Table(value ="metadata\".\"rw_report_definitions") +public class StoredReport { + + @Id + public int id; + + public String name; + + public String description; + + public String definition; + + @Column(value = "show_total_column") + @JsonProperty("show_total_column") + public int showTotalColumn; + + @Transient + public Boolean isReadOnly = Boolean.FALSE; + + @Transient + public ReportDefinition reportDefinition; + + @Transient + public Result exportedResult; + + @Transient + public ArrayList hierarchy; + + public StoredReport(String name, ReportDefinition reportDefinition, Result exportedResult) { + super(); + this.name = name; + this.reportDefinition = reportDefinition; + this.exportedResult = exportedResult; + } + + public StoredReport() { + super(); + } + + public static void setReportDefinitionJson(StoredReport report) { + ObjectWriter ow = new ObjectMapper().writer(); + String reportDefinitionJson = null; + try { + reportDefinitionJson = ow.writeValueAsString(report.reportDefinition); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + report.definition = reportDefinitionJson; + } + + public static void setReportDefinitionFromJson(StoredReport report) { + ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); + ReportDefinition reportDefinition = null; + try { + reportDefinition = mapper.readValue(report.definition, ReportDefinition.class); + } catch (Exception e) { + e.printStackTrace(); + } + report.reportDefinition = reportDefinition; + report.definition = ""; + } + +} diff --git a/src/de/superx/bianalysis/bin/BiAnalysisCLI.java b/src/de/superx/bianalysis/bin/BiAnalysisCLI.java new file mode 100644 index 0000000..68ccd17 --- /dev/null +++ b/src/de/superx/bianalysis/bin/BiAnalysisCLI.java @@ -0,0 +1,302 @@ +package de.superx.bianalysis.bin; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.metadata.MetaImport; +import de.superx.bianalysis.metadata.MetaJson; +import de.superx.bianalysis.metadata.MetadataImporter; +import de.superx.bianalysis.metadata.models.json.MetaDimension; +import de.superx.bianalysis.metadata.models.json.MetaDimensionAttribute; +import de.superx.bianalysis.metadata.models.json.MetaFact; +import de.superx.bianalysis.metadata.models.json.MetaMeasure; +import de.superx.bianalysis.metadata.models.json.MetaObject; +import de.superx.bianalysis.metadata.models.yml.MetaYml; +import de.superx.servlet.SuperXManager; +import de.superx.util.PathAndFileUtils; + +public class BiAnalysisCLI { + + private static final String DEFAULT_RELEASE_BRANCH = "2025_12"; + + public static void main(String[] args) throws IOException { + Options options = createOptions(); + CommandLine parsedArgs = readArgs(args, options); + if(parsedArgs.hasOption("-i")) { + addMissingIdsInMetadataDir(parsedArgs); + } else if(parsedArgs.hasOption("-m")) { + convertJsonFilesToSql(); + } else if(parsedArgs.hasOption("-y")) { + generateYmlForJsonFile(parsedArgs); + } else if(parsedArgs.hasOption("-d")) { + generateWikiMarkdown(parsedArgs); + } else { + printHelp(options); + } + } + + private static void generateWikiMarkdown(CommandLine parsedArgs) throws IOException { + SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath()); + String facttable = parsedArgs.getOptionValue("d"); + String filePath = PathAndFileUtils.getReportGeneratorDir("hisinone"); + String ymlPath = PathAndFileUtils.getDbtTransformDirectory("hisinone") + File.separator + "docs_and_tests"; + MetadataImporter importer = new MetadataImporter(ymlPath); + Logger.getLogger(MetadataImporter.class).setLevel(Level.ERROR); + importer.deserializeMetadataFromJsonFiles(filePath); + + String docDirectory = PathAndFileUtils.getModulePath("biad"); + docDirectory = String.join(File.separator, docDirectory, "conf", "his1", "edustore_doc"); + + for(MetaFact fact : importer.getAllFactTables()) { + if("all".equals(facttable) || fact.getFacttable().equals(facttable)) { + PrintWriter writer = new PrintWriter(docDirectory + File.separator + fact.getFacttable() + "_mediawiki.txt", "UTF-8"); + writer.println("===Kennzahlen ==="); + writer.println(";Kennzahlen " + fact.getCaption()); + writer.println(); + writer.println("{| class=\"wikitable\"\n ! Kennzahl !! Beschreibung"); + writer.println("|-"); + for(int i = 0; i < fact.getMeasures().size(); i++) { + MetaMeasure m = fact.getMeasures().get(i); + writer.println("| "+m.getCaption()); + writer.println("| "+m.getDescription()); + if(i != fact.getMeasures().size() - 1) { + writer.println("|-"); + + } + } + writer.println("|-\n|}"); + writer.println(); + writer.println(); + + writer.println("===Dimension ==="); + writer.println(";"+fact.getCaption()); + writer.println(":"+fact.getDescription()); + writer.println(); + writer.println("'''Dimension und Dimensionsattribut'''"); + writer.println(); + for(MetaDimension d : fact.getDimensions()) { + + String dimCaption = d.getCaption(); + if((dimCaption == null || dimCaption.isBlank()) && d.getConformedDimension() != null) { + dimCaption = d.getConformedDimension().getCaption(); + } + + String dimDescription = d.getDescription(); + if((dimDescription == null || dimDescription.isBlank()) && d.getConformedDimension() != null) { + dimDescription = d.getConformedDimension().getDescription(); + } + + writer.println(";"+dimCaption); + writer.println(":"+dimDescription); + + List attributes = d.getAttributes(); + if(attributes == null || attributes.size() == 0) { + attributes = d.getConformedDimension().getAttributes(); + } + + for(MetaDimensionAttribute a : attributes) { + String caption = a.getCaption() == null ? a.getConfDimAttrRef().getCaption() : a.getCaption(); + writer.println("*"+caption); + try { + String desc = a.getDescription() == null ? a.getConfDimAttrRef().getDescription() : a.getDescription(); + if(!desc.equals("null")) { + writer.println("*:"+desc); + } + } catch (Exception e) { + // TODO: handle exception + } + } + writer.println(); + } + writer.close(); + } + } + + } + + private static void generateYmlForJsonFile(CommandLine parsedArgs) { + String file = parsedArgs.getOptionValue("y"); + if(file == null || !new File(file).exists()) { + throw new RuntimeException("File " + file +" is not valid."); + } + MetadataImporter importer = new MetadataImporter(); + Logger.getLogger(MetadataImporter.class).setLevel(Level.ERROR); + importer.setShouldReadYMLDoc(false); + importer.deserializeMetadataFromJsonFiles(file); + MetaImport metaImport = importer.getMetaImports().get(0); + MetaYml yml = importer.createYMLFileForMetaJson(metaImport); + System.out.println(MetadataImporter.writeYmlToString(yml)); + } + + private static void addMissingIdsInMetadataDir(CommandLine parsedArgs) { + String[] files = parsedArgs.getOptionValues("i"); + BasicConfigurator.configure(); // initializes console logging to stdout + try { + MetadataImporter metaImporter = new MetadataImporter(); + metaImporter.setShouldReadYMLDoc(false); + metaImporter.deserializeMetadataFromJsonFiles(files); + for (MetaJson meta : metaImporter.getMetaJsons()) { + List allIds = meta.getIds(); + boolean isFileUpdateNecessary = false; + for (MetaObject obj : meta.getMetaObjects()) { + if(obj.getId() == null) { + Identifier id = Identifier.getNewIdentifierValue(allIds, obj.getNamespace()); + obj.setId(id); + allIds.add(id); + isFileUpdateNecessary = true; + } + if(obj.getDefaultRelease() == null) { + obj.setDefaultRelease(DEFAULT_RELEASE_BRANCH); + isFileUpdateNecessary = true; + } + } + if(isFileUpdateNecessary) { + writeMetaImportToFile(meta); + Logger.getRootLogger().info("Updated file " + meta.getFile().getPath()); + } + } + if (!metaImporter.errorMessages.isEmpty()) { + System.out.println(metaImporter.getPrintableErrorMessages()); + System.exit(1); + } + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + private static void convertJsonFilesToSql() { + SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath()); + String filePath = PathAndFileUtils.getReportGeneratorDir("hisinone"); + String ymlPath = PathAndFileUtils.getDbtTransformDirectory("hisinone"); + + String out = + "DROP TABLE IF EXISTS metadata.facttable; " + + "DROP TABLE IF EXISTS metadata.measure; " + + "DROP TABLE IF EXISTS metadata.measure_filter; " + + "DROP TABLE IF EXISTS metadata.dimension; " + + "DROP TABLE IF EXISTS metadata.dimension_attribute; "; + + Path schemaSqlDir = Path.of("superx", "WEB-INF", "conf", "edustore", "db", "install", "schluesseltabellen"); + + out += readLinesWithNewline(new File(schemaSqlDir.toString() + File.separator + "biad_create_meta_tables.sql")); + out += readLinesWithNewline(new File(schemaSqlDir.toString() + File.separator + "biad_alter_meta_tables.sql")); + out += readLinesWithNewline(new File(schemaSqlDir.toString() + File.separator + "biad_metadaten_fuellen.sql")); + + Logger.getLogger(MetadataImporter.class).setLevel(Level.ERROR); + MetadataImporter importer = new MetadataImporter(ymlPath); + importer.deserializeMetadataFromJsonFiles(filePath); + out += String.join("\n", importer.getAllUpsertStrings(false)); + + if(!importer.errorMessages.isEmpty()) { + System.out.println(importer.getPrintableErrorMessages()); + System.exit(1); + } else { + System.out.println(out); + + } + + } + + private static CommandLine readArgs(String[] args, Options options) { + CommandLineParser parser = new GnuParser(); + try { + return parser.parse(options, args, false); + } catch (ParseException e) { + e.printStackTrace(); + System.exit(1); + } + return null; + } + + private static void printHelp(Options options) { + HelpFormatter help = new HelpFormatter(); + help.setOptionComparator(null); + help.setWidth(200); + help.printHelp("This tool streamlines common tasks during development for the BIAnalysis.", options); + } + + private static Options createOptions() { + Options options = new Options(); + + OptionBuilder.withDescription("convert metadata directory to sql"); + OptionBuilder.withLongOpt("convert-metadata"); + Option outMeta = OptionBuilder.create("m"); + + OptionBuilder.withDescription("generate yml documentation for json file"); + OptionBuilder.withLongOpt("generate-yml"); + OptionBuilder.withArgName("json-file"); + OptionBuilder.hasArg(true); + Option generateYml = OptionBuilder.create("y"); + + + OptionBuilder.withDescription("generate wiki documentation for measures and dimensions"); + OptionBuilder.withLongOpt("generate-doc"); + OptionBuilder.withArgName("facttable"); + OptionBuilder.hasArg(true); + Option generateDoc = OptionBuilder.create("d"); + + OptionBuilder.withLongOpt("add-ids"); + OptionBuilder.withDescription("add missing ids to json files"); + OptionBuilder.withArgName("directories"); + OptionBuilder.hasArgs(); + Option updateIds = OptionBuilder.create("i"); + + options.addOption(updateIds); + options.addOption(generateYml); + options.addOption(generateDoc); + options.addOption(outMeta); + options.addOption(new Option("h", "help", false, "get help")); + + return options; + } + + public static void writeMetaImportToFile(MetaJson meta) { + ObjectMapper mapper = new ObjectMapper(); + DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter(" ", DefaultIndenter.SYS_LF); + DefaultPrettyPrinter printer = new DefaultPrettyPrinter(); + printer.indentObjectsWith(indenter); + printer.indentArraysWith(indenter); + try { + mapper.writer(printer).writeValue(meta.getFile(), meta); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String readLinesWithNewline(File file) { + String result = ""; + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String line; + while ((line = br.readLine()) != null) { + result += line; + } + } catch (IOException e) { + e.printStackTrace(); + } + return result+"\n"; + } + +} diff --git a/src/de/superx/bianalysis/metadata/Identifier.java b/src/de/superx/bianalysis/metadata/Identifier.java new file mode 100644 index 0000000..aff28eb --- /dev/null +++ b/src/de/superx/bianalysis/metadata/Identifier.java @@ -0,0 +1,74 @@ +package de.superx.bianalysis.metadata; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@JsonSerialize(using = IdentifierSerializer.class) +public class Identifier { + + private static final String ID_SEPARATOR = ":"; + + @JsonIgnore + public Integer value; + @JsonIgnore + public String namespace; + + public String composedId; + + public Identifier(String composedId) { + this.composedId = composedId; + String[] result = composedId.split(ID_SEPARATOR); + this.namespace = result[0]; + this.value = Integer.valueOf(result[1]); + } + + public Identifier(Identifier id) { + this.value = id.value; + this.namespace = id.namespace; + this.composedId = id.composedId; + } + + @JsonIgnore + public static Identifier getNewIdentifierValue(List list, String namespace) { + List values = list + .stream() + .filter(i->i.value!=null) + .map(i->i.value) + .collect(Collectors.toList()); + Integer value; + if(values.isEmpty()) { + value = Integer.valueOf(1); + } else { + value = Integer.valueOf(Collections.max(values).intValue() + 1); + } + return new Identifier(namespace + ID_SEPARATOR + value); + } + + @Override + @JsonIgnore + public boolean equals(Object obj) { + if(obj == null || !(obj instanceof Identifier)) { + return false; + } + Identifier id = (Identifier) obj; + return id.composedId.equals(this.composedId); + } + + @Override + @JsonIgnore + public int hashCode() { + return this.value.hashCode() + this.namespace.hashCode(); + } + + @Override + public String toString() { + return composedId; + } + + + +} diff --git a/src/de/superx/bianalysis/metadata/IdentifierSerializer.java b/src/de/superx/bianalysis/metadata/IdentifierSerializer.java new file mode 100644 index 0000000..39b4891 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/IdentifierSerializer.java @@ -0,0 +1,27 @@ +package de.superx.bianalysis.metadata; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +public class IdentifierSerializer extends StdSerializer{ + + public IdentifierSerializer() { + this(null); + } + + public IdentifierSerializer(Class t) { + super(t); + } + + @Override + public void serialize( + Identifier id, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + jgen.writeRawValue('"'+id.composedId+'"'); + } + +} diff --git a/src/de/superx/bianalysis/metadata/MetaImport.java b/src/de/superx/bianalysis/metadata/MetaImport.java new file mode 100644 index 0000000..6e74a50 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/MetaImport.java @@ -0,0 +1,148 @@ +package de.superx.bianalysis.metadata; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.log4j.Logger; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import de.superx.bianalysis.FaultyMetadataException; +import de.superx.bianalysis.metadata.models.json.MetaDimension; +import de.superx.bianalysis.metadata.models.json.MetaDimensionAttribute; +import de.superx.bianalysis.metadata.models.json.MetaFact; +import de.superx.bianalysis.metadata.models.json.MetaMeasure; +import de.superx.bianalysis.metadata.models.json.MetaMeasureFilter; +import de.superx.bianalysis.metadata.models.json.MetaObject; + +public class MetaImport extends MetaJson { + + public List facts; + + private static Logger log = Logger.getLogger(MetaImport.class); + + @JsonIgnore + private Map keysForMeasureFilter = new HashMap<>(); + + @JsonIgnore + public List conformedDimensions; + + @JsonIgnore + public void setConformedDimensions(List conformedDimensions){ + this.conformedDimensions = conformedDimensions; + if(this.conformedDimensions != null) { + for (MetaDimension dim : this.conformedDimensions) { + for (MetaDimensionAttribute attr : dim.getAttributes()) { + attr.setDimension(dim); + keysForMeasureFilter.put(dim.getDimension()+"."+attr.getDimColumn(), attr); + } + } + } + } + + @JsonIgnore + @Override + public void init() { + this.allMetaObj = new ArrayList(); + for (MetaFact fact : this.facts) { + allMetaObj.add(fact); + for (MetaDimension dim : fact.getDimensions()) { + allMetaObj.add(dim); + if(dim.getRefTo() != null && !dim.getRefTo().isEmpty()) { + MetaDimension conformedDim = findByRef(dim); + dim.setConformedDimension(conformedDim); + } + for (MetaDimensionAttribute attr : dim.getAttributes()) { + allMetaObj.add(attr); + if(attr.getRefTo() != null && !attr.getRefTo().isEmpty()) { + attr.setConformedDimensionAttribute(findByRefAttr(dim.getConformedDimension().getDimension(), attr)); + } + keysForMeasureFilter.put(dim.getDimension()+"."+attr.getDimColumn(), attr); + } + } + if(fact.getMeasures() != null) { + for (MetaMeasure measure : fact.getMeasures()) { + allMetaObj.add(measure); + MetaMeasureFilter filter = measure.getFilter(); + if(filter != null) { + if(filter.getDimensionRef() != null && !filter.getDimensionRef().isBlank()) { + MetaDimensionAttribute attr = keysForMeasureFilter.get(filter.getDimensionRef()); + if(attr == null) { + throw new FaultyMetadataException("Could not resolve dimensionRef '" + filter.getDimensionRef() + + "' (" + file.getName() + " -> " + fact.getFacttable() + ")"); + } + filter.setAttribute(attr); + allMetaObj.add(filter); + } else if(filter.getFactColumnRef() != null && !filter.getFactColumnRef().isBlank()) { + allMetaObj.add(filter); + } + } + } + } + } + } + + private MetaDimensionAttribute findByRefAttr(String dimensionTable, MetaDimensionAttribute attribute) { + String attributeColumn = attribute.getRefTo(); + MetaDimensionAttribute confAttr = null; + for (MetaDimension confDim : this.conformedDimensions) { + if(!confDim.getDimension().equals(dimensionTable)) { + continue; + } + for (MetaDimensionAttribute attr : confDim.getAttributes()) { + if(attr.getDimColumn().equals(attributeColumn)) { + confAttr = attr; + break; + } + } + } + if(confAttr == null) { + throw new FaultyMetadataException( + "Could not resolve attribute reference '" + attributeColumn + "' (" + + file.getName() + " -> " + + attribute.getDimension().getFact().getFacttable() + " -> " + + attribute.getDimension().getRefTo() + ")" + ); + } + return confAttr; + } + + + @JsonIgnore + private MetaDimension findByRef(MetaDimension dim) { + String refTo = dim.getRefTo(); + MetaDimension resolvedRefTo = null; + for (MetaDimension dimConf : this.conformedDimensions) { + if (dimConf.getDimension() == null) { + log.error("Missing dimension attribute for " + dimConf.getCaption()); + continue; + } + if (dimConf.getDimension().equals(refTo)) { + resolvedRefTo = dimConf; + break; + } + } + if (resolvedRefTo == null) { + throw new FaultyMetadataException("Could not resolve dimension reference '" + refTo + "' (" + file.getName() + " -> " + dim.getFact().getFacttable() + ")"); + } + return resolvedRefTo; + } + + @JsonIgnore + public List getDimensionsWithoutRefTo() { + List dims = new ArrayList<>(); + for (MetaFact fact : facts) { + for (MetaDimension dim : fact.getDimensions()) { + if(dim.getRefTo() == null) { + dims.add(dim); + } + } + } + return dims; + } + + +} diff --git a/src/de/superx/bianalysis/metadata/MetaImportConformedDimensions.java b/src/de/superx/bianalysis/metadata/MetaImportConformedDimensions.java new file mode 100644 index 0000000..f84652e --- /dev/null +++ b/src/de/superx/bianalysis/metadata/MetaImportConformedDimensions.java @@ -0,0 +1,38 @@ +package de.superx.bianalysis.metadata; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import de.superx.bianalysis.metadata.models.json.MetaDimension; +import de.superx.bianalysis.metadata.models.json.MetaDimensionAttribute; +import de.superx.bianalysis.metadata.models.json.MetaMeasure; +import de.superx.bianalysis.metadata.models.json.MetaObject; + +public class MetaImportConformedDimensions extends MetaJson { + + @JsonProperty("conformed_dimensions") + public List conformedDimensions; + + @Override + public void init() { + this.allMetaObj = new ArrayList(); + for (MetaDimension metaDimension : this.conformedDimensions) { + metaDimension.setConformed(true); + this.allMetaObj.add(metaDimension); + for (MetaDimensionAttribute attr : metaDimension.getAttributes()) { + attr.setDimension(metaDimension); + this.allMetaObj.add(attr); + } + } + + for (MetaObject metaObject : allMetaObj) { + metaObject.setNamespace(this.namespace); + if(metaObject.getId() != null) { + metaObject.getId().namespace = this.namespace; + } + } + } + +} diff --git a/src/de/superx/bianalysis/metadata/MetaJson.java b/src/de/superx/bianalysis/metadata/MetaJson.java new file mode 100644 index 0000000..4d06aa4 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/MetaJson.java @@ -0,0 +1,58 @@ +package de.superx.bianalysis.metadata; + +import java.io.File; +import java.util.List; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import de.superx.bianalysis.metadata.models.json.MetaObject; + +public abstract class MetaJson { + + public String namespace; + + @JsonIgnore + protected File file; + + @JsonIgnore + protected List allMetaObj; + + @JsonIgnore + public abstract void init(); + + @JsonIgnore + public File getFile() { + return this.file; + } + + @JsonIgnore + public List getMetaObjects() { + return this.allMetaObj; + } + + @JsonIgnore + public void setFile(File file) { + this.file = file; + } + + @JsonIgnore + public List getIds() { + return this.getMetaObjects() + .stream() + .map(o -> o.getId()) + .filter(i -> i != null && i.composedId != null) + .collect(Collectors.toList()); + } + + @JsonIgnore + public void setNamespaceToMetaObjects() { + for (MetaObject metaObject : allMetaObj) { + metaObject.setNamespace(this.namespace); + if(metaObject.getId() != null) { + metaObject.getId().namespace = this.namespace; + } + } + } + +} diff --git a/src/de/superx/bianalysis/metadata/MetadataImporter.java b/src/de/superx/bianalysis/metadata/MetadataImporter.java new file mode 100644 index 0000000..177c0c6 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/MetadataImporter.java @@ -0,0 +1,593 @@ +package de.superx.bianalysis.metadata; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FilenameFilter; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import javax.sql.DataSource; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + +import de.superx.bianalysis.StoredReport; +import de.superx.bianalysis.metadata.models.json.MetaDimension; +import de.superx.bianalysis.metadata.models.json.MetaDimensionAttribute; +import de.superx.bianalysis.metadata.models.json.MetaFact; +import de.superx.bianalysis.metadata.models.json.MetaObject; +import de.superx.bianalysis.metadata.models.yml.MetaYml; +import de.superx.bianalysis.metadata.models.yml.MetaYmlModel; +import de.superx.bianalysis.metadata.models.yml.MetaYmlModelColumns; +import de.superx.util.PathAndFileUtils; + +/** + * Provides functionality for updating the tables in the metadata schema. + * The tables are updated by reading the metadata information from various + * metaimport.json files and transforming that information into executable sql. + * + * The BIAnalysis Tool uses the tables to read information about the different + * meta objects and more importantly to figure out their relationships, e.g. what + * dimension is part of which facttable or which attribute belongs to which + * dimension. + * + * To learn more about the metadata concept for the BIAnalysis Tool see: + * doc\bi_analysis\report_wizard\metadaten.adoc + * + */ +public final class MetadataImporter { + + /** + * Each file containing metadata information must have the following file suffix. + */ + private static final String METAIMPORT_FILE_SUFFIX = "_metaimport.json"; + + protected static final String CONFORMED_DIMENSIONS_FILE_SUFFIX = "conformed_dimensions" + METAIMPORT_FILE_SUFFIX; + + /** + * Holds all Metaimport objects with which this instance was initalized. + * (One MetaImport object corresponds to exactly one deserialized json file) + */ + private List metaImports = new ArrayList<>(); + + public List errorMessages = new ArrayList<>(); + + /** + * SQL String for deleting from all metadata tables except 'custom' releases. + */ + public static final String TRUNCATE_METADATA_SQL = + "DELETE FROM metadata.facttable WHERE default_release != 'custom' or default_release is null; " + + "DELETE FROM metadata.measure WHERE default_release != 'custom' or default_release is null; " + + "DELETE FROM metadata.measure_filter WHERE default_release != 'custom' or default_release is null; " + + "DELETE FROM metadata.dimension WHERE default_release != 'custom' or default_release is null; " + + "DELETE FROM metadata.dimension_attribute WHERE default_release != 'custom' or default_release is null; "; + + private static Logger log = Logger.getLogger(MetadataImporter.class); + + private boolean shouldReadYMLDoc = true; + + private String ymlDir = ""; + + public MetadataImporter() {} + + public MetadataImporter(String ymlDir) {this.ymlDir = ymlDir;} + + /** + * Calling this method initalizes the MetadataImporter by deserializing all unique meta objects + * from the provided json files. Faulty json files are ignored. + * + * @param paths Path(s) to the metadata file(s). A path can point to a directory or a file. + * Multiple paths and/or directories can be provided. + */ + public void deserializeMetadataFromJsonFiles(String... paths) { + + ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); + List conformedDimension = new ArrayList<>(); + List conformedDims = new ArrayList<>(); + + for (String path : paths) { + List metaFiles = readMetaImportFiles(path); + for (File file : metaFiles) { + MetaJson meta = null; + try{ + if(file.getName().endsWith(CONFORMED_DIMENSIONS_FILE_SUFFIX)) { + meta = mapper.readValue(file, MetaImportConformedDimensions.class); + conformedDimension.add((MetaImportConformedDimensions) meta); + } else { + meta = mapper.readValue(file, MetaImport.class); + conformedDims.addAll(((MetaImport) meta).getDimensionsWithoutRefTo()); + } + + } catch(JsonMappingException e) { + String message = "Could not deserialize metadata from file: " + file.getName() + "\n"; + message += e.getMessage(); + errorMessages.add(message); + } catch(Exception e) { + errorMessages.add(ExceptionUtils.getFullStackTrace(e)); + } + if(meta != null) { + log.info("Read metadata from file: " + file.getName()); + meta.setFile(file); + metaImports.add(meta); + } + } + } + + // gather all conformed dimensions + List confDims = new ArrayList<>(); + confDims.addAll(conformedDims); + for (MetaImportConformedDimensions conf : conformedDimension) { + confDims.addAll(conf.conformedDimensions); + } + + // resolve conformed references ('ref_to' attributes) + for (MetaJson metaJson : metaImports) { + if (conformedDimension.size() > 0 && metaJson instanceof MetaImport) { + ((MetaImport) metaJson).setConformedDimensions(confDims); + } + try { + metaJson.init(); + metaJson.setNamespaceToMetaObjects(); + } catch (Exception e) { + errorMessages.add(ExceptionUtils.getFullStackTrace(e)); + } + } + + if(shouldReadYMLDoc) { + addDescriptionsFromYMLFiles(); + } + + } + + public List readStoredReports() { + List result = new ArrayList<>(); + try { + String dir = PathAndFileUtils.getStoredReportDir("hisinone"); + File[] files = new File(dir).listFiles(); + if(files == null) { + return result; + } + for (File file : files) { + try { + ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); + StoredReport report = mapper.readValue(file, StoredReport.class); + UpsertStringBuilder builder = new UpsertStringBuilder() + .forTable("metadata", "rw_report_definitions") + .withIntCol("id", Integer.valueOf(report.id)) + .withStringCol("name", report.name) + .withStringCol("definition", report.definition) + .withIntCol("show_total_column", Integer.valueOf(report.showTotalColumn)); + result.add(builder.build(true)); + } catch (JsonMappingException e) { + String message = "Could not deserialize stored report from file: " + file.getName() + "\n"; + message += e.getMessage(); + errorMessages.add(message); + } + } + // After inserting the stored reports with a fixed id we need to re-sync the + // id column of the rw_report_definitions table + if(result.size() != 0) { + result.add("SELECT setval(pg_get_serial_sequence('metadata.rw_report_definitions', 'id')," + + "(SELECT max(id) FROM metadata.rw_report_definitions ));"); + } + } catch(Exception e) { + errorMessages.add("Unable to read stored report:\n"); + errorMessages.add(ExceptionUtils.getFullStackTrace(e)); + } + return result; + } + + public void addDescriptionsFromYMLFiles() { + String dir = ymlDir; + if(ymlDir == null || ymlDir.isBlank()) { + dir = PathAndFileUtils.getDbtTransformDirectory("hisinone"); + } + HashMap map = getMarkdownDefinitions(dir); + addYMLDescriptionsToMetaObjects(dir, map); + } + + public void addYMLDescriptionsToMetaObjects(String ymlDir, HashMap mdDefs){ + log.info("Adding descriptions from yml files"); + HashMap descriptions = createDescriptions(new File(ymlDir), mdDefs); + List objs = getAllMetaObjectsWithConformed(); + for (MetaObject metaObj : objs ) { + String docIdentifier = metaObj.getDocIdentifier(); + if(docIdentifier == null || docIdentifier.isBlank()) { + continue; + } + // only use yml doc if json description does not exist + if(metaObj.getDescription() == null || metaObj.getDescription().isBlank()) { + String desc = descriptions.get(docIdentifier); + if(desc == null) { + log.warn("Missing yml description for: " + docIdentifier); + } else { + metaObj.setDescription(desc); + if(desc.isBlank()) { + log.warn("Empty yml description for MetaObject: " + docIdentifier); + } + } + } + } + } + + public MetaYml createYMLFileForMetaJson(MetaJson metaJson) { + + MetaYml newYml = new MetaYml(); + List newYmlModels = new ArrayList<>(); + newYml.setVersion(1); + newYml.setModels(newYmlModels); + + if(metaJson instanceof MetaImport) { + MetaImport metaimport = (MetaImport) metaJson; + for (MetaFact fact : metaimport.facts) { + MetaYmlModel factModel = new MetaYmlModel(fact.getFacttable(), " "); + newYmlModels.add(factModel); + List factCols = new ArrayList<>(); + for(MetaDimension dim : fact.getDimensions()) { + if(dim.getRefTo() == null) { + factCols.add(new MetaYmlModelColumns(dim.getFactColumn(), " ", "not_null")); + MetaYmlModel dimModel = new MetaYmlModel(dim.getDimension(), " "); + newYmlModels.add(dimModel); + List dimCols = new ArrayList<>(); + for(MetaDimensionAttribute attr : dim.getAttributes()) { + dimCols.add(new MetaYmlModelColumns(attr.getDimColumn(), " ", "not_null")); + } + dimModel.setColumns(dimCols); + } + } + factModel.setColumns(factCols); + } + } else { + MetaImportConformedDimensions metaimport = (MetaImportConformedDimensions) metaJson; + for(MetaDimension dim : metaimport.conformedDimensions) { + MetaYmlModel dimModel = new MetaYmlModel(dim.getDimension(), " "); + newYmlModels.add(dimModel); + List dimCols = new ArrayList<>(); + for(MetaDimensionAttribute attr : dim.getAttributes()) { + dimCols.add(new MetaYmlModelColumns(attr.getDimColumn(), " ", "not_null")); + } + dimModel.setColumns(dimCols); + } + } + return newYml; + } + + private HashMap createDescriptions(File startDir, HashMap mdDefs){ + HashMap result = new HashMap<>(); + for (MetaYml yml : getDescriptionYMLs(startDir)) { + for (MetaYmlModel model : yml.getModels()) { + String modelName = model.getName(); + String modelDesc = model.getDescription(); + result.put(modelName, getDescription(modelDesc, mdDefs)); + for (MetaYmlModelColumns column : model.getColumns()) { + String colName = column.getName(); + String colDesc = column.getDescription(); + result.put(modelName + "." + colName, getDescription(colDesc, mdDefs)); + } + } + } + return result; + } + + private static String getDescription(String desc, HashMap mdDefs) { + if(desc == null) { + return ""; + } + if(desc.startsWith("{{")) { + String[] parts = desc.split("\""); + String docRef = parts[1]; + return mdDefs.get(docRef); + } + return desc; + } + + private List getDescriptionYMLs(File startDir){ + List ymls = new ArrayList<>(); + List files = getFiles(startDir, "", ".yml"); + ObjectMapper mapperYml = new ObjectMapper(new YAMLFactory()); + for (String f : files) { + File file = new File(startDir + File.separator + f); + MetaYml doc = null; + try { + doc = mapperYml.readValue(file, MetaYml.class); + } catch (Exception e) { + String message = "Could not read documentation from file: " + file.getName() + "\n"; + errorMessages.add(message); + errorMessages.add(ExceptionUtils.getFullStackTrace(e)); + } + if(doc != null) { + log.info("Read documentation from file: " + file.getName()); + ymls.add(doc); + } + } + return ymls; + } + + + /** + * Gathers all metadata json files. + * + * @param path A path to a metadata json file or a directory containing metadata json files. + * @return A list of files matching the metadata json suffix. + */ + private static List readMetaImportFiles(String path) { + File metaimportPath = new File(PathAndFileUtils.getDbtJsonPath(path)); + List metaimportFiles = new ArrayList<>(); + if (metaimportPath.isDirectory()) { + metaimportPath.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (name.endsWith(METAIMPORT_FILE_SUFFIX)) { + File file = new File(dir.getAbsolutePath() + File.separator + name); + metaimportFiles.add(file); + return true; + } + return false; + } + }); + } else { + metaimportFiles.add(metaimportPath); + } + return metaimportFiles; + } + + private static List getFiles(File startDir, String subDir, String extension) { + List filtered = new ArrayList(); + for (File file : startDir.listFiles()) { + String name = file.getName(); + if(file.isDirectory()) { + filtered.addAll(getFiles(file, subDir + File.separator + name, extension)); + } + String filename = name.strip().toLowerCase(); + if(filename.endsWith(extension)) { + filtered.add(subDir + File.separator + name); + } + } + return filtered; + } + + /* + + /** + * Generates the sql upsert strings for all unique, deserialized MetaObjects. + * + * @param hasOnConflictConstruct If set to true generates upsert strings with the postgres-specific "ON CONFLICT" clause. + * @return All the generated upserts from the metadata files. + */ + public List getAllUpsertStrings(boolean hasOnConflictConstruct) { + + List upsertStmts = new ArrayList<>(); + List ids = new ArrayList<>(); + + for (MetaJson meta : metaImports) { + for(MetaObject obj: meta.getMetaObjects()) { + Identifier id = obj.getId(); + if(id == null) { + String message = String.format("Missing ID for Element '%s' in file: %s.", obj.getCaption(), meta.getFile().getAbsolutePath()); + errorMessages.add(message); + continue; + } + if(ids.contains(id)) { + String message = String.format("Duplicate ID '%s'. Ignoring Element '%s'.", obj.getCaption(), obj.getId().composedId); + errorMessages.add(message); + continue; + } + ids.add(obj.getId()); + String stmt = obj.getUpsertBuilder().build(hasOnConflictConstruct); + upsertStmts.add(stmt); + } + } + return upsertStmts; + } + + public void updateMetadataForH2Database(DataSource dataSource) throws Exception { + String metaFilesDir = String.join(File.separator, new String[] {"test", "resources", "db", "fixtures", "reportwizard", "metadata"}); + deserializeMetadataFromJsonFiles(metaFilesDir); + JdbcTemplate jt = new JdbcTemplate(dataSource); + String upserts = String.join("\n", getAllUpsertStrings(false).toString()); + jt.execute(upserts); + } + + /** + * Updates tables in the metadata schema. + * + * @param metaPath Location of the metadata file or directory. + * @param dataSource The datasource on which the sql is executed. + * @throws Exception + */ + public void updateMetadataSchema(String project, DataSource dataSource) throws Exception { + String metaFilesDir = PathAndFileUtils.getReportGeneratorDir(project); + deserializeMetadataFromJsonFiles(metaFilesDir); + + try (Connection con = dataSource.getConnection()) { + try (Statement st = con.createStatement()) { + log.info("Update Metadata for BIAnalysis."); + st.execute(TRUNCATE_METADATA_SQL); + List upserts = getAllUpsertStrings(true); + upserts.addAll(readStoredReports()); + for (String sql : upserts) { + log.info(sql); + try (Statement stUpsert = con.createStatement()) { + stUpsert.execute(sql); + } catch (Exception e) { + throw e; + } + } + } + + + // execute sql in "attributes_sql" in metadata files to build the attributes + // dynamically + for (MetaJson i : this.metaImports) { + for (MetaObject obj : i.getMetaObjects()) { + if (!(obj instanceof MetaDimension)) continue; + MetaDimension dim = (MetaDimension) obj; + if (dim.getAttributesSql() == null) continue; + + String sqlDone = ""; + String sql = "select param_val from unload_params where param_id = '" + dim.getAttributesSql() + "';"; + try (Statement stAttr = con.createStatement(); ResultSet rs = stAttr.executeQuery(sql)) { + if(rs.next()) { + sqlDone = rs.getString("param_val"); + } + } + + try (Statement stAttr = con.createStatement(); ResultSet rs = stAttr.executeQuery(sqlDone)) { + int numAttributes = 0; + while (rs.next()) { + + MetaDimensionAttribute attribute = new MetaDimensionAttribute(); + attribute.setDimension(dim); + attribute.setCaption(rs.getString("caption")); + attribute.setDimColumn(rs.getString("dim_column")); + + // create a new 'on the fly' identifier for the new metadata + // attribute + Identifier id = Identifier.getNewIdentifierValue(i.getIds(), dim.getNamespace()); + Integer val = Integer.valueOf(id.value.intValue() + numAttributes); + attribute.setId(new Identifier(dim.getNamespace() + ":" +val)); + numAttributes++; + + String stmt = attribute.getUpsertBuilder().build(true); + try (Statement stUpsert = con.createStatement()) { + stUpsert.execute(stmt); + } + } + } + } + } + + } + } + + public static String writeYmlToString(MetaYml yml) { + YAMLFactory yf = new YAMLFactory() + .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES) + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER); + ObjectMapper mapper = new ObjectMapper(yf); + DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter(" ", DefaultIndenter.SYS_LF); + DefaultPrettyPrinter printer = new DefaultPrettyPrinter(); + printer.indentObjectsWith(indenter); + printer.indentArraysWith(indenter); + try { + //mapper.writer(printer).writeValue(file, yml); + return mapper.writer(printer).writeValueAsString(yml); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public String getPrintableErrorMessages() { + String output = ""; + if(!errorMessages.isEmpty()) { + output += "The following errors occured:\n"; + for (String message : errorMessages) { + output += message + "\n"; + } + } + return output; + } + + public HashMap getMarkdownDefinitions(String ymlDir) { + List files = getFiles(new File(ymlDir), "", ".md"); + HashMap map = new HashMap<>(); + for (String file : files) { + try { + try (BufferedReader br = new BufferedReader(new FileReader(ymlDir + File.separator + file))) { + String line; + String key = null; + boolean readHeading = false; + while ((line = br.readLine()) != null) { + if (line.startsWith("{% docs ")) { + key = line.split(" ")[2]; + map.put(key, ""); + } else if(key != null && line.startsWith("# ")) { + readHeading = true; + } else { + if(readHeading && !line.isBlank()) { + map.put(key, line); + key = null; + readHeading = false; + } + } + } + } + } catch (Exception e) { + String message = "ERROR getting markdown definitions from file: " + file + "\n"; + errorMessages.add(message); + errorMessages.add(ExceptionUtils.getFullStackTrace(e)); + } + } + return map; + } + + public Optional getMetaImport(String fileName) { + return metaImports.stream() + .filter(json -> (json instanceof MetaImport) && json.file.getName().equals(fileName)) + .map(json -> (MetaImport) json) + .findFirst(); + } + + public Optional getMetaJson(String fileName) { + return metaImports.stream() + .filter(json -> json.file.getName().equals(fileName)) + .findFirst(); + } + + public List getMetaImports() { + return metaImports.stream() + .filter(json -> (json instanceof MetaImport)) + .map(json -> (MetaImport) json) + .collect(Collectors.toList()); + } + + public List getMetaJsons() { + return metaImports.stream().collect(Collectors.toList()); + } + + public List getAllMetaObjects(){ + return metaImports.stream() + .filter(json -> (json instanceof MetaImport)) + .map(meta -> ((MetaImport) meta).getMetaObjects()) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List getAllMetaObjectsWithConformed(){ + return metaImports.stream() + .map(meta -> meta.getMetaObjects()) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public List getAllFactTables(){ + return metaImports.stream() + .filter(json -> (json instanceof MetaImport)) + .map(meta -> ((MetaImport) meta).facts) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + + public void setShouldReadYMLDoc(boolean shouldReadYMLDoc) { + this.shouldReadYMLDoc = shouldReadYMLDoc; + } + + +} diff --git a/src/de/superx/bianalysis/metadata/UpsertStringBuilder.java b/src/de/superx/bianalysis/metadata/UpsertStringBuilder.java new file mode 100644 index 0000000..3ba8bfc --- /dev/null +++ b/src/de/superx/bianalysis/metadata/UpsertStringBuilder.java @@ -0,0 +1,99 @@ +package de.superx.bianalysis.metadata; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; + +public class UpsertStringBuilder { + + private StringJoiner values; + private StringJoiner columns; + private StringJoiner onConflict; + private String schema; + private String tablename; + + private List builders = new ArrayList<>(); + + public UpsertStringBuilder() { + values = new StringJoiner(", "); + columns = new StringJoiner(", "); + onConflict = new StringJoiner(", "); + } + + public void addUpsertStringBuilder(UpsertStringBuilder builder) { + this.builders.add(builder); + } + + public UpsertStringBuilder forTable(String schema, String tablename) { + this.tablename = tablename; + this.schema = schema; + return this; + } + + public UpsertStringBuilder withStringCol(String colName, String value) { + appendToSelection(colName); + if(value == null) { + values.add("null"); + } else { + values.add("'"+value+"'"); + } + return this; + } + + public UpsertStringBuilder withStringCol(String colName, Object value) { + if(value != null) { + return this.withStringCol(colName, value.toString()); + } + return this.withStringCol(colName, "unknown"); + } + + public UpsertStringBuilder withStringCol(String colName, String value, String defaultVal) { + if(value != null) { + return this.withStringCol(colName, value); + } + return this.withStringCol(colName, defaultVal); + } + + public UpsertStringBuilder withStringCol(String colName, Object value, Object defaultVal) { + if(value != null) { + return this.withStringCol(colName, value); + } + return this.withStringCol(colName, defaultVal); + } + + public UpsertStringBuilder withIntCol(String colName, Integer value) { + appendToSelection(colName); + values.add(String.valueOf(value)); + return this; + } + + public UpsertStringBuilder withIdCol(String colName, Identifier id) { + if(id != null) { + return this.withStringCol(colName, id.composedId); + } + return this.withStringCol(colName, null); + } + + private void appendToSelection(String colName) { + onConflict.add(String.format("%s = EXCLUDED.%s", colName, colName)); + columns.add(colName); + } + + public String build(boolean hasOnConflictConstruct) { + String result = "INSERT INTO %s.%s(%s) VALUES(%s)"; + result = String.format(result, this.schema, this.tablename, this.columns, this.values); + if(hasOnConflictConstruct) { + // TODO: log message for on id conflict + //result += " ON CONFLICT(id) DO UPDATE SET " + this.onConflict; + result += " ON CONFLICT(id) DO NOTHING"; + } + result += ";\n"; + if(this.builders.size() > 0) { + for (UpsertStringBuilder upsertStringBuilder : builders) { + result += upsertStringBuilder.build(hasOnConflictConstruct); + } + } + return result; + } + +} diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaDimension.java b/src/de/superx/bianalysis/metadata/models/json/MetaDimension.java new file mode 100644 index 0000000..e4d0bc1 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaDimension.java @@ -0,0 +1,269 @@ +package de.superx.bianalysis.metadata.models.json; + +import java.util.ArrayList; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.metadata.UpsertStringBuilder; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonInclude(Include.NON_DEFAULT) +@JsonPropertyOrder({ "id", "default_release", "ref_to", "caption", "dimension", "fact_column", "alias", "bridge", "attributes"}) +public class MetaDimension extends MetaObject { + + @JsonProperty("ref_to") + private String refTo; + private String dimension; + @JsonProperty("fact_column") + private String factColumn; + private String alias; + private String view; + @JsonProperty("id_column") + private String idColumn; + private List attributes; + + @JsonProperty("is_hierarchy") + private boolean isHierarchy; + @JsonProperty("is_historical") + private boolean isHistorical; + + @JsonProperty("attributes_sql") + private String attributesSql; + + @JsonIgnore + private MetaFact fact; + + // true if dimension is from the conformed_dimensions_metaimport.json + @JsonIgnore + private boolean isConformed = false; + + // isConformed must be false + // the correpsonding dimension from the conformed_dimensions_metaimport.json + // referenced by 'ref_to' + @JsonIgnore + private MetaDimension conformedDimension; + + public MetaDimension() { + super("dimension"); + } + + public void setConformedDimension(MetaDimension dimension) { + this.conformedDimension = dimension; + } + + @Override + public UpsertStringBuilder getUpsertBuilder() { + Identifier factId = (isConformed) ? null : this.fact.id; + UpsertStringBuilder builder = new UpsertStringBuilder(); + if(conformedDimension == null) { + builder = super.getUpsert() + .withIdCol("facttable_id", factId) + .withStringCol("joincolumn", this.factColumn) + .withStringCol("alias", this.alias) + .withStringCol("is_hierarchy", String.valueOf(this.isHierarchy)) + .withStringCol("is_historical", String.valueOf(this.isHistorical)) + //.withStringCol("attributes_sql", this.attributesSql) + .withStringCol("tablename", this.dimension) + .withStringCol("id_column", this.idColumn); + } else { + builder = new UpsertStringBuilder() + .forTable("metadata", this.sourceTable) + .withStringCol("namespace", this.namespace) + .withIdCol("id", this.id) + .withIntCol("default_release", Integer.valueOf(1)); + builder = builder.withIdCol("facttable_id", factId); + + if(this.idColumn != null && !this.idColumn.isBlank()) { + builder = builder.withStringCol("id_column", idColumn); + } else { + builder = builder.withStringCol("id_column", this.conformedDimension.getIdColumn()); + } + + if(this.caption != null && !this.caption.isBlank()) { + builder = builder.withStringCol("caption", caption); + } else { + builder = builder.withStringCol("caption", this.conformedDimension.getCaption()); + } + + if(this.factColumn != null && !this.factColumn.isBlank()) { + builder = builder.withStringCol("joincolumn", this.factColumn); + } else { + builder = builder.withStringCol("joincolumn", this.conformedDimension.getFactColumn()); + } + + if(this.alias != null && !this.factColumn.isBlank()) { + builder = builder.withStringCol("alias", this.alias); + } else { + builder = builder.withStringCol("alias", this.conformedDimension.getAlias()); + } + + if(this.isHierarchy) { + builder = builder.withStringCol("is_hierarchy", String.valueOf(isHierarchy)); + } else { + builder = builder.withStringCol("is_hierarchy", String.valueOf(conformedDimension.isHierarchy)); + } + + if(this.isHistorical) { + builder = builder.withStringCol("is_historical", String.valueOf(isHistorical)); + } else { + builder = builder.withStringCol("is_historical", String.valueOf(conformedDimension.isHistorical)); + } + + if(this.conformedDimension.getDimension() != null && !this.conformedDimension.getDimension().isBlank()) { + + if(view != null && !view.isBlank()) { + builder = builder.withStringCol("tablename", this.view); + } else { + builder = builder.withStringCol("tablename", this.conformedDimension.getDimension()); + } + + } else { + builder = builder.withStringCol("tablename", this.dimension); + } + builder = builder.withIdCol("conformed", this.conformedDimension.id); + } + + if(this.description != null && !this.description.isBlank()) { + builder = builder.withStringCol("description", this.description); + } else { + if(conformedDimension != null) { + builder = builder.withStringCol("description", conformedDimension.getDescription()); + } else { + builder = builder.withStringCol("description", ""); + } + } + return builder; + } + + public String getRefTo() { + return refTo; + } + + public void setRefTo(String refTo) { + this.refTo = refTo; + } + + public String getDimension() { + return dimension; + } + + public void setDimension(String dimension) { + this.dimension = dimension; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public List getAttributes() { + if(this.attributes == null) { + return new ArrayList<>(); + } + return attributes; + } + + public void setAttributes(List attributes) { + for (MetaDimensionAttribute metaDimensionAttribute : attributes) { + metaDimensionAttribute.setDimension(this); + } + this.attributes = attributes; + } + + public MetaFact getFact() { + return fact; + } + + public void setFact(MetaFact fact) { + this.fact = fact; + } + + @JsonIgnore + public boolean isConformed() { + return isConformed; + } + + @JsonIgnore + public void setConformed(boolean isConformed) { + this.isConformed = isConformed; + } + + public MetaDimension getConformedDimension() { + return conformedDimension; + } + + public String getFactColumn() { + return factColumn; + } + + public void setFactColumn(String factColumn) { + this.factColumn = factColumn; + } + + public void addAttribute(MetaDimensionAttribute metaDimensionAttribute) { + if(this.attributes == null) { + this.attributes = new ArrayList<>(); + } + this.attributes.add(metaDimensionAttribute); + } + + @Override + @JsonIgnore + public String getDocIdentifier() { + if(this.conformedDimension != null) { + return conformedDimension.getDocIdentifier(); + } + return this.dimension; + } + + public boolean isHierarchy() { + return isHierarchy; + } + + public void setHierarchy(boolean isHierarchy) { + this.isHierarchy = isHierarchy; + } + + public boolean isHistorical() { + return isHistorical; + } + + public void setHistorical(boolean isHistorical) { + this.isHistorical = isHistorical; + } + + public String getView() { + return view; + } + + public void setView(String view) { + this.view = view; + } + + public String getIdColumn() { + return idColumn; + } + + public void setIdColumn(String idColumn) { + this.idColumn = idColumn; + } + + public String getAttributesSql() { + return attributesSql; + } + + public void setAttributesSql(String attributesSql) { + this.attributesSql = attributesSql; + } + + +} diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaDimensionAttribute.java b/src/de/superx/bianalysis/metadata/models/json/MetaDimensionAttribute.java new file mode 100644 index 0000000..7f70479 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaDimensionAttribute.java @@ -0,0 +1,179 @@ +package de.superx.bianalysis.metadata.models.json; + + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +import de.superx.bianalysis.metadata.UpsertStringBuilder; + +@JsonInclude(Include.NON_DEFAULT) +@JsonPropertyOrder({ "id", "default_release", "ref", "caption", "dim_column"}) +public class MetaDimensionAttribute extends MetaObject { + + @JsonProperty("dim_column") + private String dimColumn; + + @JsonProperty("sort_order_column") + private String sortOrderColumn; + + @JsonProperty("hierarchical_filter") + private boolean hierarchicalFilter; + + @JsonIgnore + private MetaDimension dimension; + + @JsonIgnore + private MetaDimensionAttribute confDimAttrRef; + + @JsonProperty("ref_to") + private String refTo; + + @JsonProperty("filter_selection") + private String filterSelection; + + public MetaDimensionAttribute() { + super("dimension_attribute"); + } + + public MetaDimensionAttribute(String attrColumn) { + super("dimension_attribute"); + this.refTo = attrColumn; + //this.refTo = dimensionTable + "." + attrColumn; + } + + public void setConformedDimensionAttribute(MetaDimensionAttribute attribute) { + this.confDimAttrRef = attribute; + } + + @Override + public UpsertStringBuilder getUpsertBuilder() { + UpsertStringBuilder builder = new UpsertStringBuilder(); + if(confDimAttrRef == null) { + builder = super.getUpsert() + .withIdCol("dimension_id", this.dimension.id) + .withStringCol("columnname", this.dimColumn) + .withStringCol("sort_order_column", this.sortOrderColumn) + .withStringCol("filter_selection", this.filterSelection); + } else { + builder = new UpsertStringBuilder() + .forTable("metadata", this.sourceTable) + .withStringCol("namespace", this.namespace) + .withIdCol("id", this.id) + .withIntCol("default_release", Integer.valueOf(1)); + + if(getCaption() != null && !getCaption().isBlank()) { + builder = builder.withStringCol("caption", caption); + } else { + builder = builder.withStringCol("caption", confDimAttrRef.getCaption()); + } + + if(getDimColumn() != null && !getDimColumn().isBlank()) { + builder = builder.withStringCol("columnname", this.dimColumn); + } else { + builder = builder.withStringCol("columnname", confDimAttrRef.getDimColumn()); + } + + if(getFilterSelection() != null && !getFilterSelection().isBlank()) { + builder = builder.withStringCol("filter_selection", this.filterSelection); + } else { + builder = builder.withStringCol("filter_selection", confDimAttrRef.getFilterSelection()); + } + + if(getSortOrderColumn() != null && !getSortOrderColumn().isBlank()) { + builder = builder.withStringCol("sort_order_column", this.sortOrderColumn); + } else { + builder = builder.withStringCol("sort_order_column", confDimAttrRef.getSortOrderColumn()); + } + + builder = builder.withIdCol("dimension_id", this.dimension.id); + builder = builder.withIdCol("conformed", this.confDimAttrRef.id); + } + + if(confDimAttrRef != null && confDimAttrRef.hierarchicalFilter) { + builder = builder.withStringCol("hierarchical_filter", String.valueOf(confDimAttrRef.hierarchicalFilter)); + } else { + builder = builder.withStringCol("hierarchical_filter", String.valueOf(hierarchicalFilter)); + } + + if(this.description != null && !this.description.isBlank()) { + builder = builder.withStringCol("description", this.description); + } else { + if(confDimAttrRef != null) { + builder = builder.withStringCol("description", confDimAttrRef.getDescription()); + } else { + builder = builder.withStringCol("description", ""); + } + } + + return builder; + } + + public String getDimColumn() { + return dimColumn; + } + + public void setDimColumn(String dimColumn) { + this.dimColumn = dimColumn; + } + + public String getSortOrderColumn() { + return sortOrderColumn; + } + + public void setSortOrderColumn(String sortOrderColumn) { + this.sortOrderColumn = sortOrderColumn; + } + + public String getFilterSelection() { + return filterSelection; + } + + public void setFilterSelection(String filterSelection) { + this.filterSelection = filterSelection; + } + + public MetaDimension getDimension() { + return dimension; + } + + public void setDimension(MetaDimension dimension) { + this.dimension = dimension; + } + + public MetaDimensionAttribute getConfDimAttrRef() { + return confDimAttrRef; + } + + public void setConfDimAttrRef(MetaDimensionAttribute confDimAttrRef) { + this.confDimAttrRef = confDimAttrRef; + } + + public String getRefTo() { + return refTo; + } + + public void setRefTo(String refTo) { + this.refTo = refTo; + } + + @JsonIgnore + @Override + public String getDocIdentifier() { + if(refTo != null) { + return this.confDimAttrRef.getDocIdentifier(); + } + return this.dimension.getDocIdentifier()+"."+this.dimColumn; + } + + public boolean isHierarchicalFilter() { + return hierarchicalFilter; + } + + public void setHierarchicalFilter(boolean hierarchicalFilter) { + this.hierarchicalFilter = hierarchicalFilter; + } + +} diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaFact.java b/src/de/superx/bianalysis/metadata/models/json/MetaFact.java new file mode 100644 index 0000000..bb74767 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaFact.java @@ -0,0 +1,77 @@ +package de.superx.bianalysis.metadata.models.json; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.metadata.UpsertStringBuilder; + +@JsonPropertyOrder({ "id", "default_release", "caption", "sachgebiettid", "facttable", "conformed_dimensions", "dimensions", "measures" }) +public class MetaFact extends MetaObject { + + private Integer sachgebiettid; + private String facttable; + private List dimensions; + private List measures; + + public MetaFact() { + super("facttable"); + } + + @Override + public UpsertStringBuilder getUpsertBuilder() { + UpsertStringBuilder builder = super.getUpsert() + .withIntCol("sachgebiettid", this.sachgebiettid) + .withStringCol("tablename", this.facttable) + .withStringCol("description", super.getDescription()); + + return builder; + } + + public Integer getSachgebiettid() { + return sachgebiettid; + } + + public void setSachgebiettid(Integer sachgebiettid) { + this.sachgebiettid = sachgebiettid; + } + + public String getFacttable() { + return facttable; + } + + public void setFacttable(String facttable) { + this.facttable = facttable; + } + + public List getDimensions() { + return dimensions; + } + + public void setDimensions(List dimensions) { + for (MetaDimension metaDimension : dimensions) { + metaDimension.setFact(this); + } + this.dimensions = dimensions; + } + + public List getMeasures() { + return measures; + } + + public void setMeasures(List measures) { + for (MetaMeasure metaMeasure : measures) { + metaMeasure.setFact(this); + } + this.measures = measures; + } + + @JsonIgnore + @Override + public String getDocIdentifier() { + return this.facttable; + } + +} diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaMeasure.java b/src/de/superx/bianalysis/metadata/models/json/MetaMeasure.java new file mode 100644 index 0000000..41d51e0 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaMeasure.java @@ -0,0 +1,82 @@ +package de.superx.bianalysis.metadata.models.json; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import de.superx.bianalysis.metadata.UpsertStringBuilder; +import de.superx.rest.model.ColumnType; + +@JsonPropertyOrder({ "id", "default_release"} ) +public class MetaMeasure extends MetaObject { + + private String factcolumn; + private String aggregation; + private ColumnType type; + private MetaMeasureFilter filter; + + @JsonIgnore + private MetaFact fact; + + public MetaMeasure() { + super("measure"); + } + + @Override + public UpsertStringBuilder getUpsertBuilder() { + return super.getUpsert() + .withIdCol("facttable_id", (this.fact != null) ? this.fact.id : null) + .withIdCol("measure_filter_id", (this.filter != null) ? this.filter.id : null) + .withStringCol("columnname", this.factcolumn) + .withStringCol("aggregation_type", this.aggregation) + .withStringCol("description", this.description) + .withStringCol("measure_type", this.type, ColumnType.IntegerColumn); + } + + public String getFactcolumn() { + return factcolumn; + } + + public void setFactcolumn(String factcolumn) { + this.factcolumn = factcolumn; + } + + public String getAggregation() { + return aggregation; + } + + public void setAggregation(String aggregation) { + this.aggregation = aggregation; + } + + public ColumnType getType() { + return type; + } + + public void setType(ColumnType type) { + this.type = type; + } + + public MetaMeasureFilter getFilter() { + return filter; + } + + public void setFilter(MetaMeasureFilter filter) { + this.filter = filter; + } + + public MetaFact getFact() { + return fact; + } + + public void setFact(MetaFact fact) { + this.fact = fact; + } + + @JsonIgnore + @Override + public String getDocIdentifier() { + return this.factcolumn; + } + +} + diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaMeasureFilter.java b/src/de/superx/bianalysis/metadata/models/json/MetaMeasureFilter.java new file mode 100644 index 0000000..6bb03ca --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaMeasureFilter.java @@ -0,0 +1,106 @@ +package de.superx.bianalysis.metadata.models.json; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import de.superx.bianalysis.metadata.UpsertStringBuilder; + +@JsonPropertyOrder({ "id", "default_release"} ) +public class MetaMeasureFilter extends MetaObject { + + private String dimensionRef; + + private String factColumnRef; + + @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + private List included; + + @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + private List excluded; + + @JsonIgnore + private MetaDimensionAttribute attribute; + + public MetaMeasureFilter() { + super("measure_filter"); + } + + @Override + public UpsertStringBuilder getUpsertBuilder() { + + UpsertStringBuilder builder = super.getUpsert(); + builder.withStringCol("included_values", concatValues(included)); + builder.withStringCol("excluded_values", concatValues(excluded)); + + if(this.dimensionRef != null) { + builder.withIdCol("dimension_attribute_id", this.attribute.id); + } else if(this.factColumnRef != null){ + builder.withStringCol("fact_column_filter", this.factColumnRef); + } + + return builder; + } + + private static String concatValues(List values) { + if (values == null || values.isEmpty()) { + return null; + } + String result = ""; + int size = values.size(); + for (int i = 0; i < size - 1; i++) { + result += "''" + values.get(i) + "'', "; + } + result += "''" + values.get(size - 1) + "''"; + return result; + } + + public String getDimensionRef() { + return dimensionRef; + } + + public void setDimensionRef(String dimensionRef) { + this.dimensionRef = dimensionRef; + } + + public List getIncluded() { + return included; + } + + public void setIncluded(List included) { + this.included = included; + } + + public List getExcluded() { + return excluded; + } + + public void setExcluded(List excluded) { + this.excluded = excluded; + } + + public MetaDimensionAttribute getAttribute() { + return attribute; + } + + public void setAttribute(MetaDimensionAttribute attribute) { + this.attribute = attribute; + } + + @JsonIgnore + @Override + public String getDocIdentifier() { + return ""; + } + + public String getFactColumnRef() { + return factColumnRef; + } + + public void setFactColumnRef(String factColumnRef) { + this.factColumnRef = factColumnRef; + } + +} diff --git a/src/de/superx/bianalysis/metadata/models/json/MetaObject.java b/src/de/superx/bianalysis/metadata/models/json/MetaObject.java new file mode 100644 index 0000000..c11fe54 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/json/MetaObject.java @@ -0,0 +1,112 @@ +package de.superx.bianalysis.metadata.models.json; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.metadata.UpsertStringBuilder; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(Include.NON_NULL) +public abstract class MetaObject { + + protected Identifier id; + protected String caption; + protected String description; + + @JsonProperty("default_release") + protected String defaultRelease; + + @JsonIgnore + protected String sourceTable; + + @JsonIgnore + protected String namespace; + + protected MetaObject(String sourceTable) { + this.sourceTable = sourceTable; + } + + /** + * Returns the documentation identifier for a specific meta object. + * The Identifier is used in a *.md file and is referenced in the yml + * file like the following: '{{ doc("") }}'. + */ + @JsonIgnore + public abstract String getDocIdentifier(); + + @JsonIgnore + public abstract UpsertStringBuilder getUpsertBuilder(); + + @JsonIgnore + protected UpsertStringBuilder getUpsert() { + return new UpsertStringBuilder() + .forTable("metadata", this.sourceTable) + .withStringCol("namespace", this.namespace) + .withIdCol("id", this.id) + .withStringCol("default_release", this.defaultRelease) + .withStringCol("caption", this.caption); + } + + @Override + public boolean equals(Object obj) { + if(!(obj instanceof MetaObject)) { + return false; + } else if(((MetaObject)obj).id == null) { + return false; + } + return this.id.equals(((MetaObject)obj).id); + } + + public Identifier getId() { + return id; + } + + public void setId(Identifier id) { + this.id = id; + } + + public String getCaption() { + return caption; + } + + public void setCaption(String caption) { + this.caption = caption; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDefaultRelease() { + return defaultRelease; + } + + public void setDefaultRelease(String defaultRelease) { + this.defaultRelease = defaultRelease; + } + +} diff --git a/src/de/superx/bianalysis/metadata/models/yml/MetaYml.java b/src/de/superx/bianalysis/metadata/models/yml/MetaYml.java new file mode 100644 index 0000000..802adb6 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/yml/MetaYml.java @@ -0,0 +1,32 @@ +package de.superx.bianalysis.metadata.models.yml; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MetaYml { + + private int version; + private List models; + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public List getModels() { + if(this.models == null) { + return new ArrayList(); + } + return models; + } + + public void setModels(List models) { + this.models = models; + } +} diff --git a/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModel.java b/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModel.java new file mode 100644 index 0000000..9cdccf7 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModel.java @@ -0,0 +1,49 @@ +package de.superx.bianalysis.metadata.models.yml; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MetaYmlModel { + + private String name; + private String description; + private List columns; + + public MetaYmlModel() { } + + public MetaYmlModel(String name, String description) { + super(); + this.name = name; + this.description = description; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public List getColumns() { + if(this.columns == null) { + return new ArrayList(); + } + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } +} diff --git a/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModelColumns.java b/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModelColumns.java new file mode 100644 index 0000000..7a12c53 --- /dev/null +++ b/src/de/superx/bianalysis/metadata/models/yml/MetaYmlModelColumns.java @@ -0,0 +1,52 @@ +package de.superx.bianalysis.metadata.models.yml; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MetaYmlModelColumns { + + private String name; + private String description; + private List tests; + + public MetaYmlModelColumns() {} + + public MetaYmlModelColumns(String name, String description, String test) { + super(); + this.name = name; + this.description = description; + this.tests = new ArrayList(); + this.tests.add(test); + } + + public MetaYmlModelColumns(String name, String description) { + super(); + this.name = name; + this.description = description; + } + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public String getDescription() { + return description; + } + public void setDescription(String description) { + this.description = description; + } + + public List getTests() { + return tests; + } + + public void setTests(List tests) { + this.tests = tests; + } + +} diff --git a/src/de/superx/bianalysis/models/Dimension.java b/src/de/superx/bianalysis/models/Dimension.java new file mode 100644 index 0000000..76802b5 --- /dev/null +++ b/src/de/superx/bianalysis/models/Dimension.java @@ -0,0 +1,94 @@ +package de.superx.bianalysis.models; + +import java.util.List; + +import org.springframework.data.annotation.Transient; +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.DimensionDto; + +public class Dimension { + + private DimensionDto dimensionDto; + + @Transient + public String conformedCaption; + + @Transient + public String conformedDescription; + + @Transient + public List dimensionAttributes; + + public Dimension(DimensionDto dimDto) { + this.setDimensionDto(dimDto); + } + + public void setDimensionAttributes(List lda) { + for (DimensionAttribute dimensionAttribute : lda) { + dimensionAttribute.setDimensionColumnAlias(dimensionAttribute.getColumnname() + "_" + getId().value); + } + this.dimensionAttributes = lda; + } + + public boolean isHidden() { + if(this.dimensionDto.isHidden == null) { + return false; + } + return this.dimensionDto.isHidden.booleanValue(); + } + + public DimensionDto getDimensionDto() { + return dimensionDto; + } + + public void setDimensionDto(DimensionDto dimensionDto) { + this.dimensionDto = dimensionDto; + } + + public Identifier getId() { + return this.dimensionDto.id; + } + + public String getCaption() { + return this.dimensionDto.caption; + } + + public Identifier getFactTableId() { + return this.dimensionDto.factTableId; + } + + public String getTablename() { + return this.dimensionDto.tablename; + } + + public String getJoincolumn() { + return this.dimensionDto.joincolumn; + } + + public String getAlias() { + return this.dimensionDto.alias; + } + + public String getDescription() { + return this.dimensionDto.description; + } + + public boolean isHierarchy() { + if(this.dimensionDto.isHierarchy == null) return false; + return this.dimensionDto.isHierarchy.booleanValue(); + } + + public boolean isHistorical() { + if(this.dimensionDto.isHistorical == null) return false; + return this.dimensionDto.isHistorical.booleanValue(); + } + + public String getConformed() { + return this.dimensionDto.conformed; + } + + public String getIdColumn() { + return this.dimensionDto.idColumn; + } + +} diff --git a/src/de/superx/bianalysis/models/DimensionAttribute.java b/src/de/superx/bianalysis/models/DimensionAttribute.java new file mode 100644 index 0000000..6502e0e --- /dev/null +++ b/src/de/superx/bianalysis/models/DimensionAttribute.java @@ -0,0 +1,276 @@ +package de.superx.bianalysis.models; + +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.AttributeDto; + +public class DimensionAttribute { + + public static final List SPECIAL_VALUES = List.of("n. v.", "k.A.", "k.a.", "unbekannt", "Unbekannt", "ungültig", "Ungültig"); + + public static final Comparator SPECIAL_VALUE_COMPARATOR = (a1, a2) -> { + if (a1.equals(a2)) { + return 0; + } + if (DimensionAttribute.SPECIAL_VALUES.contains(a1)) { + return -1; + } + if (DimensionAttribute.SPECIAL_VALUES.contains(a2)) { + return 1; + } + return a1.compareTo(a2); + }; + + public static String specialValueListForSql() { + String result = String.join("', '", SPECIAL_VALUES); + return "'" + result + "'"; + } + + + private String conformedCaption; + + private String conformedDescription; + + @JsonIgnore + private AttributeDto attributeTable; + + @JsonIgnore + private String dimCaption; + + @JsonIgnore + private String dimId; + + @JsonIgnore + private String dimConformedId; + + @JsonIgnore + private String tablename; + + @JsonIgnore + private String joincolumn; + + private boolean isHierarchy; + + @JsonIgnore + private boolean isHistorical; + + @JsonIgnore + private String dimensionTableAlias; + + @JsonIgnore + private String dimensionColumnAlias; + + private List dimensionAttributeValues; + + @JsonIgnore + private String dimIdJoinColumn; + + public DimensionAttribute() { + super(); + } + + public DimensionAttribute(AttributeDto attributeTable) { + this.attributeTable = attributeTable; + } + + public void setDimension(Dimension dim) { + this.dimCaption = dim.getCaption(); + this.dimId = dim.getId().composedId; + this.tablename = dim.getTablename(); + this.joincolumn = dim.getJoincolumn(); + this.isHierarchy = dim.isHierarchy(); + this.isHistorical = dim.isHistorical(); + this.dimIdJoinColumn = dim.getIdColumn(); + if(dim.getAlias() != null){ + this.dimensionTableAlias = dim.getAlias(); + } else { + this.dimensionTableAlias = generateDimensionTableAlias(joincolumn); + } + this.dimensionColumnAlias = getColumnname() + "_" + getId().value; + } + + public static String generateDimensionTableAlias(String joincolumn) { + if (joincolumn != null) { + return joincolumn.replaceFirst("_id$", ""); + } + return null; + } + + @Override + public int hashCode() { + return Objects.hash(getId()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + DimensionAttribute other = (DimensionAttribute) obj; + return Objects.equals(this.getId(), other.getId()); + } + + public String getCaption() { + return this.attributeTable.caption; + } + + public boolean isHidden() { + if(this.attributeTable.isHidden == null) { + return false; + } + return this.attributeTable.isHidden.booleanValue(); + } + + public String getColumnname() { + return this.attributeTable.columnname; + } + + public String getSortOrderColumn() { + return this.attributeTable.sortOrderColumn; + } + + public String getFilterSelection() { + return this.attributeTable.filterSelection; + } + + public String getDimId() { + return dimId; + } + + public void setDimId(String dimId) { + this.dimId = dimId; + } + + public String getDimConformedId() { + return dimConformedId; + } + + public void setDimConformedId(String dimConformedId) { + this.dimConformedId = dimConformedId; + } + + public String getAttrConformedId() { + return this.attributeTable.attrConformedId; + } + + public Identifier getId() { + return this.attributeTable.id; + } + + @JsonIgnore + public String getStringId() { + return this.attributeTable.id.composedId; + } + + public String getDimCaption() { + return dimCaption; + } + + public void setDimCaption(String dimCaption) { + this.dimCaption = dimCaption; + } + + public String getDimensionTableAlias() { + return dimensionTableAlias; + } + + public void setDimensionTableAlias(String dimensionTableAlias) { + this.dimensionTableAlias = dimensionTableAlias; + } + + public String getDimensionColumnAlias() { + return dimensionColumnAlias; + } + + public void setDimensionColumnAlias(String dimensionColumnAlias) { + this.dimensionColumnAlias = dimensionColumnAlias; + } + + public String getConformedCaption() { + return conformedCaption; + } + + public void setConformedCaption(String conformedCaption) { + this.conformedCaption = conformedCaption; + } + + public String getConformedDescription() { + return conformedDescription; + } + + public void setConformedDescription(String conformedDescription) { + this.conformedDescription = conformedDescription; + } + + public String getDescription() { + return this.attributeTable.description; + } + + public Identifier getDimensionId() { + return this.attributeTable.dimensionId; + } + + public String getTablename() { + return tablename; + } + + public void setTablename(String tablename) { + this.tablename = tablename; + } + + public String getJoincolumn() { + return joincolumn; + } + + public void setJoincolumn(String joincolumn) { + this.joincolumn = joincolumn; + } + + @JsonProperty(value="isHierarchy") + public boolean isHierarchy() { + return isHierarchy; + } + + public boolean isHistorical() { + return isHistorical; + } + + public void setHierarchy(boolean isHierarchy) { + this.isHierarchy = isHierarchy; + } + + public void setHistorical(boolean isHistorical) { + this.isHistorical = isHistorical; + } + + public boolean isHierarchicalFilter() { + return this.attributeTable.hierarchicalFilter.booleanValue(); + } + + public List getDimensionAttributeValues() { + return dimensionAttributeValues; + } + + public void setDimensionAttributeValues(List dimensionAttributeValues) { + this.dimensionAttributeValues = dimensionAttributeValues; + } + + public void setDimIdJoinColumn(String idColumn) { + this.dimIdJoinColumn = idColumn; + } + + public String getDimIdJoinColumn() { + return this.dimIdJoinColumn; + } + + public void setAttrConformedId(String stringId) { + this.attributeTable.attrConformedId = stringId; + } + +} diff --git a/src/de/superx/bianalysis/models/FactTable.java b/src/de/superx/bianalysis/models/FactTable.java new file mode 100644 index 0000000..5237df5 --- /dev/null +++ b/src/de/superx/bianalysis/models/FactTable.java @@ -0,0 +1,71 @@ +package de.superx.bianalysis.models; + +import java.util.List; + +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.Transient; +import org.springframework.data.relational.core.mapping.Table; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.FactDto; +import de.superx.jdbc.entity.Sachgebiet; + +@Table(value ="metadata\".\"facttable") +public class FactTable { + + private FactDto factDto; + + @Transient + private Sachgebiet sachgebiet; + + @Transient + private List conformedDimensions; + + public FactTable() {} + + public FactTable(FactDto factDto) { + this.factDto = factDto; + } + + public Identifier getId() { + return this.factDto.id; + } + + public String getCaption() { + if(this.factDto != null) { + return this.factDto.caption; + } + return null; + } + + public int getSachgebiettid() { + return this.factDto.sachgebiettid.intValue(); + } + + public String getDescription() { + return this.factDto.description; + } + + public String getTablename() { + return this.factDto.tablename; + } + + public List getConformedDimensions() { + return conformedDimensions; + } + + public void setConformedDimensions(List conformedDimensions) { + this.conformedDimensions = conformedDimensions; + } + + public Sachgebiet getSachgebiet() { + return sachgebiet; + } + + public void setSachgebiet(Sachgebiet sachgebiet) { + this.sachgebiet = sachgebiet; + } + +} diff --git a/src/de/superx/bianalysis/models/Filter.java b/src/de/superx/bianalysis/models/Filter.java new file mode 100644 index 0000000..c4467f4 --- /dev/null +++ b/src/de/superx/bianalysis/models/Filter.java @@ -0,0 +1,89 @@ +package de.superx.bianalysis.models; + +import java.util.List; +import java.util.StringJoiner; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import de.superx.bianalysis.ReportMetadata; +import de.superx.bianalysis.metadata.Identifier; + +public class Filter { + + public Identifier dimensionAttributeId; + public List filterValues; + public String columnname; + public String tablename; + public String joincolumn; + public String dimensionTableAlias; + + public Filter() { + super(); + } + + public Filter(List values, Identifier dimAttrId) { + this.filterValues = values; + this.dimensionAttributeId = dimAttrId; + } + + public Filter(Filter filter) { + this.dimensionAttributeId = filter.dimensionAttributeId; + this.filterValues = filter.filterValues; + this.columnname = filter.columnname; + this.tablename = filter.tablename; + this.joincolumn = filter.joincolumn; + this.dimensionTableAlias = filter.dimensionTableAlias; + } + + public void setDimensionAttribute(DimensionAttribute attr) { + this.columnname = attr.getColumnname(); + } + + public void setDimension(Dimension dim) { + this.tablename = dim.getTablename(); + this.joincolumn = dim.getJoincolumn(); + if(dim.getAlias() != null) { + this.dimensionTableAlias = dim.getAlias(); + } else { + this.dimensionTableAlias = joincolumn.replaceFirst("_id$", ""); + } + } + + public DimensionAttribute getDimAttribute(ReportMetadata reportMetadata) { + return reportMetadata.getDimAttrById(this.dimensionAttributeId); + } + + @Override + public String toString() { + return String.valueOf(this.dimensionAttributeId); + } + + public static Filter findFilterById(List filters, Identifier id) { + return filters + .stream() + .filter(f -> f.dimensionAttributeId.equals(id)) + .findFirst() + .orElse(null); + } + + @JsonIgnore + public String getValuesAsString() { + if(this.filterValues == null || this.filterValues.isEmpty()) { + return null; + } + return String.join(", ", this.filterValues); + } + + @JsonIgnore + public String getValues() { + if(this.filterValues == null || this.filterValues.isEmpty()) { + return null; + } + StringJoiner joiner = new StringJoiner(", "); + for (String value : filterValues) { + joiner.add("'"+value+"'"); + } + return joiner.toString(); + } + +} diff --git a/src/de/superx/bianalysis/models/Info.java b/src/de/superx/bianalysis/models/Info.java new file mode 100644 index 0000000..d2bc85c --- /dev/null +++ b/src/de/superx/bianalysis/models/Info.java @@ -0,0 +1,89 @@ +package de.superx.bianalysis.models; + +import java.util.ArrayList; +import java.util.List; + +import de.superx.rest.model.Item; + +public class Info { + + public String segmentCaption; + public String lastUpdateBiad; + + + public List sachgebiete = new ArrayList(); + public List facttables = new ArrayList(); + public List measures = new ArrayList(); + public List leftDimensionAttributes = new ArrayList(); + public List topDimensionAttributes = new ArrayList(); + public List filter = new ArrayList(); + public String hideEmptyColumns; + + public List sqlStatements = new ArrayList(); + + public String error; + + + public void addSachgebiet(String sachgebiet) { + sachgebiete.add(sachgebiet); + } + + public void addFacttable(InfoItem facttable) { + facttables.add(facttable); + } + + public void setMeasures(List measures) { + this.measures = measures; + } + + public void setLeftDimensionAttributes(List leftDimensionAttributes) { + this.leftDimensionAttributes = leftDimensionAttributes; + } + + public void setTopDimensionAttributes(List topDimensionAttributes) { + this.topDimensionAttributes = topDimensionAttributes; + } + + public void setSachgebiete(List sachgebiete) { + this.sachgebiete = sachgebiete; + } + + public void setFacttables(List facttables) { + this.facttables = facttables; + } + + public void setFilter(List filter) { + this.filter = filter; + } + + public void setSqlStatements(List sqlStatements) { + this.sqlStatements = sqlStatements; + } + + public void setLastUpdateBiad(String lastUpdateBiad) { + this.lastUpdateBiad = lastUpdateBiad; + } + + public void setErrorMessage(String error) { + this.error = error; + } + + public void setSegmentCaption(String segmentCaption) { + this.segmentCaption = segmentCaption; + } + + public void hideEmptyColumns(boolean hideEmptyColumns) { + if(hideEmptyColumns) { + this.hideEmptyColumns = "Ja"; + } else { + this.hideEmptyColumns = "Nein"; + } + } + + public void setHideEmptyColumns(String hideEmptyColumns) { + this.hideEmptyColumns = hideEmptyColumns; + } + + +} + diff --git a/src/de/superx/bianalysis/models/InfoItem.java b/src/de/superx/bianalysis/models/InfoItem.java new file mode 100644 index 0000000..c0d7eed --- /dev/null +++ b/src/de/superx/bianalysis/models/InfoItem.java @@ -0,0 +1,19 @@ +package de.superx.bianalysis.models; + +public class InfoItem { + + public String id; + public String caption; + public String description; + + public InfoItem(String id, String caption, String description) { + this.id = id; + this.caption = caption; + this.description = description; + } + + public InfoItem() { + super(); + } + +} diff --git a/src/de/superx/bianalysis/models/Measure.java b/src/de/superx/bianalysis/models/Measure.java new file mode 100644 index 0000000..fde33e1 --- /dev/null +++ b/src/de/superx/bianalysis/models/Measure.java @@ -0,0 +1,130 @@ +package de.superx.bianalysis.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.MeasureDto; +import de.superx.bianalysis.repository.dto.MeasureFilterDto; +import de.superx.rest.model.ColumnType; + +public class Measure { + + private MeasureDto measureDto; + + @JsonIgnore + public String filterTablename; + + @JsonIgnore + public String filterJoincolumn; + + @JsonIgnore + public String filterColumnname; + + @JsonIgnore + public String filterInclude; + + @JsonIgnore + public String filterExclude; + + @JsonIgnore + public String filterDimensionTableAlias; + + @JsonIgnore + public String filterCondition; + + @JsonIgnore + public String factColumnFilter; + + @JsonIgnore + public Identifier filterAttributeId; + + public Measure() { + super(); + } + + public Measure(MeasureDto measureDTO) { + this.measureDto = measureDTO; + } + + public void setMeasureFilterAttributes(MeasureFilterDto filter, DimensionAttribute attribute, Dimension dimension) { + this.filterInclude = filter.includedValues; + this.filterExclude = filter.excludedValues; + this.filterTablename = dimension.getTablename(); + this.filterJoincolumn = dimension.getJoincolumn(); + this.filterColumnname = attribute.getColumnname(); + this.filterAttributeId = attribute.getId(); + if (dimension.getAlias() != null) { + this.filterDimensionTableAlias = dimension.getAlias(); + } else { + this.filterDimensionTableAlias = generatefilterDimensionTableAlias(filterJoincolumn); + } + this.filterCondition = generateFilterCondition(); + } + + public void setFactColumnFilter(MeasureFilterDto filter) { + this.factColumnFilter = filter.factColumnFilter; + this.filterInclude = filter.includedValues; + this.filterExclude = filter.excludedValues; + this.filterCondition = generateFilterCondition(); + } + + private static String generatefilterDimensionTableAlias(String filterJoincolumn) { + if (filterJoincolumn != null) { + return filterJoincolumn.replaceFirst("_id$", ""); + } + return null; + } + + private String generateFilterCondition() { + if (this.measureDto.measureFilterId.value != null) { + StringBuilder filterConditionStatement = new StringBuilder(); + String tableDotColumn = this.filterDimensionTableAlias + "." + this.filterColumnname; + if(factColumnFilter != null && !factColumnFilter.isBlank()) { + tableDotColumn = factColumnFilter; + } + if (this.filterInclude != null) { + filterConditionStatement.append(tableDotColumn + " IN (" + + this.filterInclude + ")"); + } + if (this.filterInclude != null && this.filterExclude != null) { + filterConditionStatement.append(" AND "); + } + if (this.filterExclude != null) { + filterConditionStatement.append(tableDotColumn + + " NOT IN (" + this.filterExclude + ")"); + } + return filterConditionStatement.toString(); + } + return null; + } + + public Identifier getId() { + return this.measureDto.id; + } + + public String getCaption() { + return this.measureDto.caption; + } + + public String getColumnname() { + return this.measureDto.columnname; + } + + public String getDescription() { + return this.measureDto.description; + } + + public String getAggregationType() { + return this.measureDto.aggregationType; + } + + public ColumnType getMeasureType() { + return this.measureDto.measureType; + } + + public Identifier getMeasureFilterId() { + return this.measureDto.measureFilterId; + } + + +} \ No newline at end of file diff --git a/src/de/superx/bianalysis/models/Right.java b/src/de/superx/bianalysis/models/Right.java new file mode 100644 index 0000000..8c921a6 --- /dev/null +++ b/src/de/superx/bianalysis/models/Right.java @@ -0,0 +1,29 @@ +package de.superx.bianalysis.models; + +public enum Right { + + VIEW_REPORT("RIGHT_CS_BIA_ANALYSIS_VIEW_ANALYSIS_TABLE"), + CREATE_ANALYSIS("RIGHT_CS_BIA_ANALYSIS_CREATE_ANALYSIS"); + + private String string; + + Right(String string) { + this.setString(string); + } + + public String getString() { + return string; + } + + public void setString(String string) { + this.string = string; + } + + public static String getPrintableRights(Right... rights) { + String out = ""; + for (Right right : rights) { + out += right.toString(); + } + return out; + } +} diff --git a/src/de/superx/bianalysis/models/RightParam.java b/src/de/superx/bianalysis/models/RightParam.java new file mode 100644 index 0000000..2b28623 --- /dev/null +++ b/src/de/superx/bianalysis/models/RightParam.java @@ -0,0 +1,21 @@ +package de.superx.bianalysis.models; + +public enum RightParam { + + TOPIC_AREA("bianalysis.topic_area"), + TOPIC("bianalysis.topic"); + + private String string; + + RightParam(String string) { + this.setString(string); + } + + public String getString() { + return string; + } + + public void setString(String string) { + this.string = string; + } +} diff --git a/src/de/superx/bianalysis/repository/DimensionAttributeRepository.java b/src/de/superx/bianalysis/repository/DimensionAttributeRepository.java new file mode 100644 index 0000000..a301357 --- /dev/null +++ b/src/de/superx/bianalysis/repository/DimensionAttributeRepository.java @@ -0,0 +1,42 @@ +package de.superx.bianalysis.repository; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jdbc.repository.query.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; +import org.springframework.data.repository.query.Param; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.AttributeDto; +import de.superx.jdbc.repository.BiaAdminCrudRepository; + +@RepositoryDefinition(domainClass = AttributeDto.class, idClass = Identifier.class) +public interface DimensionAttributeRepository extends BiaAdminCrudRepository { + + List findByDimensionId(Identifier dimensionId); + + Optional findById(Identifier id); + + @Query( + "SELECT da.id" + + " FROM metadata.dimension_attribute da" + + " LEFT JOIN metadata.dimension d" + + " ON d.id = da.dimension_id" + + " WHERE da.conformed = :confAttrId" + + " AND facttable_id = :factId" + ) + List findAttributesByConformedAttributeAndFactTable(@Param("confAttrId") String confAttrId, @Param("factId") String factId); + + @Query( + "SELECT da.id" + + " FROM metadata.dimension_attribute da" + + " LEFT JOIN metadata.dimension d" + + " ON d.id = da.dimension_id" + + " WHERE da.id = :attrId" + + " AND d.facttable_id = :factId" + ) + Identifier findAttributesByIdAndFactTable(@Param("attrId") String confAttrId, @Param("factId") String factId); + +} \ No newline at end of file diff --git a/src/de/superx/bianalysis/repository/DimensionRepository.java b/src/de/superx/bianalysis/repository/DimensionRepository.java new file mode 100644 index 0000000..cc9f340 --- /dev/null +++ b/src/de/superx/bianalysis/repository/DimensionRepository.java @@ -0,0 +1,50 @@ +package de.superx.bianalysis.repository; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.jdbc.repository.query.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; +import org.springframework.data.repository.query.Param; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.DimensionDto; +import de.superx.jdbc.repository.BiaAdminCrudRepository; + +@RepositoryDefinition(domainClass = DimensionDto.class, idClass = Identifier.class) +public interface DimensionRepository extends BiaAdminCrudRepository { + + List findByFactTableId(Identifier factTableId); + + Optional findById(Identifier id); + + @Override + List findAll(); + + @Query( + " SELECT d.id" + + " FROM metadata.dimension d" + + " LEFT JOIN metadata.dimension_attribute da" + + " ON da.dimension_id = d.id" + + " WHERE da.id is null" + + " AND d.conformed = :confDim" + + " AND d.facttable_id = :factId" + ) + List getRolePlayingIds(@Param("confDim") String confDim, @Param("factId") String factId); + + @Query( + "SELECT dimension_id " + + "FROM metadata.dimension_attribute da " + + "WHERE id = :attrId" + ) + Identifier findDimensionIdForAttribute(@Param("attrId") String attrId); + + @Query( + "SELECT conformed" + + " FROM metadata.dimension" + + " WHERE facttable_id = :factId" + + " AND conformed IS NOT NULL" + ) + List getUsedConformedDimensionsByFactTable(@Param("factId") String factId); +} diff --git a/src/de/superx/bianalysis/repository/FactRepository.java b/src/de/superx/bianalysis/repository/FactRepository.java new file mode 100644 index 0000000..95de0da --- /dev/null +++ b/src/de/superx/bianalysis/repository/FactRepository.java @@ -0,0 +1,47 @@ +package de.superx.bianalysis.repository; + +import java.util.List; +import java.util.Optional; +import org.springframework.data.jdbc.repository.query.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; +import org.springframework.data.repository.query.Param; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.FactDto; +import de.superx.jdbc.repository.BiaAdminCrudRepository; + +@RepositoryDefinition(domainClass = FactDto.class, idClass = Identifier.class) +public interface FactRepository extends BiaAdminCrudRepository { + + @Override + List findAll(); + + Optional findById(Identifier id); + + Optional findByTablename(String tablename); + + @Query( + "SELECT COUNT(*) > 0" + + " FROM metadata.facttable f" + + " LEFT JOIN metadata.measure m" + + " ON m.facttable_id = f.id" + + " WHERE f.id = :factId" + + " AND m.id = :measureId" + ) + boolean hasFactTableMeasure(@Param("factId") String factId, @Param("measureId") String measureId); + + @Query( + "SELECT f.tablename" + + " FROM metadata.dimension_attribute da" + + " LEFT JOIN metadata.dimension d" + + " ON d.id = da.dimension_id" + + " LEFT JOIN metadata.facttable f" + + " ON f.id = d.facttable_id" + + " WHERE f.id = :factId" + + " AND (da.conformed = :attrId OR da.id = :attrId)" + ) + String getFactTableNameForAttribute(@Param("factId") String factId, @Param("attrId") String attrId); + +} + diff --git a/src/de/superx/bianalysis/repository/MeasureFilterRepository.java b/src/de/superx/bianalysis/repository/MeasureFilterRepository.java new file mode 100644 index 0000000..6eff313 --- /dev/null +++ b/src/de/superx/bianalysis/repository/MeasureFilterRepository.java @@ -0,0 +1,13 @@ +package de.superx.bianalysis.repository; + +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.MeasureFilterDto; +import de.superx.jdbc.repository.BiaAdminCrudRepository; + +@RepositoryDefinition(domainClass = MeasureFilterDto.class, idClass = Identifier.class) +public interface MeasureFilterRepository extends BiaAdminCrudRepository { + +} diff --git a/src/de/superx/bianalysis/repository/MeasureRepository.java b/src/de/superx/bianalysis/repository/MeasureRepository.java new file mode 100644 index 0000000..f34e222 --- /dev/null +++ b/src/de/superx/bianalysis/repository/MeasureRepository.java @@ -0,0 +1,17 @@ +package de.superx.bianalysis.repository; + +import java.util.List; + +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.repository.dto.MeasureDto; +import de.superx.jdbc.repository.BiaAdminCrudRepository; + +@RepositoryDefinition(domainClass = MeasureDto.class, idClass = Identifier.class) +public interface MeasureRepository extends BiaAdminCrudRepository { + + List findByFactTableId(Identifier id); + +} diff --git a/src/de/superx/bianalysis/repository/StoredReportRepository.java b/src/de/superx/bianalysis/repository/StoredReportRepository.java new file mode 100644 index 0000000..fe7aa23 --- /dev/null +++ b/src/de/superx/bianalysis/repository/StoredReportRepository.java @@ -0,0 +1,23 @@ +package de.superx.bianalysis.repository; + +import java.util.List; +import java.util.Optional; + +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.RepositoryDefinition; + +import de.superx.bianalysis.StoredReport; + +@RepositoryDefinition(domainClass = StoredReport.class, idClass = Integer.class) +public interface StoredReportRepository extends CrudRepository { + + Optional findByName(String name); + + Optional findById(int id); + + void deleteById(int id); + + @Override + List findAll(); + +} diff --git a/src/de/superx/bianalysis/repository/dto/AttributeDto.java b/src/de/superx/bianalysis/repository/dto/AttributeDto.java new file mode 100644 index 0000000..3ce0e73 --- /dev/null +++ b/src/de/superx/bianalysis/repository/dto/AttributeDto.java @@ -0,0 +1,74 @@ +package de.superx.bianalysis.repository.dto; + +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.jdbc.entity.EntityBase; +import de.superx.jdbc.model.DynamicFieldType; +import de.superx.jdbc.model.EntityDescriptor; +import de.superx.jdbc.model.TableRef; +import de.superx.rest.model.ColumnType; +import de.superx.rest.model.FieldType; + +@Table(schema = "metadata", value = "dimension_attribute") +public class AttributeDto extends EntityBase { + + @Id + @DynamicFieldType(label="ID", readOnly = true, visibleInSimplifiedForm = false) + public Identifier id; + + @EntityDescriptor + @DynamicFieldType(label="Titel") + public String caption; + + @DynamicFieldType(label="Beschreibung", editControlType=FieldType.TextArea) + public String description; + + @DynamicFieldType(label="Dimension", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "dimension", keyField = "id", labelField = "caption") + @Column(value = "dimension_id") + public Identifier dimensionId; + + @DynamicFieldType(label="Spaltenname", readOnly = true) + public String columnname; + + @DynamicFieldType(label="Sortierspalte", readOnly = true, visibleInSimplifiedForm = false) + @Column(value = "sort_order_column") + public String sortOrderColumn; + + @DynamicFieldType(label="Filter-Auswahl", visibleInSimplifiedForm = false) + @Column(value = "filter_selection") + public String filterSelection; + + @DynamicFieldType(label="Hierarchie", editControlType=FieldType.Select, columnType = ColumnType.BooleanColumnBiAnalysis, readOnly = true) + @Column(value = "hierarchical_filter") + public Boolean hierarchicalFilter; + + @DynamicFieldType(label="Ausgeblendet", editControlType=FieldType.Select, columnType = ColumnType.BooleanColumnBiAnalysis) + @Column(value = "is_hidden") + public Boolean isHidden; + + @DynamicFieldType(label="Conformed Attribute", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "dimension_attribute", keyField = "id", labelField = "caption") + @Column(value = "conformed") + public String attrConformedId; + + @DynamicFieldType(label="Auslieferungsversion", visibleInSimplifiedForm = false) + @Column(value = "default_release") + public String defaultRelease; + + public AttributeDto() {} + + @Override + public boolean canBeCreatedByUser() { + return false; + } + + @Override + public boolean canBeDeletedByUser() { + return false; + } + +} diff --git a/src/de/superx/bianalysis/repository/dto/DimensionDto.java b/src/de/superx/bianalysis/repository/dto/DimensionDto.java new file mode 100644 index 0000000..f265d89 --- /dev/null +++ b/src/de/superx/bianalysis/repository/dto/DimensionDto.java @@ -0,0 +1,79 @@ +package de.superx.bianalysis.repository.dto; + +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.jdbc.entity.EntityBase; +import de.superx.jdbc.model.DynamicFieldType; +import de.superx.jdbc.model.EntityDescriptor; +import de.superx.rest.model.ColumnType; +import de.superx.rest.model.FieldType; +import de.superx.jdbc.model.TableRef; + +@Table(schema="metadata", value = "dimension") +public class DimensionDto extends EntityBase{ + + @Id + @DynamicFieldType(label="ID", readOnly = true) + public Identifier id; + + @EntityDescriptor + @DynamicFieldType(label="Titel") + public String caption; + + @DynamicFieldType(label="Beschreibung", editControlType=FieldType.TextArea) + public String description; + + @DynamicFieldType(label="Faktentabelle", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "facttable", keyField = "id", labelField = "caption") + @Column(value = "facttable_id") + public Identifier factTableId; + + @DynamicFieldType(label="Tabellenname", readOnly = true) + public String tablename; + + @DynamicFieldType(label="Join-Spalte", readOnly = true, visibleInSimplifiedForm = false) + public String joincolumn; + + @DynamicFieldType(label="Join-Alias", readOnly = true, visibleInSimplifiedForm = false) + public String alias; + + @DynamicFieldType(label="Hierarchie", editControlType=FieldType.Select, columnType = ColumnType.BooleanColumnBiAnalysis, readOnly = true) + @Column(value = "is_hierarchy") + public Boolean isHierarchy; + + @DynamicFieldType(label="Historisch", editControlType=FieldType.Select, columnType = ColumnType.BooleanColumnBiAnalysis, readOnly = true) + @Column(value = "is_historical") + public Boolean isHistorical; + + @DynamicFieldType(label="Conformed Dimension", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "dimension", keyField = "id", labelField = "caption") + public String conformed; + + @DynamicFieldType(label="ID Spalte", readOnly = true, visibleInSimplifiedForm = false) + @Column(value = "id_column") + public String idColumn; + + @DynamicFieldType(label="Auslieferungsversion", visibleInSimplifiedForm = false) + @Column(value = "default_release") + public String defaultRelease; + + @DynamicFieldType(label="Ausgeblendet", editControlType=FieldType.Select, columnType = ColumnType.BooleanColumnBiAnalysis) + @Column(value = "is_hidden") + public Boolean isHidden; + + public DimensionDto() {} + + @Override + public boolean canBeCreatedByUser() { + return false; + } + + @Override + public boolean canBeDeletedByUser() { + return false; + } + +} diff --git a/src/de/superx/bianalysis/repository/dto/FactDto.java b/src/de/superx/bianalysis/repository/dto/FactDto.java new file mode 100644 index 0000000..90a7d4f --- /dev/null +++ b/src/de/superx/bianalysis/repository/dto/FactDto.java @@ -0,0 +1,50 @@ +package de.superx.bianalysis.repository.dto; + +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.jdbc.entity.EntityBase; +import de.superx.jdbc.model.DynamicFieldType; +import de.superx.jdbc.model.EntityDescriptor; +import de.superx.jdbc.model.TableRef; +import de.superx.rest.model.FieldType; + +@Table(schema = "metadata", value = "facttable") +public class FactDto extends EntityBase { + + @Id + @DynamicFieldType(label="ID", readOnly = true, visibleInSimplifiedForm = false) + public Identifier id; + + @EntityDescriptor + @DynamicFieldType(label="Titel") + public String caption; + + @DynamicFieldType(label="Beschreibung", editControlType=FieldType.TextArea) + public String description; + + @DynamicFieldType(label = "Sachgebiet", editControlType = FieldType.Select) + @TableRef(table = "sachgebiete", keyField = "tid", labelField = "name") + public Integer sachgebiettid; + + @DynamicFieldType(label="Tabellenname", readOnly = true) + public String tablename; + + @DynamicFieldType(label="Auslieferungsversion", visibleInSimplifiedForm = false) + @Column(value = "default_release") + public String defaultRelease; + + public FactDto() {} + + @Override + public boolean canBeCreatedByUser() { + return false; + } + + @Override + public boolean canBeDeletedByUser() { + return false; + } +} diff --git a/src/de/superx/bianalysis/repository/dto/MeasureDto.java b/src/de/superx/bianalysis/repository/dto/MeasureDto.java new file mode 100644 index 0000000..201be5b --- /dev/null +++ b/src/de/superx/bianalysis/repository/dto/MeasureDto.java @@ -0,0 +1,66 @@ +package de.superx.bianalysis.repository.dto; + +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.jdbc.entity.EntityBase; +import de.superx.jdbc.model.DynamicFieldType; +import de.superx.jdbc.model.EntityDescriptor; +import de.superx.rest.model.ColumnType; +import de.superx.rest.model.FieldType; +import de.superx.jdbc.model.TableRef; + +@Table(schema = "metadata", value = "measure") +public class MeasureDto extends EntityBase { + + @Id + @DynamicFieldType(label="ID", readOnly = true, visibleInSimplifiedForm = false) + public Identifier id; + + @EntityDescriptor + @DynamicFieldType(label="Titel") + public String caption; + + @DynamicFieldType(label="Beschreibung", editControlType=FieldType.TextArea) + public String description; + + @DynamicFieldType(label="Spaltenname") + public String columnname; + + @DynamicFieldType(label="Faktentabelle", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "facttable", keyField = "id", labelField = "caption") + @Column(value = "facttable_id") + public Identifier factTableId; + + @DynamicFieldType(label="Filter", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "measure_filter", keyField = "id", labelField = "caption") + @Column(value = "measure_filter_id") + public Identifier measureFilterId; + + @DynamicFieldType(label="Aggregationstyp") + @Column(value = "aggregation_type") + public String aggregationType; + + @DynamicFieldType(label="Datentyp", readOnly = true, visibleInSimplifiedForm = false) + @Column(value = "measure_type") + public ColumnType measureType; + + @DynamicFieldType(label="Auslieferungsversion", visibleInSimplifiedForm = false) + @Column(value = "default_release") + public String defaultRelease; + + public MeasureDto() {}; + + @Override + public boolean canBeCreatedByUser() { + return false; + } + + @Override + public boolean canBeDeletedByUser() { + return false; + } + +} diff --git a/src/de/superx/bianalysis/repository/dto/MeasureFilterDto.java b/src/de/superx/bianalysis/repository/dto/MeasureFilterDto.java new file mode 100644 index 0000000..0a4141c --- /dev/null +++ b/src/de/superx/bianalysis/repository/dto/MeasureFilterDto.java @@ -0,0 +1,61 @@ +package de.superx.bianalysis.repository.dto; + +import org.springframework.data.annotation.Id; +import org.springframework.data.relational.core.mapping.Column; +import org.springframework.data.relational.core.mapping.Table; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.jdbc.entity.EntityBase; +import de.superx.jdbc.model.DynamicFieldType; +import de.superx.jdbc.model.EntityDescriptor; +import de.superx.rest.model.FieldType; +import de.superx.jdbc.model.TableRef; + +@Table(schema = "metadata", value = "measure_filter") +public class MeasureFilterDto extends EntityBase { + + @Id + @DynamicFieldType(label="ID", readOnly = true, visibleInSimplifiedForm = false) + public Identifier id; + + @EntityDescriptor + @DynamicFieldType(label="Titel") + public String caption; + + @DynamicFieldType(label="Beschreibung", editControlType=FieldType.TextArea) + public String description; + + @DynamicFieldType(label="Attribute", readOnly = true, visibleInSimplifiedForm = false) + @TableRef(schema = "metadata", table = "dimension_attribute", keyField = "id", labelField = "caption") + @Column(value = "dimension_attribute_id") + public Identifier dimensionAttributeId; + + @DynamicFieldType(label="Faktenspalte Filter", readOnly = true) + @Column(value = "fact_column_filter") + public String factColumnFilter; + + @DynamicFieldType(label="Einbezogene Werte", readOnly = true) + @Column(value = "included_values") + public String includedValues; + + @DynamicFieldType(label="Ausgeschlossene Werte", readOnly = true) + @Column(value = "excluded_values") + public String excludedValues; + + @DynamicFieldType(label="Auslieferungsversion", visibleInSimplifiedForm = false) + @Column(value = "default_release") + public String defaultRelease; + + public MeasureFilterDto() {} + + @Override + public boolean canBeCreatedByUser() { + return false; + } + + @Override + public boolean canBeDeletedByUser() { + return false; + } + +} diff --git a/src/de/superx/bianalysis/rest/BiAnalysisApi.java b/src/de/superx/bianalysis/rest/BiAnalysisApi.java new file mode 100644 index 0000000..600567c --- /dev/null +++ b/src/de/superx/bianalysis/rest/BiAnalysisApi.java @@ -0,0 +1,277 @@ +package de.superx.bianalysis.rest; + +import java.io.ByteArrayOutputStream; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Base64; +import java.util.Date; +import org.apache.log4j.Logger; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import de.superx.bianalysis.ExcelSheetBuilder; +import de.superx.bianalysis.ReportDefinition; +import de.superx.bianalysis.StoredReport; +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.Dimension; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.FactTable; +import de.superx.bianalysis.models.Measure; +import de.superx.bianalysis.models.Right; +import de.superx.bianalysis.service.BiAnalysisManager; +import de.superx.bianalysis.service.BiAnalysisRightService; +import de.superx.bianalysis.service.DbMetaAdapter; +import de.superx.common.NotYetImplementedException; +import de.superx.rest.RestControllerBase; +import de.superx.rest.model.Download; +import de.superx.rest.model.Result; +import de.superx.rest.model.ResultType; +import de.superx.rest.model.Row; + +@RestController +@RequestMapping("/api/reportwizard") +public class BiAnalysisApi extends RestControllerBase { + + /* Autor: Robin Wübbeling + * Achtung: Code ist schnell und unschön zusammengestellt, für einen ersten Entwurf + * TODO: Filter + * TODO: Berechnete Kennzahlen + * TODO: Kennzahlenfilter momentan nur über 1 Dimensionsattribut möglich + * TODO: Strings schöner durch "Platzhalter" + * */ + + static Logger logger = Logger.getLogger(BiAnalysisApi.class); + + @Autowired + DbMetaAdapter dbAdapter; + + @Autowired + BiAnalysisRightService rightsService; + + @Autowired + BiAnalysisManager biAnalysisManager; + + @Override + protected Logger getLogger() { + return logger; + } + + @RequestMapping(method = RequestMethod.GET, path = "/facttables") + public List listFactTables() throws NotYetImplementedException { + List sachgebiete = rightsService.getSachgebiete(Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + List factTables = rightsService.getFactTables(Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + List facts = dbAdapter.getFactTables(sachgebiete, factTables); + return facts; + } + + @RequestMapping(method = RequestMethod.GET, path = "/dimensions") + public List listDimensions(@RequestParam(value = "facttable_id") String facttable_id) { + int sachgebietTid = dbAdapter.getSachgebietByFactTableId(facttable_id); + rightsService.checkSachgebiet(sachgebietTid, Right.CREATE_ANALYSIS); + rightsService.checkFactTable(new Identifier(facttable_id), Right.CREATE_ANALYSIS); + return dbAdapter.getDimensions(new Identifier(facttable_id)); + } + + // TODO: zeig in benamung does es sich vllt. um eine reduzierte liste handelt + // granted vs allowed + // umstellen auf camel case + @RequestMapping(method = RequestMethod.GET, path = "/dimensionAttributeValues") + public List listAttributeValues (@RequestParam(value = "attribute_id") List attribute_id, @RequestParam(value = "facts") List facts) { + List tids = rightsService.getSachgebiete(Right.CREATE_ANALYSIS); + List factTables = rightsService.getFactTables(Right.CREATE_ANALYSIS); + List attributes = dbAdapter.getAllowedDimensionAttributes(attribute_id, tids, factTables); + return dbAdapter.getDimensionAttributeValues(attributes, facts); + } + + @RequestMapping(method = RequestMethod.GET, path = "/dimensionAttributeValuesHierarchy") + public List> listAttributeValuesHierarchy(@RequestParam(value = "attribute_id") String attribute_id) { + rightsService.checkCreateRights(); + return dbAdapter.getDimensionAttributeValuesHierarchy(new Identifier(attribute_id)); + } + + @RequestMapping(method = RequestMethod.GET, path = "/measures") + public List listMeasures(@RequestParam(value = "facttable_id") String facttable_id) { + rightsService.checkSachgebiet(dbAdapter.getSachgebietByFactTableId(facttable_id), Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + rightsService.checkFactTable(new Identifier(facttable_id), Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + return dbAdapter.getMeasures(new Identifier(facttable_id)); + } + + @RequestMapping(method = RequestMethod.GET, path = "/findReportDefinition") + public List findReportDefinition( + @RequestParam(value = "title") Optional title, + @RequestParam(value = "sach") Optional sach, + @RequestParam(value = "facts") Optional> facts) { + + List allowedSachgebiete = rightsService.getSachgebiete(Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + List allowedFacts = rightsService.getFactTables(Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + + List result = new ArrayList<>(); + for (StoredReport report : dbAdapter.findAllStoredReports()) { + + List sachgebieteOfReport = dbAdapter.getSachgebieteForReport(report.reportDefinition); + if (!allowedSachgebiete.isEmpty() && !allowedSachgebiete.containsAll(sachgebieteOfReport)) { + continue; + } + + if (title.isPresent()) { + if (!report.name.toLowerCase().contains(title.get().toLowerCase())) { + continue; + } + } + + if (sach.isPresent()) { + if (!sachgebieteOfReport.contains(sach.get())) { + continue; + } + } + + if(allowedFacts != null && allowedFacts.size() > 0) { + boolean isFactAllowed = true; + for (Identifier reportFactId : report.reportDefinition.factTableIds) { + if(!allowedFacts.isEmpty() && !allowedFacts.contains(reportFactId)) { + isFactAllowed = false; + } + } + if (!isFactAllowed) { + continue; + } + } + + if (facts.isPresent()) { + boolean isFactMissing = false; + for (Identifier reportFactId : report.reportDefinition.factTableIds) { + if (!facts.get().contains(reportFactId.composedId)) { + isFactMissing = true; + break; + } + } + if (isFactMissing) { + continue; + } + } + result.add(report); + } + return result; + } + + @RequestMapping(method = RequestMethod.GET, path = "/getStoredReport") + public StoredReport getStoredReport(@RequestParam(value = "id") int id) { + Optional storedReportOpt = dbAdapter.findById(id); + if(storedReportOpt.isPresent()) { + StoredReport storedReport = storedReportOpt.get(); + checkCreateOrViewRightForFactTables(storedReport.reportDefinition.factTableIds); + try { + storedReport.exportedResult = biAnalysisManager.createResult(storedReport.reportDefinition, dbAdapter); + } catch (Exception e) { + logger.error("Couldn't create report", e); + e.printStackTrace(); + } + + storedReport.isReadOnly = Boolean.valueOf(!this.rightsService.isCreateRight()); + return storedReport; + } + return null; + } + + @RequestMapping(method = RequestMethod.POST, path = "/report") + public Result getReport(@RequestBody final ReportDefinition reportDefinition) throws Exception { + List factTableIds = reportDefinition.factTableIds; + checkCreateRightForFactTables(factTableIds); + return biAnalysisManager.createResult(reportDefinition, dbAdapter); + } + + @RequestMapping(method = RequestMethod.POST, path = "/persistReportDefinition") + public int persistReportDefinition(@RequestBody final StoredReport storedReport) throws Exception { + StoredReport.setReportDefinitionJson(storedReport); + List factTableIds = storedReport.reportDefinition.factTableIds; + checkCreateRightForFactTables(factTableIds); + return dbAdapter.saveReportDefinition(storedReport); + } + + @RequestMapping(value = "/report/download", method = RequestMethod.POST) + public Download getFile(@RequestBody final StoredReport storedReport) throws Exception { + checkCreateOrViewRightForFactTables(storedReport.reportDefinition.factTableIds); + Date date = new Date(); + String fileName = "BI-Analyse_"; + if(storedReport.id != 0) { + fileName += dbAdapter.findById(storedReport.id).get().name + "_"; + if(fileName.length() > 206) { + fileName = fileName.substring(0, 160); + } + fileName = fileName.replaceAll("[^a-zA-Z0-9äöüÄÖÜß_]+", "_"); + } + fileName += new SimpleDateFormat("yyyyMMdd_HHmmss").format(date); + if(!storedReport.exportedResult.resultType.equals(ResultType.FlatTable)) { + Row totalRow = storedReport.exportedResult.getTotalRow(); + storedReport.exportedResult.rows = (BiAnalysisManager.hierarchyToRows(storedReport.hierarchy)); + storedReport.exportedResult.rows.add(totalRow); + } + XSSFWorkbook workbook = new ExcelSheetBuilder(storedReport.exportedResult) + .withFileName(fileName) + .withReportName(storedReport.name) + .withDescription(storedReport.description) + .withDate(date) + .build(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + workbook.write(bos); + String base64String = Base64.getEncoder().encodeToString(bos.toByteArray()); + String contentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + return new Download(fileName, contentType, base64String); + } + + @RequestMapping(method = RequestMethod.POST, path = "/deleteReportDefinition") + public boolean deleteReportDefinition(@RequestBody final int id) throws Exception { + try { + Optional reportOpt = dbAdapter.findById(id); + if(reportOpt.isEmpty()) { + throw new Exception("FEHLER: Berichtskonfiguration konnte nicht gefunden werden."); + } + checkCreateRightForFactTables(reportOpt.get().reportDefinition.factTableIds); + dbAdapter.deleteById(id); + return true; + } catch(Exception e) { + throw new Exception("FEHLER: Berichtskonfiguration konnte nicht gelöscht werden.", e); + } + } + + @RequestMapping(method = RequestMethod.GET, path = "/reportDefinitions") + public List listReportDefinitions() throws Exception { + rightsService.checkCreateOrViewRights(); + List storedReports = null; + try { + storedReports = dbAdapter.findAllStoredReports(); + // TODO mit Marnie abklären, für überschreiben notwendig? + } catch (Exception e) { + e.printStackTrace(); + if (e.getCause().getMessage().contains("FEHLER: Relation »metadata.rw_report_definitions« existiert nicht")) { + throw new NotYetImplementedException("Bitte installieren Sie zuerst die Komponente 'BI-Analyse-Daten' und führen Sie anschließend den Konnektor aus."); + } + throw e; + } + return storedReports; + } + + private void checkCreateOrViewRightForFactTables(List factTableIds) { + for (Identifier factId : factTableIds) { + int sachgebiet = dbAdapter.getSachgebietByFactTableId(factId.composedId); + rightsService.checkSachgebiet(sachgebiet, Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + rightsService.checkFactTable(factId, Right.CREATE_ANALYSIS, Right.VIEW_REPORT); + } + } + + private void checkCreateRightForFactTables(List factTableIds) { + for (Identifier factId : factTableIds) { + int sachgebiet = dbAdapter.getSachgebietByFactTableId(factId.composedId); + rightsService.checkSachgebiet(sachgebiet, Right.CREATE_ANALYSIS); + rightsService.checkFactTable(factId, Right.CREATE_ANALYSIS); + } + } + +} diff --git a/src/de/superx/bianalysis/service/BiAnalysisManager.java b/src/de/superx/bianalysis/service/BiAnalysisManager.java new file mode 100644 index 0000000..00a5aac --- /dev/null +++ b/src/de/superx/bianalysis/service/BiAnalysisManager.java @@ -0,0 +1,141 @@ +package de.superx.bianalysis.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import de.superx.bianalysis.ColumnElement; +import de.superx.bianalysis.ColumnElementBuilder; +import de.superx.bianalysis.ReportDefinition; +import de.superx.bianalysis.ReportMetadata; +import de.superx.bianalysis.ResultBuilder; +import de.superx.bianalysis.ResultMerger; +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.sqlgeneration.SQLGenerator; +import de.superx.bianalysis.sqlgeneration.SQLGeneratorTotals; +import de.superx.common.NotYetImplementedException; +import de.superx.rest.model.Item; +import de.superx.rest.model.Result; +import de.superx.rest.model.Row; +import de.superx.rest.model.TreeNode; + +@Service +public class BiAnalysisManager { + + static Logger logger = Logger.getLogger(BiAnalysisManager.class); + + @Autowired + BiAnalysisRightService biAnalysisRightService; + + public Result createResult(ReportDefinition reportDefinition, DbMetaAdapter dbAdapter) throws Exception { + + List results = new ArrayList<>(); + ResultMerger resultMerger = new ResultMerger(dbAdapter); + + for (Identifier factTableId : reportDefinition.factTableIds) { + ReportDefinition definition = resultMerger.createFactTableSpecificReportDefinition(reportDefinition, factTableId); + if(definition.leftDimensionAttributeIds.isEmpty() || + definition.measureIds.isEmpty()) { + continue; + } + try { + biAnalysisRightService.checkCreateOrViewRights(); + ReportMetadata metadata = new ReportMetadata(definition, factTableId, dbAdapter); + checkColLimit(reportDefinition, dbAdapter, metadata); + Result reportSegment = getReportData(metadata, dbAdapter); + results.add(reportSegment); + } catch (Exception e) { + logger.error("Couldn't create report", e); + throw e; + } + } + Result result; + if(reportDefinition.factTableIds.size() > 1) { + result = resultMerger.buildMergedReport(reportDefinition, results); + } else { + result = results.get(0); + } + return result; + } + + private static void checkColLimit(ReportDefinition reportDefinition, DbMetaAdapter dbAdapter, ReportMetadata metadata) throws NotYetImplementedException { + final int POSTGRES_MAX_COL_LIMIT = 1664; + int resultCols = dbAdapter.getColNumbers(metadata.topDimensionAttributes, metadata.filters); + resultCols *= reportDefinition.measureIds.size(); + if(resultCols > POSTGRES_MAX_COL_LIMIT - 1) { + throw new NotYetImplementedException("FEHLER: Ihre Anfrage überschreitet das Spaltenlimit. " + + "Bitte wählen Sie eine andere Kombination an Attributen."); + } + } + + public static String getSqlStatement(ReportDefinition definition, DbMetaAdapter dbAdapter) { + String sqlStatement = ""; + ReportMetadata reportMetadata = definition.getReportMetadata(dbAdapter, definition.factTableIds.get(0)); + List columnElements = ColumnElementBuilder.buildColumnElements(reportMetadata); + SQLGenerator sqlGenerator = new SQLGenerator(reportMetadata, columnElements); + sqlStatement = sqlGenerator.buildSqlStatement(); + return sqlStatement; + } + + private Result getReportData(ReportMetadata metadata, DbMetaAdapter dbAdapter) throws Exception { + List columnElements = ColumnElementBuilder.buildColumnElements(metadata); + + List sqlStatements = new ArrayList<>(); + String reportSQL = new SQLGenerator(metadata, columnElements).buildFormattedSqlStatement(); + String totalsColumnSQL = SQLGeneratorTotals.generateTotalsColumnSQL(metadata); + sqlStatements.add(new Item("noAggregatesSQL", reportSQL)); + sqlStatements.add(new Item("totalsColumnSQL", totalsColumnSQL)); + + ResultBuilder resultBuilder = new ResultBuilder(dbAdapter.getDataSource()); + resultBuilder.setColumnElements(columnElements); + resultBuilder.setReportMetadata(metadata); + + Result report = resultBuilder.buildReport(sqlStatements, biAnalysisRightService.isCreateRight()); + return report; + } + + public static List hierarchyToRows(ArrayList hierarchy) { + List rows = new ArrayList(); + for (TreeNode treeNode : hierarchy) { + nodeToRow(treeNode, rows); + } + return rows; + } + + private static List nodeToRow(TreeNode> treeNode, List rows) { + splitHierarchyColumn(treeNode); + rows.add(new Row(treeNode.data, true)); + for (TreeNode child: treeNode.children) { + List childRows = nodeToRow(child, rows); + for (Row row : childRows) { + if (!rows.contains(row)){ + rows.add(row); + } + } + } + return rows; + } + + //Im Treenode wird die Hierarchie in einer Spalte abgebildet, diese muss wieder auf die ursprünglichen Spalten aufgeuteilt werden + private static void splitHierarchyColumn(TreeNode> node) { + String realColumn = node.data.get("column").toString(); + if( realColumn.contains(" (Ebene ")) { + String[] splittedString = realColumn.split(" \\(Ebene "); + String mainColumn = splittedString[0]; + node.data.put(realColumn, node.data.get(mainColumn)); + node.data.put(mainColumn, ""); + } + } + + public static String getTotalsColumnSqlStatement(ReportDefinition definition, DbMetaAdapter dbAdapter) { + String sqlStatement = ""; + ReportMetadata metadata = new ReportMetadata(definition, definition.factTableIds.get(0), dbAdapter); + sqlStatement = SQLGeneratorTotals.generateTotalsColumnSQL(metadata); + return sqlStatement; + } + +} diff --git a/src/de/superx/bianalysis/service/BiAnalysisRightService.java b/src/de/superx/bianalysis/service/BiAnalysisRightService.java new file mode 100644 index 0000000..a27687d --- /dev/null +++ b/src/de/superx/bianalysis/service/BiAnalysisRightService.java @@ -0,0 +1,135 @@ +package de.superx.bianalysis.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.Right; +import de.superx.bianalysis.models.RightParam; +import de.superx.common.AccessDeniedException; +import de.superx.common.SxUser; +import de.superx.spring.service.UserService; + +@Service() +public class BiAnalysisRightService { + + @Autowired + DbMetaAdapter dbAdapter; + + @Autowired + UserService userService; + + public void checkCreateRights() { + SxUser user = (SxUser) userService.currentUserDetails(); + if ( user != null && !user.getHis1Rights().contains("RIGHT_CS_BIA_ANALYSIS_CREATE_ANALYSIS") ) { + throw new AccessDeniedException("No right to access Create functions"); + } + } + + public boolean isCreateRight() { + SxUser user = (SxUser) userService.currentUserDetails(); + if ( user != null && !user.getHis1Rights().contains("RIGHT_CS_BIA_ANALYSIS_CREATE_ANALYSIS") ) { + return false; + } + return true; + } + + public void checkCreateOrViewRights() { + SxUser user = (SxUser) userService.currentUserDetails(); + if ( user != null && !user.getHis1Rights().contains("RIGHT_CS_BIA_ANALYSIS_CREATE_ANALYSIS") && + !user.getHis1Rights().contains("RIGHT_CS_BIA_ANALYSIS_VIEW_ANALYSIS_TABLE") ) { + throw new AccessDeniedException("No right to access Create and View functions"); + } + } + + public List getSachgebiete(Right... rights) { + List values = getRightParamValues(RightParam.TOPIC_AREA, rights); + List valuesForTopics = dbAdapter.getSachgebieteForFactTables(getRightParamValues(RightParam.TOPIC, rights)); + values.addAll(valuesForTopics); + if (values.isEmpty()) { + return new ArrayList<>(); + } + List sachgebietValues = values.stream() + .map(Integer::parseInt).collect(Collectors.toList()); + return sachgebietValues; + } + + public List getFactTables(Right... rights) { + List values = getRightParamValues(RightParam.TOPIC, rights); + if (values.isEmpty()) { + return new ArrayList<>(); + } + List factsValues = values.stream() + .map(value -> new Identifier(value)).collect(Collectors.toList()); + return factsValues; + } + + public void checkSachgebiet(int sachgebiet, Right... rights) { + List sachgebiete = getSachgebiete(rights); + if (sachgebiete.isEmpty()) { + return; + } else if (!sachgebiete.contains(Integer.valueOf(sachgebiet))){ + throw new AccessDeniedException("No right to access Sachgebiet " + sachgebiet); + } + } + + public void checkFactTable(Identifier fact, Right... rights) { + List factTables = getFactTables(rights); + if (factTables.isEmpty()) { + return; + } else if (!factTables.contains(fact)){ + throw new AccessDeniedException("No right to access Fact Table " + fact.composedId); + } + } + + /** + * Return the values of a RightParam for a given array of Rights. + * @param param RightParam which can be assigned to multiple Rights. + * @param rights Rights which may or may not contain the RightParam. + * @return List of values for the RightParam (first occurrence for multiple Rights). + */ + public List getRightParamValues(RightParam param, Right... rights) { + SxUser user = (SxUser) userService.currentUserDetails(); + if(user == null) { + return new ArrayList<>(); + } + + Map> rightsMap = user.getRightsMap(); + Map rightParamMap = null; + boolean noRights = true; + for (Right right : rights) { + if(rightsMap.containsKey(right.getString())) { + rightParamMap = rightsMap.get(right.getString()); + noRights = false; + if(rightParamMap != null) { + break; + } + } + } + + if(noRights) { + throw new AccessDeniedException("Missing rights: " + Right.getPrintableRights(rights)); + } + + if(rightParamMap == null || rightParamMap.isEmpty()) { + return new ArrayList<>(); + } + + String paramValues = rightParamMap.get(param.getString()); + if (paramValues != null) { + List paramValuesResult = new ArrayList<>(); + for (String string : StringUtils.split(paramValues, ',')) { + paramValuesResult.add(string); + } + return paramValuesResult; + } + return new ArrayList<>(); + } + +} diff --git a/src/de/superx/bianalysis/service/DbMetaAdapter.java b/src/de/superx/bianalysis/service/DbMetaAdapter.java new file mode 100644 index 0000000..c24d5d6 --- /dev/null +++ b/src/de/superx/bianalysis/service/DbMetaAdapter.java @@ -0,0 +1,637 @@ +package de.superx.bianalysis.service; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Service; + +import de.superx.bianalysis.FaultyMetadataException; +import de.superx.bianalysis.ReportDefinition; +import de.superx.bianalysis.ReportMetadata; +import de.superx.bianalysis.StoredReport; +import de.superx.bianalysis.metadata.Identifier; +import de.superx.bianalysis.models.Dimension; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.FactTable; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.Measure; +import de.superx.bianalysis.repository.DimensionAttributeRepository; +import de.superx.bianalysis.repository.DimensionRepository; +import de.superx.bianalysis.repository.FactRepository; +import de.superx.bianalysis.repository.MeasureFilterRepository; +import de.superx.bianalysis.repository.MeasureRepository; +import de.superx.bianalysis.repository.StoredReportRepository; +import de.superx.bianalysis.repository.dto.AttributeDto; +import de.superx.bianalysis.repository.dto.DimensionDto; +import de.superx.bianalysis.repository.dto.FactDto; +import de.superx.bianalysis.repository.dto.MeasureDto; +import de.superx.bianalysis.repository.dto.MeasureFilterDto; +import de.superx.common.NotYetImplementedException; +import de.superx.jdbc.entity.Sachgebiet; +import de.superx.jdbc.entity.Systeminfo; +import de.superx.jdbc.repository.SachgebieteRepository; +import de.superx.jdbc.repository.SysteminfoRepository; + +@Service() +public class DbMetaAdapter implements InitializingBean { + + @Autowired + FactRepository factRepository; + + @Autowired + MeasureRepository measureRepository; + + @Autowired + DimensionRepository dimensionRepository; + + @Autowired + DimensionAttributeRepository dimensionAttrRepo; + + @Autowired + MeasureFilterRepository measureFilterRepo; + + @Autowired + StoredReportRepository storedReportRepository; + + @Autowired + SachgebieteRepository sachgebieterepository; + + @Autowired + SysteminfoRepository systeminfoRepository; + + @Autowired + DataSource dataSource; + + private JdbcTemplate jt; + + @Override + public void afterPropertiesSet() throws Exception { + this.jt = new JdbcTemplate(dataSource); + } + + public List getFactTables(List sachgebieteTids, List facts) throws NotYetImplementedException { + try { + List factTables = this.factRepository.findAll() + .stream() + .filter(f -> getSachgebietForFacttable(f.sachgebiettid.intValue()).tid.intValue() != -1) + .filter(f -> sachgebieteTids.isEmpty() || sachgebieteTids.contains(Integer.valueOf(f.sachgebiettid.intValue()))) + .filter(f -> facts.isEmpty() || facts.contains(f.id) || !hasSachgebietTopicRestrictions(facts, f.sachgebiettid.intValue()) ) + .map(f -> { + FactTable fact = new FactTable(f); + fact.setSachgebiet(getSachgebietForFacttable(f.sachgebiettid.intValue())); + fact.setConformedDimensions(getConformedDimensionsForFacttable(f.id)); + return fact; + }) + .collect(Collectors.toList()); + return factTables; + } catch (Exception e) { + + e.printStackTrace(); + if (e.getCause().getMessage().contains("FEHLER: Relation »metadata.facttable« existiert nicht")) { + throw new NotYetImplementedException("Bitte installieren Sie zuerst die Komponente 'BI-Analyse-Daten' und führen Sie anschließend den Konnektor aus."); + } + throw e; + } + } + + private boolean hasSachgebietTopicRestrictions(List facts, int tid) { + for (Identifier factId : facts) { + if(getFactTable(factId).getSachgebiettid() == tid) { + return true; + } + } + return false; + } + + public List getConformedDimensionsForFacttable(Identifier factId){ + List conformedDimension = new ArrayList<>(); + List ids = dimensionRepository.getUsedConformedDimensionsByFactTable(factId.composedId); + for (Identifier id : ids) { + Dimension dimension = getDimension(id); + List attributes = getAttributesOfDimension(dimension.getId()); + + dimension.setDimensionAttributes(attributes); + for(DimensionAttribute a : attributes) { + a.setDimension(dimension); + } + conformedDimension.add(getDimension(id)); + } + return conformedDimension; + } + + public List getDimensionAttributeMetadata(List attributeIds, Identifier factId) { + + if (attributeIds == null || attributeIds.size() <= 0 ) { + return null; + } + + List result = new ArrayList(); + for (Identifier id : attributeIds) { + Optional optAttribute = dimensionAttrRepo.findById(id); + if(optAttribute.isEmpty()) { + throw new FaultyMetadataException(id, "Attribute"); + } + + DimensionAttribute attr = new DimensionAttribute(optAttribute.get()); + Dimension dim = null; + + if(factId != null) { + // case 1: id is role playing -> rpId is null because the attribute array of the conf dim is not empty + // case 2: id is NOT role playing because the attribute array of the conf dim is empty + Identifier rpId = getRolePlayingDimensionWithNoAttributes(id.composedId, factId.composedId); + if(rpId != null) { + dim = getDimension(rpId); + } + } + + if(dim == null) { + dim = getDimension(attr.getDimensionId()); + } + + attr.setDimension(dim); + + if(attr.getAttrConformedId() != null) { + Optional optAttributeConf = dimensionAttrRepo.findById(new Identifier(attr.getAttrConformedId())); + if(optAttributeConf.isPresent()) { + Dimension dimConf = getDimension(optAttributeConf.get().dimensionId); + attr.setDimConformedId(dimConf.getId().composedId); + } + } + result.add(attr); + } + return result; + } + + public List getMeasureMetadata(List measureIds) { + if (measureIds != null && measureIds.size() > 0 ) { + List result = new ArrayList(); + for (Identifier id : measureIds) { + Measure measure = getMeasure(id); + if(measure.getMeasureFilterId() != null) { + MeasureFilterDto filter = measureFilterRepo.findById(measure.getMeasureFilterId()).get(); + if(filter.dimensionAttributeId != null) { + DimensionAttribute attribute = getDimensionAttributeById(filter.dimensionAttributeId); + Dimension dimension = getDimension(attribute.getDimensionId()); + measure.setMeasureFilterAttributes(filter, attribute, dimension); + } else if(filter.factColumnFilter != null) { + measure.setFactColumnFilter(filter); + } + } + result.add(measure); + } + return result; + } + return null; + } + + + + public List getDimensions(Identifier factTableId) { + List dimensions = new ArrayList<>(); + for (DimensionDto dimensionDto : dimensionRepository.findByFactTableId(factTableId)) { + if(dimensionDto.isHidden != null && dimensionDto.isHidden.booleanValue() == true) { + continue; + } + Dimension dimension = new Dimension(dimensionDto); + dimensions.add(dimension); + List attr = getAttributesOfDimension(dimension.getId()); + dimension.setDimensionAttributes(attr); + if(dimension.getConformed() != null) { + Dimension dimConf = getDimension(new Identifier(dimension.getConformed())); + dimension.conformedCaption = dimConf.getCaption(); + dimension.conformedDescription = dimConf.getDescription(); + if(attr.isEmpty() && !dimension.getConformed().isEmpty()) { + attr = getAttributesOfDimension(new Identifier(dimension.getConformed())); + for( DimensionAttribute a : attr) { + a.setAttrConformedId(a.getStringId()); + a.setHierarchy(dimConf.isHierarchy()); + } + dimension.setDimensionAttributes(attr); + continue; + } + } + for(DimensionAttribute a : attr) { + if(a.getAttrConformedId() != null) { + DimensionAttribute attribute = getDimensionAttributeById(new Identifier(a.getAttrConformedId())); + a.setConformedCaption(attribute.getCaption()); + a.setConformedDescription(attribute.getDescription()); + } + a.setDimension(dimension); + } + } + + return dimensions; + } + + public List getAllowedDimensionAttributes(List ids, List sachgebietTids, List factTables){ + List attributes = new ArrayList<>(); + for (Identifier id : ids) { + DimensionAttribute attr = getDimensionAttributeById(id); + Dimension dim = getDimension(attr.getDimensionId()); + attr.setDimension(dim); + Optional factOpt = factRepository.findById(dim.getId()); + if(factOpt.isPresent()) { + if(!factTables.contains(factOpt.get().id)) { + continue; + } + Integer sachgebiettsTid = Integer.valueOf(factOpt.get().sachgebiettid.intValue()); + if(!sachgebietTids.isEmpty() && !sachgebietTids.contains(sachgebiettsTid)) { + continue; + } + } + attributes.add(attr); + } + return attributes; + } + + public List getDimensionAttributeValues(List attributes, List factTables) { + List result = new ArrayList(); + List tables = new ArrayList(); + for (DimensionAttribute attr : attributes) { + if(tables.contains(attr.getTablename())) { + continue; + } + tables.add(attr.getTablename()); + for(Identifier factId : factTables) { + FactTable fact = getFactTable(factId); + Dimension dim = getDimension(attr.getDimensionId()); + List values = getDimensionAttributeValues(attr, dim, fact); + result.addAll(values); + } + } + return result; + } + + public List> getDimensionAttributeValuesHierarchy(Identifier attribute_id) { + DimensionAttribute attr = getDimensionAttributeById(attribute_id); + Dimension dim = getDimension(attr.getDimensionId()); + return getDimensionAttributeValuesHierarchy(attr.getColumnname(), dim.getTablename()); + } + + public List getFilterMetadata(List filters) { + for (Filter filter : filters) { + DimensionAttribute attr = getDimensionAttributeById(filter.dimensionAttributeId); + Dimension dim = getDimension(attr.getDimensionId()); + attr.setDimension(dim); + filter.setDimension(dim); + filter.setDimensionAttribute(attr); + } + return filters; + } + + public List getDimensionAttributeValues(DimensionAttribute attr, Dimension dim, FactTable factTable) { + String sortOrderColumnName = attr.getSortOrderColumn() != null ? attr.getSortOrderColumn(): attr.getColumnname(); + + String templateSql = + "SELECT DISTINCT d.%s AS value, d.%s, " + + "array_position(array[%s], d.%s::text) " + + "FROM presentation.%s d"; + + if(dim != null + && attr.getFilterSelection() != null + && attr.getFilterSelection().equals("show_existing_only")) { + + templateSql += " INNER JOIN presentation." + factTable.getTablename() + + " f ON d.id = f." + dim.getJoincolumn(); + } + + templateSql += " WHERE d.%s IS NOT NULL "; + + if(dim != null + && attr.getFilterSelection() != null + && attr.getFilterSelection().equals("show_range")) { + + templateSql += " AND d.id BETWEEN (" + + "SELECT MIN(" + dim.getJoincolumn() + ")" + + " FROM presentation." + factTable.getTablename() + + ") AND (" + + "SELECT MAX(" + dim.getJoincolumn() + + " FROM presentation." + factTable.getTablename() + ")"; + } + + // TODO: sometimes we need DESC + templateSql += " ORDER BY 3, 2 ASC;"; + + String query = String.format( + templateSql, + attr.getColumnname(), + sortOrderColumnName, + DimensionAttribute.specialValueListForSql(), + attr.getColumnname(), + attr.getTablename(), + attr.getColumnname() + ); + + List values = jt.query(query, (rs, rowNum) -> rs.getString("value")) + .stream().distinct().collect(Collectors.toList()); + + return values; + } + + public List> getDimensionAttributeValuesHierarchy(String columname, String tablename) { + String query = "select distinct id, parent_id, "+columname+" from presentation." + tablename; + List> values = jt.query(query, + new Object[0], + new RowMapper>() { + @Override + public List mapRow(ResultSet rs, int rowNum) throws SQLException { + List result = new ArrayList<>(); + result.add(Integer.valueOf(rs.getInt("id"))); + result.add(Integer.valueOf(rs.getInt("parent_id"))); + result.add(rs.getString(columname)); + return result; + } + } + ); + + return values; + } + + public DimensionAttribute getDimensionAttributeMetadataById(Identifier id) { + DimensionAttribute attr = getDimensionAttributeById(id); + Dimension dim = getDimension(attr.getDimensionId()); + attr.setDimension(dim); + return attr; + } + + public Sachgebiet getSachgebietById(int sachgebietId) { + return this.sachgebieterepository.findById(Integer.valueOf(sachgebietId)).get(); + } + + public int saveReportDefinition(StoredReport report) { + StoredReport savedStoredReport = this.storedReportRepository.save(report); + return savedStoredReport.id; + } + + public int getBridgeMaxLevel(DimensionAttribute bridgeAttr, ReportMetadata metadata) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + String sql = buildMaxHierarchyLvlSQL(bridgeAttr, metadata.factTable.getTablename(), metadata.getFilterNoHierarchy(), bridgeAttr.getTablename()); + int value = jdbcTemplate.queryForObject(sql, Integer.class).intValue() + 1; + return value; + } + + public int getBridgeMaxLevel(DimensionAttribute bridgeAttr, ReportMetadata metadata, String factTableName) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + String sql = buildMaxHierarchyLvlSQL(bridgeAttr, factTableName, metadata.getFilterNoHierarchy(), bridgeAttr.getTablename()); + int value = jdbcTemplate.queryForObject(sql, Integer.class).intValue() + 1; + return value; + } + + public int getBridgeMinLevel(List filters, int maxLvl, String dimTable) { + if(filters == null || filters.isEmpty() ) { + return 0; + } + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + List filterValues = new ArrayList<>(); + + for (Filter filter : filters) { + filterValues.add("ancestor_" + filter.columnname + "[%s] IN ( " + filter.getValues() + " )"); + } + + for (int i = 0; i < maxLvl; i++) { + String sql = "select count(*) from presentation." + dimTable + "_hierarchy" + + " where "; + StringJoiner filterJoiner = new StringJoiner(" OR "); + + for (String string : filterValues) { + filterJoiner.add(String.format(string, Integer.valueOf(i+1))); + } + + sql += filterJoiner.toString(); + int value = jdbcTemplate.queryForObject(sql, Integer.class).intValue(); + if(value > 0) { + return i; + } + } + return -1; + } + + public boolean isAttributeHierarchyBridge(Identifier leftDimensionAttributeId) { + Optional dimAttrOpt = this.dimensionAttrRepo.findById(leftDimensionAttributeId); + if(dimAttrOpt.isEmpty()) { + return false; + } + Optional dimOpt = this.dimensionRepository.findById(dimAttrOpt.get().dimensionId); + if(dimOpt.isEmpty()) { + return false; + } + + if(dimOpt.get().isHierarchy == null) return false; + return dimOpt.get().isHierarchy.booleanValue(); + } + + public int getColNumbers(List topDimensionAttributes, List filters) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + int num = 1; + for (DimensionAttribute dimensionAttribute : topDimensionAttributes) { + if(filters != null && filters.size() > 0) { + Filter filter = Filter.findFilterById(filters, dimensionAttribute.getId()); + if(filter != null && filter.filterValues.size() > 0) { + num *= filter.filterValues.size(); + continue; + } + } + String sql = String.format("SELECT count(DISTINCT %s) FROM presentation.%s WHERE %s IS NOT NULL", dimensionAttribute.getColumnname(), dimensionAttribute.getTablename(), dimensionAttribute.getColumnname()); + int value = jdbcTemplate.queryForObject(sql, new Object[] {}, Integer.class).intValue(); + num *= value; + } + return num; + } + + /** + * Given a conformed attribute and a facttable this function determines the following: + * + * Does this conformed attribute belong to a role playing dimension with no attributes ? + * + * If it is the case: return the id of the rp dimension + */ + public Identifier getRolePlayingDimensionWithNoAttributes(String attrId, String factId) { + Identifier confDimId = dimensionRepository.findDimensionIdForAttribute(attrId); + System.out.println(attrId); + List preids = dimensionRepository.getRolePlayingIds(confDimId.composedId, factId); + if(preids.size() == 1) { + return preids.get(0); + } else if (preids.size() > 1) { + throw new RuntimeException("Not yet implemented: Can't use conformed dimension " + + "more than once for the same facttable."); + } + return null; + } + + public Identifier checkIfFactTableHasDimensionAttribute(Identifier conformedAttrId, Identifier fact) { + Identifier rpId = getRolePlayingDimensionWithNoAttributes(conformedAttrId.composedId, fact.composedId); + + if(rpId != null) { + return conformedAttrId; + } + List ids = dimensionAttrRepo + .findAttributesByConformedAttributeAndFactTable(conformedAttrId.composedId, fact.composedId); + if (ids.size() == 1) { + return ids.get(0); + } else if (ids.size() > 1) { + throw new FaultyMetadataException("Für die Faktentabelle mit der ID '" + fact.composedId + + "' existieren zwei Role-Playing Dimensions " + "zugehörig zu der Conformed Dimension mit der ID '" + + conformedAttrId.composedId + "'. " + "Dieser Fall ist zurzeit noch nicht umgesetzt."); + } else { + Identifier id = dimensionAttrRepo.findAttributesByIdAndFactTable(conformedAttrId.composedId, fact.composedId); + return id; + } + } + + public boolean checkIfFactTableHasMeasure(Identifier measure, Identifier fact) { + return factRepository.hasFactTableMeasure(fact.composedId, measure.composedId); + } + + public String getLastUpdate(int tid) { + Optional systeminfoOpt = systeminfoRepository.findById(Integer.valueOf(tid)); + if(systeminfoOpt.isPresent()) { + Date lastUpdate = systeminfoOpt.get().datum; + return new SimpleDateFormat("dd.MM.yyyy hh:mm").format(lastUpdate); + } + return "Unknown"; + } + + public String getFactTableNameMaxBridgeLvl(Identifier fact, Identifier attr) { + return factRepository.getFactTableNameForAttribute(fact.composedId, attr.composedId); + } + + public DataSource getDataSource() { + return this.dataSource; + } + + public int getSachgebietByFactTableId(String factTableId) { + Optional fact = this.factRepository.findById(new Identifier(factTableId)); + if(fact.isPresent()) { + return fact.get().sachgebiettid.intValue(); + } + return -1; + } + + public List getSachgebieteForReport(ReportDefinition reportDefinition) { + List result = new ArrayList<>(); + for (Identifier factTableId : reportDefinition.factTableIds) { + int sachgebiet = getSachgebietByFactTableId(factTableId.composedId); + result.add(Integer.valueOf(sachgebiet)); + } + return result; + } + + public List findAllStoredReports(){ + List reports = new ArrayList<>(); + for (StoredReport report : this.storedReportRepository.findAll()) { + StoredReport.setReportDefinitionFromJson(report); + reports.add(report); + } + return reports; + } + + public Optional findById(int id) { + Optional report = storedReportRepository.findById(id); + if(report.isPresent()) { + StoredReport.setReportDefinitionFromJson(report.get()); + } + return report; + } + + public void deleteById(int id) { + storedReportRepository.deleteById(id); + } + + private Sachgebiet getSachgebietForFacttable(int sachgebiettid) { + Optional sachgebiet = sachgebieterepository + .findById(Integer.valueOf(sachgebiettid)); + if(sachgebiet.isPresent()) { + return sachgebiet.get(); + } + return new Sachgebiet(Integer.valueOf(-1), "Unknown", null); + } + + public List getSachgebieteForFactTables(List rightParamValues) { + List result = new ArrayList<>(); + for (String id : rightParamValues) { + int sachgebiet = getSachgebietByFactTableId(id); + if(sachgebiet != -1) { + result.add(String.valueOf(sachgebiet)); + } + } + return result; + } + + public List getAttributesOfDimension(Identifier dimId){ + List attr = dimensionAttrRepo.findByDimensionId(dimId); + List attributes = attr.stream() + .map(a -> new DimensionAttribute(a)) + .filter(a -> !a.isHidden()) + .collect(Collectors.toList()); + return attributes; + } + + public Dimension getDimension(Identifier id) { + Optional dimOpt = dimensionRepository.findById(id); + if(dimOpt.isEmpty()) { + throw new FaultyMetadataException(id, "Dimension"); + } + Dimension dimension = new Dimension(dimOpt.get()); + return dimension; + } + + public Measure getMeasure(Identifier id) { + Optional measureOpt = measureRepository.findById(id); + if(measureOpt.isEmpty()) { + throw new FaultyMetadataException(id, "Measure"); + } + Measure measure = new Measure(measureOpt.get()); + return measure; + } + + public DimensionAttribute getDimensionAttributeById(Identifier dimAttrId) { + Optional optAttr = this.dimensionAttrRepo.findById(dimAttrId); + if(optAttr.isEmpty()) { + throw new FaultyMetadataException(dimAttrId, "Attribute"); + } + return new DimensionAttribute(optAttr.get()); + } + + public List getMeasures(Identifier factTableId) { + List measures = new ArrayList<>(); + for (MeasureDto measureDto : measureRepository.findByFactTableId(factTableId)) { + measures.add(new Measure(measureDto)); + } + return measures; + } + + public FactTable getFactTable(Identifier factTableId) { + Optional optFact = factRepository.findById(factTableId); + if(optFact.isEmpty()) { + throw new FaultyMetadataException(factTableId, "Fact"); + } + return new FactTable(optFact.get()); + } + + /** + * Generates the SQL which returns the max depth for a hierarchy. + * (Unfortunately i can't think of a better way to achieve this without relying on additional SQL) + */ + private static String buildMaxHierarchyLvlSQL(DimensionAttribute bridgeAttr, String factTable, List filters, String dimTab) { + // TODO: filters + String sql = "SELECT MAX(lvl) " + + "FROM presentation."+ factTable + " fw " + + "LEFT JOIN presentation."+dimTab+"_hierarchy h " + + "ON h." + bridgeAttr.getDimIdJoinColumn() +" = fw."+bridgeAttr.getJoincolumn(); + return sql; + } + +} diff --git a/src/de/superx/bianalysis/sqlgeneration/SQLGenerator.java b/src/de/superx/bianalysis/sqlgeneration/SQLGenerator.java new file mode 100644 index 0000000..d85acee --- /dev/null +++ b/src/de/superx/bianalysis/sqlgeneration/SQLGenerator.java @@ -0,0 +1,378 @@ +package de.superx.bianalysis.sqlgeneration; + +import java.util.List; +import java.util.StringJoiner; + +import de.superx.bianalysis.ColumnElement; +import de.superx.bianalysis.ReportMetadata; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.Measure; + +/** + * Lets consider the following example for the SQL Generation: + * + * Dimensions: X, Y, Z with one attribute each + * Attributes + * - X: DA + * - values: DA1, DA1 + * - Y: DB + * - values: DB1, DB1 + * - Z: DC + * - values: DC1, DC1 + * Measures: + * - M1: count on col_a + * - M2: sum on col_b + * + * For the simplest use case (all attributes and measures selected without any + * filters or bridge tables) the generated table would look like this: + * + * +---------+-----------------------+----------------------+ + * | | DA1 | DA2 | + * | |-----------+-----------+-----------+----------+ + * | DC | DB1 | DB2 | DB1 | DB2 | + * | |-----+-----+-----+-----+-----+-----+-----+----+ + * | | M1 | M2 | M1 | M2 | M1 | M2 | M1 | M2 | + * +=========+=====+=====+=====+=====+=====+=====+=====+====+ + * | DC1 | | | | | | | | | + * +---------+-----+-----+-----+-----+-----+-----+-----+----+ + * | DC2 | | | | | | | | | + * +---------+-----+-----+-----+-----+-----+-----+-----+----+ + * + * and the generated SQL would look like this: + * + * SELECT + * DC, + * COUNT(col_a) FILTER (WHERE DA = 'DA1' AND DB = 'DB1') as "col0", + * SUM(col_b) FILTER (WHERE DA = 'DA1' AND DB = 'DB1') as "col1", + * COUNT(col_a) FILTER (WHERE DA = 'DA1' AND DB = 'DB2') as "col2", + * SUM(col_b) FILTER (WHERE DA = 'DA1' AND DB = 'DB2') as "col3", + * COUNT(col_a) FILTER (WHERE DA = 'DA2' AND DB = 'DB1') as "col4", + * SUM(col_b) FILTER (WHERE DA = 'DA2' AND DB = 'DB1') as "col5", + * COUNT(col_a) FILTER (WHERE DA = 'DA2' AND DB = 'DB2') as "col6", + * SUM(col_b) FILTER (WHERE DA = 'DA2' AND DB = 'DB2') as "col7" + * FROM + * presentation.fact_table + * JOIN presentation.dim_a + * ON fact_table.dim_a = dim_a.id + * JOIN presentation.dim_b + * ON fact_table.dim_b = dim_b.id + * JOIN presentation.dim_a + * ON fact_table.dim_c = dim_c.id + * GROUP BY dim_c.DC + * + * + * !! Special Cases: + * + * 1. Filtering Attributes + * User Filtered to see only DA with values DA1. + * In this case the select section would shrink to the following four columns: + * + * COUNT(col_a) FILTER (WHERE DA = 'DA1' AND DB = 'DB1') as "col0", + * SUM(col_b) FILTER (WHERE DA = 'DA1' AND DB = 'DB1') as "col1", + * COUNT(col_a) FILTER (WHERE DA = 'DA1' AND DB = 'DB2') as "col2", + * SUM(col_b) FILTER (WHERE DA = 'DA1' AND DB = 'DB2') as "col3", + * COUNT(col_a) FILTER (WHERE DA = 'DA2' AND DB = 'DB1') as "col4" + * + * and the following where clause would be appended: + * + * WHERE dim_a.DA IN ('DA1') + * + * 2. Measures with Build-In-Filter + * Consider the measure M1 should only count the values DA1 of attribute DA. + * In this case the filter condition is prepended to the selection section + * of the specific columns for this measure: + * + * COUNT(col_a) FILTER (WHERE col_a IN ('DA1') AND DA = 'DA1' AND DB = 'DB1') as "col0", + * ... + * COUNT(col_a) FILTER (WHERE col_a IN ('DA1') AND DA = 'DA1' AND DB = 'DB2') as "col2", + * ... + * COUNT(col_a) FILTER (WHERE col_a IN ('DA1') AND DA = 'DA2' AND DB = 'DB1') as "col4", + * ... + * COUNT(col_a) FILTER (WHERE col_a IN ('DA1') AND DA = 'DA2' AND DB = 'DB2') as "col6", + * ... + * + * The filter condition for a measure can be either an IN or NOT IN condition. + * + */ +public class SQLGenerator { + + public ReportMetadata reportMetadata; + public List columnElements; + public char formatSql = ' '; + private final static String HIERARCHY_MODEL_SUFFIX = "_hierarchy"; + + public SQLGenerator(ReportMetadata reportMetadata, List columnElements) { + this.reportMetadata = reportMetadata; + this.columnElements = columnElements; + } + + public SQLGenerator(ReportMetadata reportMetadata) { + this.reportMetadata = reportMetadata; + } + + public String buildFormattedSqlStatement() { + formatSql = '\n'; + return buildSqlStatement(); + } + + public String buildSqlStatement() { + StringBuilder statement = new StringBuilder(); + statement.append("SELECT "); + statement.append(buildSelectSection()); + statement.append(formatSql + "FROM presentation." + reportMetadata.factTable.getTablename() ); + statement.append(buildJoinSection()); + statement.append(buildFilterSection()); + statement.append(buildGroupBySection()); + statement.append(buildOrderBySection()); + return statement.toString(); + } + + public String buildSelectSection() { + StringJoiner columns = new StringJoiner(", "); + String dimensionAttributesStatement = selectDimensionAttributes(); + if (dimensionAttributesStatement != null && !dimensionAttributesStatement.isBlank() ) { + columns.add(dimensionAttributesStatement); + } + StringJoiner measuresStatementJoiner = new StringJoiner(", "); + columnElements.forEach((columnElement) -> { + measuresStatementJoiner.add(selectMeasure(columnElement)); + }); + String measuresStatement = measuresStatementJoiner.toString(); + if (measuresStatement != null && !measuresStatement.isBlank()) { + columns.add(measuresStatement); + } + return columns.toString(); + } + + + public String selectDimensionAttributes() { + if (reportMetadata.leftDimensionAttributes == null) { + return null; + } + StringJoiner columns = new StringJoiner(", "); + for (DimensionAttribute attribute : reportMetadata.leftDimensionAttributes) { + + String columnName = attribute.getColumnname(); + String tableAlias = attribute.getDimensionTableAlias(); + String columnAlias = attribute.getDimensionColumnAlias(); + + if(attribute.isHierarchy()) { + + // Build select expressions for each hierarchy level (ancestor node), + // assigning aliases col0, col1, etc. + StringBuilder resultBuilder = new StringBuilder(); + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl; i++) { + resultBuilder + .append(attribute.getDimensionTableAlias()) + .append(".ancestor_") + .append(columnName) + .append('[').append(i + 1).append(']') + .append(" AS \"col").append(i).append("\""); + + if (i < reportMetadata.maxBridgeLvl - 1) { + resultBuilder.append(", "); + } + } + columns.add(resultBuilder.toString()); + + } else { + columns.add(String.format("%s.%s AS %s", tableAlias, columnName, columnAlias)); + + String sortOrderColumn = attribute.getSortOrderColumn(); + if (sortOrderColumn != null) { + columns.add(String.format("%s.%s AS %s_%s", + tableAlias, sortOrderColumn, columnAlias, sortOrderColumn)); + } + } + } + return columns.toString(); + } + + public String getMeasureTablePart(String factTableTablename, Measure measure, List dimensionAttributes) { + String result = ""; + String tableCol = factTableTablename + "." + measure.getColumnname(); + if(measure.getAggregationType().equals("sum")) { + result = "SUM(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("count")) { + result = "COUNT(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("distinct-count")) { + result = "COUNT(distinct(" + tableCol + "))"; + } else if (measure.getAggregationType().equals("avg")) { + result = "AVG(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("min")) { + result = "MIN(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("max")) { + result = "MAX(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("std")) { + result = "STDDEV_SAMP(" + tableCol + ")"; + } else if (measure.getAggregationType().equals("var")) { + result = "VAR_SAMP(" + tableCol + ")"; + } + return result; + } + + public String selectMeasure(ColumnElement columnElement) { + String factTableTablename = reportMetadata.factTable.getTablename(); + StringBuilder measureSelect = new StringBuilder(); + Measure measure = columnElement.measure; + measureSelect.append(getMeasureTablePart(factTableTablename, measure, reportMetadata.leftDimensionAttributes));//todo topdimen hinzufügen + if ( measure.filterCondition != null ) { + // if there exists a filter condition for a specific measure, prepend it to the column filter condition + measureSelect.append(formatSql+ "FILTER (WHERE " + measure.filterCondition); + if (columnElement.dimensionAttributeFilter != null) { + measureSelect.append(" AND " + columnElement.dimensionAttributeFilter); + } + measureSelect.append(")"); + } else if (columnElement.dimensionAttributeFilter != null) { + measureSelect.append(formatSql + "FILTER (WHERE " + columnElement.dimensionAttributeFilter + ")"); + } + if (measureSelect.length() != 0) { + measureSelect.append(" AS \"col" + columnElement.columnNumber + "\""); + } + return measureSelect.toString(); + } + + public String buildJoinSection() { + StringBuilder statement = new StringBuilder(); + for (DimensionAttribute attr : reportMetadata.getUniqueDimensionAttributes()) { + + String joinColumn = "id"; + if( attr.getDimIdJoinColumn() != null + && !attr.getDimIdJoinColumn().isBlank()) { + // Hierarchy dimension models must always be joined on an id column. + // See the "hierarchy_dim.sql" dbt macro for implementation details. + // For other models, the default join column can be customized in the metadata JSON files + // using the "id_column" attribute. + joinColumn = attr.getDimIdJoinColumn(); + } + + String dimensionTable = attr.getTablename(); + boolean isTopAttribute = reportMetadata.topDimensionAttributes.contains(attr); + if(attr.isHierarchy() && !isTopAttribute) { + // Hierarchy dimension tables use a dedicated join suffix. + // For example, dim_orgunit is joined as dim_orgunit_hierarchy. + // This hierarchy table contains all node paths in the hierarchy tree. + // For additional details, see the "hierarchy_dim.sql" macro. + dimensionTable += HIERARCHY_MODEL_SUFFIX; + } + + String join = String.format( + " JOIN presentation.%s AS %s ON %s.%s = %s.%s", + dimensionTable, + attr.getDimensionTableAlias(), + reportMetadata.factTable.getTablename(), + attr.getJoincolumn(), + attr.getDimensionTableAlias(), + joinColumn + ); + statement.append(join); + /* TODO userinput for histroical keys: + 1. is_current + 2. last_known + 3. specific date: (ANY_DATE BETWEEN %s.valid_from AND %s.valid_to) + */ + if(attr.isHistorical()) { + String currentFilter = String.format( + " AND %s.is_current = true ", + attr.getDimensionTableAlias() + ); + statement.append(currentFilter); + } + } + return statement.toString(); + } + + public String buildFilterSection() { + if (reportMetadata.filters == null || reportMetadata.filters.size() <= 0) { + return ""; + } + StringBuilder statement = new StringBuilder(" WHERE "); + StringJoiner groups = new StringJoiner(" AND "); + + for (Filter filter : reportMetadata.filters) { + + if(reportMetadata.isHierarchyFilter(filter)) { + StringBuilder resultBuilder = new StringBuilder(); + + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl; i++) { + resultBuilder + .append(filter.joincolumn) + .append(".ancestor_") + .append(filter.columnname) + .append('[').append(i).append("] IN (") + .append(filter.getValues()) + .append(')'); + + if (i < reportMetadata.maxBridgeLvl - 1) { + resultBuilder.append(" OR "); + } + } + + groups.add(resultBuilder.toString()); + + } else { + groups.add(filter.dimensionTableAlias + "." + filter.columnname + " IN (" + filter.getValues() + ")"); + } + } + + statement.append(groups.toString()); + if(groups.length() == 0) { + return ""; + } + return statement.toString(); + } + + public String buildGroupBySection() { + if(reportMetadata.leftDimensionAttributes == null || reportMetadata.leftDimensionAttributes.size() <= 0) { + return ""; + } + StringBuilder statement = new StringBuilder("GROUP BY ROLLUP ("); + StringJoiner groups = new StringJoiner(", "); + for (DimensionAttribute attr : reportMetadata.leftDimensionAttributes) { + if(attr.isHierarchy()) { + // TODO: what is happening here? + int numOfHierarchyAttributes = reportMetadata.getHierarchyAttributes().size(); + for (int i = 0; i < numOfHierarchyAttributes; i++) { + for (int j = 0; j < reportMetadata.maxBridgeLvl; j++) { + if(j < reportMetadata.minBridgeLvl) { + continue; + } + groups.add("col"+(j + (i * reportMetadata.maxBridgeLvl))); + } + } + } else { + groups.add(attr.getDimensionTableAlias() + "." + attr.getColumnname()); + if(attr.getSortOrderColumn() != null) { + groups.add(attr.getDimensionTableAlias() + "." + attr.getSortOrderColumn()); + } + } + } + statement.append(groups.toString()); + if(groups.length() == 0) { + return ""; + } + statement.append(")"); + return formatSql + statement.toString(); + } + + public StringJoiner buildOrderBySection() { + StringJoiner orderCols = new StringJoiner(", ", " ORDER BY ", ""); + orderCols.setEmptyValue(""); + for (DimensionAttribute attr : reportMetadata.leftDimensionAttributes) { + if(attr.isHierarchy()) { + for (int i = reportMetadata.minBridgeLvl; i < reportMetadata.maxBridgeLvl; i++) { + orderCols.add("col" + i); + } + continue; + } + if(attr.getSortOrderColumn() != null) { + orderCols.add(attr.getDimensionTableAlias() + "." + attr.getSortOrderColumn()); + } else { + orderCols.add(attr.getDimensionColumnAlias()); + } + } + return orderCols; + } + +} diff --git a/src/de/superx/bianalysis/sqlgeneration/SQLGeneratorTotals.java b/src/de/superx/bianalysis/sqlgeneration/SQLGeneratorTotals.java new file mode 100644 index 0000000..ffc0255 --- /dev/null +++ b/src/de/superx/bianalysis/sqlgeneration/SQLGeneratorTotals.java @@ -0,0 +1,82 @@ +package de.superx.bianalysis.sqlgeneration; + +import java.util.List; +import java.util.StringJoiner; + +import de.superx.bianalysis.ReportMetadata; +import de.superx.bianalysis.models.DimensionAttribute; +import de.superx.bianalysis.models.Filter; +import de.superx.bianalysis.models.Measure; + +public class SQLGeneratorTotals { + + public SQLGeneratorTotals() {} + + public static String buildTotalsColumnSQL(SQLGenerator generator, int maxBridgeLvlOfReport) { + ReportMetadata metadata = generator.reportMetadata; + StringBuilder statement = new StringBuilder(); + statement.append("SELECT "); + statement.append(buildSelectSectionForTotalsCol(generator, metadata, maxBridgeLvlOfReport)); + statement.append(generator.formatSql + "FROM presentation." + metadata.factTable.getTablename() ); + statement.append(generator.buildJoinSection()); + statement.append(buildFilterSectionForTotalsCol(generator, metadata)); + statement.append(generator.buildGroupBySection()); + return statement.toString(); + } + + private static String buildFilterSectionForTotalsCol(SQLGenerator generator, ReportMetadata metadata) { + if(metadata.topDimensionAttributes.size() == 0 && metadata.filters.size() == 0) { + return ""; + } + StringBuilder where = new StringBuilder(" WHERE "); + StringJoiner groups = new StringJoiner(" AND "); + for (DimensionAttribute attribute : metadata.topDimensionAttributes) { + groups.add(attribute.getDimensionTableAlias() + "." + attribute.getColumnname() + " IN (" + getValues(attribute.getDimensionAttributeValues()) + ")"); + } + for (Filter filter : generator.reportMetadata.filters) { + groups.add(filter.dimensionTableAlias + "." + filter.columnname + " IN (" + filter.getValues() + ")"); + } + where.append(groups); + return where.toString(); + } + + private static String buildSelectSectionForTotalsCol(SQLGenerator generator, ReportMetadata metadata, int maxBridgeLvlOfReport) { + StringJoiner columns = new StringJoiner(", "); + columns.add(generator.selectDimensionAttributes()); + //columns.add(metadata.aggregationLvl + " AS aggregationLvl"); + int numCols = generator.reportMetadata.maxBridgeLvl; + for (Measure measure : metadata.measures) { + String value = generator.getMeasureTablePart(metadata.factTable.getTablename(), measure, metadata.leftDimensionAttributes); + if ( measure.filterCondition != null ) { + value += " FILTER (WHERE " + measure.filterCondition +")"; + } + value += " AS \"col" + (numCols++) + "\""; + columns.add(value); + } + String selectSection = columns.toString(); + return selectSection; + } + + private static String getValues(List values) { + if(values == null || values.isEmpty()) { + return null; + } + StringJoiner joiner = new StringJoiner(", "); + for (String value : values) { + joiner.add("'"+value+"'"); + } + return joiner.toString(); + } + + public static String generateTotalsColumnSQL(ReportMetadata metadata) { + if(metadata.topDimensionAttributes.isEmpty()) { + return ""; + } + + SQLGenerator generator = new SQLGenerator(metadata); + String finalSQL = buildTotalsColumnSQL(generator, metadata.maxBridgeLvl) + + generator.buildOrderBySection(); + return finalSQL; + } + +} diff --git a/src/de/superx/bin/AbstractWebserviceClient.java b/src/de/superx/bin/AbstractWebserviceClient.java index 17b98e2..41c8817 100644 --- a/src/de/superx/bin/AbstractWebserviceClient.java +++ b/src/de/superx/bin/AbstractWebserviceClient.java @@ -2,16 +2,27 @@ package de.superx.bin; import java.io.BufferedReader; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; +import java.io.StringReader; import java.io.StringWriter; import java.net.Authenticator; import java.net.PasswordAuthentication; -import java.net.Authenticator.RequestorType; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.logging.FileHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; +import java.util.regex.Pattern; import javax.xml.soap.MessageFactory; import javax.xml.soap.MimeHeaders; @@ -19,25 +30,55 @@ import javax.xml.soap.SOAPConnection; import javax.xml.soap.SOAPConnectionFactory; import javax.xml.soap.SOAPException; import javax.xml.soap.SOAPMessage; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.TransformerFactoryConfigurationError; -import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.xml.sax.SAXException; import de.memtext.util.DateUtils; import de.memtext.util.StringUtils; import de.memtext.util.XMLUtils; +import de.superx.util.SqlStringUtils; public class AbstractWebserviceClient { int pause = 0; boolean isDeleteTmpXmlFileWanted = true; - - public AbstractWebserviceClient() { - + private XMLInputFactory factory = XMLInputFactory.newInstance(); + protected String auth = null; + protected final static String DATUMSPATTERN = "(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)"; + protected static final Logger logger = Logger.getLogger("wc"); + // String test="PVC-U T-Stück 90°, 3fach Klebemu��e"; + private static final Pattern INVALID_XML_PATTERN=Pattern.compile("[^\\u0009\\u000A\\u000D\\u0020-\\uD7FF\\uE000-\\uFFFD\\u10000-\\u10FFF]+"); + private static final String XSLT_REMOVE_NAMESPACE = "\n" + + " \n" + + " \n" + + " \n" + // Neuen Namen ohne Namespace erstellen + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + private Transformer transformerRemoveNamespace; + + public AbstractWebserviceClient() { + String a = System.getProperty("SOAP-AUTH"); + + if (a != null && !a.equals("")) + auth = a; Authenticator.setDefault(new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { @@ -46,21 +87,44 @@ public class AbstractWebserviceClient { String host = System.getProperty(prot + ".proxyHost", ""); String port = System.getProperty(prot + ".proxyPort", ""); String user = System.getProperty(prot + ".proxyUser", ""); - String password = System.getProperty(prot - + ".proxyPassword", ""); + String password = System.getProperty(prot + ".proxyPassword", ""); - if (getRequestingHost().toLowerCase().equals( - host.toLowerCase())) { + if (getRequestingHost().toLowerCase().equals(host.toLowerCase())) { if (Integer.parseInt(port) == getRequestingPort()) { // Seems to be OK. - return new PasswordAuthentication(user, password - .toCharArray()); + return new PasswordAuthentication(user, password.toCharArray()); } } } return null; } }); + + } + + protected static String adaptDatePattern(String input) { + input=input.replaceAll("0000-00-00",""); + + return input.replaceAll(DATUMSPATTERN, "$3.$2.$1"); + } + + /** + * Macht Probleme + * wird zu Leerzeichen/WhiteSpace -> Probleme beim CSV-Import im Copy + * Liefert Formatiertes Ergebnis wird durch Ersetzung + * zu Der 2. regex ist für den SEHR + * unwahrscheinlichen Fall, dass innerhalb eines Eintrags selbst als Fehleingabe + * ein >< vorkommen würde, der überflüssige, bei CSV evtl Fehler verursachende + * Newline wird wieder entfernt. z.B. Kauf von XY >< zum + * Rabattpreis + * + + * @param input + * @return formatted String + */ + protected String format(String input) { + // return input.replaceAll("><", ">\n<").replaceAll("(<.*>\n)(.*>)\n(<.*)(\n)", "$1$2$3$4"); + return input.replaceAll(" 0) { Thread.sleep(pause * 1000); - // System.out.println("pause "+pause); } - SOAPConnectionFactory soapConnectionFactory = SOAPConnectionFactory - .newInstance(); - SOAPConnection soapConnection = soapConnectionFactory - .createConnection(); + SOAPConnectionFactory soapConnectionFactory = SOAPConnectionFactory.newInstance(); + SOAPConnection soapConnection = soapConnectionFactory.createConnection(); SOAPMessage sr = getSoapMessageFromString(soapxml); - sr.getMimeHeaders().addHeader("SOAPAction", - "http://sap.com/xi/WebService/soap1.1"); + sr.getMimeHeaders().addHeader("SOAPAction", "http://sap.com/xi/WebService/soap1.1"); + addAuthentification(sr); SOAPMessage soapResponse = soapConnection.call(sr, url); - // soapResponse.writeTo(System.out); - - Transformer transformer = TransformerFactory.newInstance() - .newTransformer(); - // optional indenting - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty( - "{http://xml.apache.org/xslt}indent-amount", "2"); - - // File tmpF = File.createTempFile("soapdata", ".xml"); - transformer.transform( - new DOMSource(soapResponse.getSOAPPart()), - new StreamResult(sw)); + result.append(extractString(soapResponse)); soapConnection.close(); System.gc(); - // if (attempt < 3) throw new RuntimeException("RUntime"); - // StringBuffer result=readFile(tmpF); - // if (isDeleteTmpXmlFileWanted) tmpF.delete(); - // else System.out.println(" tmp File "+tmpF); + if (result.indexOf("") > -1) + throw new IllegalStateException("Webservice meldet Fehler - kein Netzwerkproblem:\n" + result); + allDone = true; } catch (Exception e) { exception = e; - System.out - .println(DateUtils.getNowString() - + " Aufruf fehlgeschlagen (s. WebserviceLog) - versuche erneut"); + System.out.println( + DateUtils.getNowString() + " Aufruf fehlgeschlagen (s. WebserviceLog) - versuche erneut"); StringWriter swException = new StringWriter(); PrintWriter pw = new PrintWriter(swException); e.printStackTrace(pw); - Logger.getLogger("wc").severe( - " Problem bei Aufruf von " + url + "\n" + soapxml - + "\n" + swException.toString()); + Logger.getLogger("wc") + .severe(" Problem bei Aufruf von " + url + "\n" + soapxml + "\n" + swException.toString()); attempt++; } } if (allDone == false) throw exception; - StringBuffer result = new StringBuffer(sw.toString()); + + return result; + } + + protected boolean isReplyOk(StringBuffer data) throws SAXException, IOException { + boolean result = false; + Document resultDocument = XMLUtils.buildDocumentFromString(data, false); + if (XMLUtils.hasANodeWithName(resultDocument, "TYPE")) { + Node type = XMLUtils.getFirstNode(resultDocument, "TYPE"); + if (XMLUtils.getTheValue(type).equals("I")) + result = true; + } return result; } - protected File createSoapFile(String soapxml, String url) throws Exception { + private String extractString(SOAPMessage soapResponse) throws SOAPException, IOException, TransformerException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + soapResponse.writeTo(out); + String strMsg = new String(out.toByteArray()); + //replace ungültige XML-Hex-Entities im High Surrogate Bereich #324507 + strMsg=replaceHighSurrogates(strMsg); + StringWriter writer = new StringWriter(); + if (transformerRemoveNamespace==null) + { + initTransformerNamespace(); + } + transformerRemoveNamespace.transform(new StreamSource(new StringReader(strMsg)), new StreamResult(writer)); + //Transformer macht zwar Einrücken/indent wird aber unten bei purge wegen optionellem Auftreten in Buchungskommentaren wieder entfernt + strMsg= writer.toString(); + + return strMsg; + } + + private void initTransformerNamespace() throws TransformerConfigurationException, TransformerFactoryConfigurationError { + transformerRemoveNamespace=TransformerFactory.newInstance().newTransformer((new StreamSource(new StringReader(XSLT_REMOVE_NAMESPACE)))); + transformerRemoveNamespace.setOutputProperty(OutputKeys.INDENT, "yes"); + transformerRemoveNamespace.setOutputProperty("{http://xml.apache.org/xslt}indent-amount","2"); +//ident wird zwar gemacht, aber da \n in + + } + + protected File createSoapFile(String soapxml, String url) throws Exception { int attempt = 1; boolean allDone = false; + // Logging von Saaj abschalten + // vergl. + // http://www.docjar.com/html/api/com/sun/xml/internal/messaging/saaj/client/p2p/HttpSOAPConnection.java.html + // Logger.getLogger(LogDomainConstants.HTTP_CONN_DOMAIN, + // "com.sun.xml.internal.messaging.saaj.client.p2p.LocalStrings").setLevel(Level.OFF); + // Die Konstante wird bei Build nicht gefunden, entspricht aber einem festen + // String + // System.out.println(LogDomainConstants.HTTP_CONN_DOMAIN); + // in OpenJDK und geergänzenden jars nicht mehr aktiv + // sicherheitshalber + try { + Logger.getLogger("com.sun.xml.internal.messaging.saaj.client.p2p", + "com.sun.xml.internal.messaging.saaj.client.p2p.LocalStrings").setLevel(Level.OFF); + } catch (Exception e) { + } Exception exception = null; File tmpF = File.createTempFile("soapdata", ".xml"); - while (allDone == false && attempt < 4) { + while (allDone == false && attempt < 5) { try { if (tmpF.exists()) tmpF.delete(); @@ -153,45 +250,33 @@ public class AbstractWebserviceClient { Thread.sleep(pause * 1000); // System.out.println("pause "+pause); } - SOAPConnectionFactory soapConnectionFactory = SOAPConnectionFactory - .newInstance(); - SOAPConnection soapConnection = soapConnectionFactory - .createConnection(); + SOAPConnectionFactory soapConnectionFactory = SOAPConnectionFactory.newInstance(); + SOAPConnection soapConnection = soapConnectionFactory.createConnection(); SOAPMessage sr = getSoapMessageFromString(soapxml); - sr.getMimeHeaders().addHeader("SOAPAction", - "http://sap.com/xi/WebService/soap1.1"); + sr.getMimeHeaders().addHeader("SOAPAction", "http://sap.com/xi/WebService/soap1.1"); + addAuthentification(sr); SOAPMessage soapResponse = soapConnection.call(sr, url); - Transformer transformer = TransformerFactory.newInstance() - .newTransformer(); - // optional indenting - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty( - "{http://xml.apache.org/xslt}indent-amount", "2"); - - transformer.transform( - new DOMSource(soapResponse.getSOAPPart()), - new StreamResult(tmpF)); + String strMsg = extractString(soapResponse); + Files.writeString(tmpF.toPath(), strMsg, StandardOpenOption.CREATE); soapConnection.close(); System.gc(); - // if (attempt<3) throw new RuntimeException("runtime"); + allDone = true; } catch (Exception e) { exception = e; - System.out - .println("\n" - + DateUtils.getNowString() - + " Aufruf fehlgeschlagen (s. WebserviceLog) - versuche erneut"); + System.out.println("\n" + DateUtils.getNowString() + + " Aufruf fehlgeschlagen (s. WebserviceLog) - versuche erneut"); StringWriter swException = new StringWriter(); PrintWriter pw = new PrintWriter(swException); e.printStackTrace(pw); - Logger.getLogger("wc").severe( - " Problem bei Aufruf von " + url + "\n" + soapxml - + "\n" + swException.toString()); + Logger.getLogger("wc").severe(" Problem bei Webservice Abruf\n" + url + "\n" + soapxml + + swException.toString() + + (attempt < 4 ? " versuche erneut:\n" : " zu viele Fehlschlaege - gebe frustriert auf")); attempt++; } @@ -201,35 +286,140 @@ public class AbstractWebserviceClient { return tmpF; } - private SOAPMessage getSoapMessageFromString(String xml) - throws SOAPException, IOException { + protected SOAPMessage getSoapMessageFromString(String xml) throws SOAPException, IOException { MessageFactory factory = MessageFactory.newInstance(); - SOAPMessage message = factory - .createMessage( - new MimeHeaders(), - new ByteArrayInputStream(xml.getBytes(Charset - .forName("UTF-8")))); + SOAPMessage message = factory.createMessage(new MimeHeaders(), + new ByteArrayInputStream(xml.getBytes(Charset.forName("UTF-8")))); return message; } + protected String removeXmlHeader(String input) { + return input.replaceAll("<\\?xml.*\\?>", ""); + } + /** + * High Surrogate Bereich durch ? ersetzen + * @param input + * @return + */ + public static String replaceHighSurrogates(String input) + { + StringBuffer buf=new StringBuffer(input); + final int codePointStringBegin = 3; + final boolean hex = true; + int pos = buf.indexOf("&#x"); + String replacer="?"; + while (pos > -1) { + int posSemikolon = buf.indexOf(";", pos); + if (posSemikolon > -1) { + String encodedString = buf.substring(pos, posSemikolon + 1); + // Hex-Wert bis zum Semikolon + String encodedHex = encodedString.substring(codePointStringBegin, encodedString.length() - 1); + //System.out.println("replace " + encodedHex); + try { + int decimalNumber = Integer.parseInt(encodedHex, hex ? 16 : 10); + if (decimalNumber>=55296) // high surrogate Bereich ersetzen + { + buf.replace(pos, posSemikolon+1, replacer); + //replacer = new String(Character.toChars(decimalNumber)); + } + } catch (NumberFormatException e) { + + } + + } + else + { + //Sicherheitshalber aus while Schleife aussteigen, falls nach &#x + // kein darauf folgendes Semikolon gefunden wurde + break; + } + pos = buf.indexOf("&#x",pos+1); + } + return buf.toString(); + } + + /** - * Entferne unnötige Zeichen und EX_JEST Block + * Entferne unnötige Zeichen und EX_JEST Block falls gewünscht * * @param data + * @param removeJest * @return */ - static String purge(String data) { + static String purge(String data, boolean removeJest) { data = XMLUtils.removeTroublesomeCharacters(data); data = data.replaceAll("\u20AC", "EUR"); - - data = data.replaceAll("\u2013", "-");// dash - - data = data.replaceAll("\u2026", "...");// ... + data = data.replaceAll("€", "EUR"); + data = data.replaceAll("€", "EUR"); + // dash + data = data.replaceAll("\u2013", "-"); + data = data.replaceAll("–", "-"); + // horizontal elipse ... + data = data.replaceAll("\u2026", "..."); + data = data.replaceAll("…", "..."); + // großes scharfes ß durch kleines ersetzen #324507 + data = data.replaceAll("\u1e9e", "ß"); + data = data.replaceAll("ẞ", "ß"); + // MB 30.3.2022 Zeilenumbrüche,Tabs,typogr. Apostroph entfernen HS Karlsruhe + data = data.replace("´","'"); + + + + data = data.replaceAll("\t"," "); + data = data.replaceAll("\r\n"," "); + data = data.replaceAll("\n"," "); + data = StringUtils.decodeHexEncodingInString(data); + // typographische durch einfache Anführungszeichen #317130 + data=StringUtils.removeTypographischeZeichen(data); + + // bei ISO-Codierung Ligaturen wie Dach auf c entfernen, da diese teils damit + // nicht dargestellt werden können + // vergl. https://en.wikipedia.org/wiki/%C4%8C oder + // https://www.codetable.net/decimal/269 ) + if (!SqlStringUtils.getEncoding().contentEquals("UTF-8")) { + data = StringUtils.removeAllButUmlauts(data); + // MB weitere Problematische Zeichen + data = data.replaceAll("¼","1/4"); + data = data.replaceAll("½","1/2"); + data = data.replaceAll("¾","3/4"); + data = replaceInvalidChars(data); + } data = data.replace('^', ' '); - data = data.replaceAll("(?s).*", ""); - data = data.replaceAll("", ""); - data = data.replaceAll("", ""); + // #324507 Entfernen von problematischen Zeichen + //Teststring am Ende xDCB0/1 : PVC-U T-Stück 90°, 3fach Klebemu��e"; + data=INVALID_XML_PATTERN.matcher(data).replaceAll("?"); + + + if (removeJest) { + data = purge(data, "EX_JEST"); + } + return data; + } + + private static String replaceInvalidChars(String input) { + StringBuilder result = new StringBuilder(); + + for (char c : input.toCharArray()) { + if (isValidISO88591Char(c)) { + result.append(c); + } else { + result.append('?'); + } + } + + return result.toString(); + } + + private static boolean isValidISO88591Char(char c) { + // ISO-8859-1 enthält Zeichen von 0x00 bis 0xFF + return c >= 0x00 && c <= 0xFF; + } + + static String purge(String data, String nodename) { + data = data.replaceAll("<" + nodename + ">(?s).*", ""); + data = data.replaceAll("<" + nodename + ">", ""); + data = data.replaceAll("<" + nodename + "/>", ""); return data; } @@ -240,18 +430,79 @@ public class AbstractWebserviceClient { return result; } - private StringBuffer readFile(File f) throws IOException { + protected void addAuthentification(SOAPMessage sr) { + if (auth != null) { + sr.getMimeHeaders().addHeader("Authorization", "Basic " + auth); + } + } + + protected StringBuffer readFile(File f) throws IOException { FileReader fr = new FileReader(f); BufferedReader bfr = new BufferedReader(fr); String line; StringBuffer result = new StringBuffer(); while ((line = bfr.readLine()) != null) { result.append(line + "\n"); - } bfr.close(); fr.close(); return result; } - + + protected boolean isReplyOk(File f) throws XMLStreamException, IOException { + InputStream in = new FileInputStream(f); + XMLStreamReader parser = factory.createXMLStreamReader(in); + boolean checkReturn = false; + String result = ""; + while (parser.hasNext()) { + switch (parser.getEventType()) { + case XMLStreamConstants.START_ELEMENT: + if (parser.getLocalName().equals("RETURN")) { + checkReturn = true; + + } + break; + + case XMLStreamConstants.CHARACTERS: + if (checkReturn && !parser.isWhiteSpace()) { + result = parser.getText(); + checkReturn = false; + } + break; + case XMLStreamConstants.END_DOCUMENT: + + parser.close(); + break; + + } + + parser.next(); + + } + in.close(); + return result.equals("I"); + } + + protected void initLogging() throws SecurityException, IOException { + //hier bewusst separates java.util.Logging für Shellaufrufe + Handler handler = new FileHandler("WebserviceClient.log",100000*1024,1,true); + handler.setFormatter(new SimpleFormatter() + { + @Override + public String format(LogRecord l) { + String result = + DateUtils.getTodayString() + " " + DateUtils.getNowString() + + ":" + l.getMessage() + "\n"; + return result; + } + @Override + public String formatMessage(LogRecord l) { + return format(l); + } + + }); + Logger.getLogger("wc").addHandler(handler); + logger.setLevel(Level.FINEST); + } + } \ No newline at end of file diff --git a/src/de/superx/bin/ComponentAdminCLI.java b/src/de/superx/bin/ComponentAdminCLI.java new file mode 100644 index 0000000..c51dd74 --- /dev/null +++ b/src/de/superx/bin/ComponentAdminCLI.java @@ -0,0 +1,510 @@ +package de.superx.bin; + +import static de.superx.servlet.SxSQL_Server.DEFAULT_MANDANTEN_ID; + +import java.io.File; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import javax.sql.DataSource; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.springframework.batch.core.ExitStatus; +import org.springframework.beans.BeansException; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import de.superx.job.ContainerNode; +import de.superx.rest.EtlJobApi; +import de.superx.rest.model.job.Component; +import de.superx.rest.model.job.JobExecutionStatus; +import de.superx.rest.model.job.StepExecutionStatus; +import de.superx.servlet.SuperXManager; +import de.superx.servlet.SxPools; +import de.superx.spring.HisInOneConfiguration; +import de.superx.spring.batch.His1DataSources; +import de.superx.spring.cli.config.CLIConfig; +import de.superx.spring.config.BatchConfig; +import de.superx.spring.config.DataJdbcConfiguration; +import de.superx.spring.config.ServiceConfig; +import de.superx.spring.service.BatchJobDescriptionAdapter; +import de.superx.spring.service.EntityJobDescriptionSource; +import de.superx.util.PathAndFileUtils; + +/*** + * This class provides functionality to run the component actions of the + * 'modernized component administration' + * of the HISinOne-BI via command line. That includes + * - listing ETL jobs + * - installing components + * - updating components + * - deinstalling components + * - running "Hauptladeroutine" ETL jobs + * - running "Unterladeroutine" ETL jobs + * The class is just meant to be a frontend, so it uses the same implementation as the web application + * (which happens to be the "EtlJobApi" bean) + * However, there are to be some things to be considered: + * - The "jobLauncher" bean from the "BatchConfig" class works asynchronously, which is fine for the + * web application context, but in the CLI context, it is overridden by a synchronous implementation + * (in fact the tweaks to get the configurations ready for the CLI context, are placed in the + * "CLIConfig" class). + * - In the web application context, the HISinOne-BI code in the HISinOne web application writes the + * database configuration to a file, which is then consumed by the "superx" application (the actual + * HISinOne-BI application). The command line application needs this file too, so before using it, you + * should start the web application one time, in order to have the file in place. + * @author witt + * + */ +public class ComponentAdminCLI { + + private static GenericApplicationContext APPLICATION_CONTEXT = null; + + private static String HELP_STRING = "Use this tool to run component actions via command line. " + + "It needs the config file 'his1_databases.properties' inside the classpath; " + + "this file gets written automatically when starting the web application."; + + private static boolean FILESYSTEM = false; + + static Logger logger = Logger.getLogger(ComponentAdminCLI.class); + + public static void main(String[] args) { + BasicConfigurator.configure(); // initializes console logging to stdout + System.setProperty(SuperXManager.SUPER_X_HISINONE_VERSION, "non-empty-value"); + Options options = createOptions(); + CommandLine parsedArgs = parseArgs(args, options); + logOptions(parsedArgs); + if (parsedArgs.hasOption("h")) { + printHelp(options); + System.exit(0); + } + if(parsedArgs.hasOption("f")) { + FILESYSTEM = true; + } + initSuperXManager(); + if(parsedArgs.hasOption("s")) { + initTablesForEmptyDB(); + } + if(parsedArgs.hasOption("lg")) { + setBatchLoggerToOneFile(); + } + SuperXManager.initKettleEnv(createContext()); + if (parsedArgs.hasOption("la")) { + printAllJobs(); + } else if (parsedArgs.hasOption("li")) { + printInstallableJobs(); + } else if (parsedArgs.hasOption("le")) { + printEtlJobs(); + } else if (parsedArgs.hasOption("i")) { + installComponents(parsedArgs); + } else if (parsedArgs.hasOption("d")) { + deinstallComponent(parsedArgs); + } else if (parsedArgs.hasOption("u")) { + upgradeComponents(parsedArgs); + } else if (parsedArgs.hasOption("ua")) { + upgradeAll(); + } else if (parsedArgs.hasOption("e")) { + etlJobs(parsedArgs); + } else if (parsedArgs.hasOption("r")) { + reloadModule(); + } else if (parsedArgs.hasOption("if")) { + installFunctions(parsedArgs); + } else { + printHelp(options); + } + } + + private static void logOptions(CommandLine parsedArgs) { + logger.info("Starting with the following options:"); + for (Option opt : parsedArgs.getOptions()) { + logger.info(opt); + if(opt.getValues() != null && opt.getValues().length > 0) { + logger.info("Values for option " + opt.getOpt() + ": " + Arrays.asList(opt.getValues())); + } + } + } + + private static void initTablesForEmptyDB() { + try { + GenericApplicationContext context = createContext(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + ContainerNode root = EntityJobDescriptionSource.getPreKernInstallJob(); + componentApi.executeJob("eduetl", root); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static void setBatchLoggerToOneFile() { + try { + GenericApplicationContext context = createContext(); + BatchJobDescriptionAdapter bjda = context.getBean(BatchJobDescriptionAdapter.class); + bjda.setLogJobToFile(false); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static void installComponents(CommandLine parsedArgs) { + String[] components = parsedArgs.getOptionValues("i"); + String currentComp = null; + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + for (String comp : components) { + try { + currentComp = comp; + Long jobStartStatus = Long.valueOf(-1); + if(!FILESYSTEM) { + logger.info("EXECUTING: install job from database"); + jobStartStatus = componentApi.executeInstall(comp); + } else { + // disable mondrian step + logger.info("EXECUTING: install job from filesystem for " + currentComp); + jobStartStatus = componentApi.executeInstallForQAMuster(comp); + logger.info("EXECUTING: workaround upgrade job from filesystem for " + currentComp); + jobStartStatus = componentApi.executeUpgradeForQAMuster(comp); + } + handleStartResult(jobStartStatus, componentApi); + if(comp.equals("kern") && FILESYSTEM) { + DataSource dataSource = context.getBean(His1DataSources.class).get("eduetl"); + SuperXManager.setWebInfFilePath(dataSource); + } + } catch (Exception e) { + logger.error("ERROR installing component " + comp, e); + } + } + } catch (BeansException be) { + handleBeansException(be); + } catch (Exception e) { + handleJobException(e, currentComp); + } + } + + private static void reloadModule() { + try (GenericApplicationContext context = createContext()){ + initSxPools(); + initSuperXManager(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + componentApi.writeJobsToDb(); + } + } + + private static void printHelp(Options options) { + HelpFormatter help = new HelpFormatter(); + help.printHelp(HELP_STRING, options); + } + + private static void printInstallableJobs() { + try (GenericApplicationContext context = createContext()) { + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + List installJobs = componentApi.getInstallJobs(); + for (Component comp : installJobs) { + printDetails(comp); + System.out.println(); + } + } + } + + private static void printEtlJobs() { + try (GenericApplicationContext context = createContext()) { + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + List etlJobs = componentApi.getEtlJobs(); + for (Component etlJob : etlJobs) { + printDetails(etlJob); + System.out.println(); + } + } + } + + private static void printDetails(Component comp) { + System.out.println("abbrev: " + comp.getAbbreviation()); + System.out.println("name: " + comp.getName()); + System.out.println("database: " + comp.getDatabase()); + System.out.println("systeminfo_id: " + comp.getSysteminfoId()); + System.out.println("installed: " + comp.isInstalled()); + } + + private static void deinstallComponent(CommandLine parsedArgs) { + String comp = parsedArgs.getOptionValue("d"); + System.out.println(comp); + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + Long jobStartStatus = componentApi.executeUninstall(comp); + handleStartResult(jobStartStatus, componentApi); + } catch (BeansException be) { + handleBeansException(be); + } catch (Exception e) { + handleJobException(e, comp); + } + } + + private static void upgradeComponents(CommandLine parsedArgs) { + String[] components = parsedArgs.getOptionValues("u"); + String currentComp = null; + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + for (String comp : components) { + currentComp = comp; + Long jobStartStatus = componentApi.executeUpgrade(comp); + if (jobStartStatus.longValue() == -1) { + logger.warn(comp + " not installed. Skipping upgrade."); + continue; + } + handleStartResult(jobStartStatus, componentApi); + } + } catch (BeansException be) { + handleBeansException(be); + } catch (Exception e) { + handleJobException(e, currentComp); + } + } + + private static void installFunctions(CommandLine parsedArgs) { + String[] components = parsedArgs.getOptionValues("if"); + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + // No components given: install + // functions for all installed components + if (components.length == 1 && "all".equals(components[0])) { + List installedComponents = componentApi.getInstallJobs(); + components = installedComponents.stream().map( + c -> c.getAbbreviation()).collect(Collectors.toList()).toArray(new String[] {}); + } + boolean exitFailure = false; + for (String comp : components) { + componentApi.installModuleFunctions(comp); + } + if(exitFailure) { + System.out.println(("Beim Ausführen einer Aktion ist ein Fehler aufgetreten:")); + System.exit(1); + } + } catch (BeansException be) { + handleBeansException(be); + } + } + + private static void upgradeAll() { + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + Long jobStartStatus = componentApi.executeUpgradeAll(); + handleStartResult(jobStartStatus, componentApi); + } catch (BeansException be) { + handleBeansException(be); + } catch (Exception e) { + handleJobException(e, null); + } + } + + private static void etlJobs(CommandLine parsedArgs) { + String[] jobIds = parsedArgs.getOptionValues("e"); + String currentJobId = null; + try (GenericApplicationContext context = createContext()) { + initSxPools(); + EtlJobApi componentApi = context.getBean(EtlJobApi.class); + Long jobStartStatus; + for (String jobId : jobIds) { + try { + currentJobId = jobId; + if (isHauptladeroutine(jobId, componentApi)) { + jobStartStatus = componentApi.complete(jobId); + } else if (isLoadTransform(jobId, componentApi)) { + jobStartStatus = componentApi.load(jobId); + } else { + jobStartStatus = componentApi.executeJob(null, jobId); + } + handleStartResult(jobStartStatus, componentApi); + } catch (Exception e) { + logger.error("ERROR executing job " + jobId, e); + } + } + } catch (BeansException be) { + handleBeansException(be); + } catch (Exception e) { + handleJobException(e, currentJobId); + } + } + + private static void handleStartResult(Long jobStartStatus, EtlJobApi componentApi) { + if (jobStartStatus.intValue() == -1) { + System.err.println("Aktion konnte nicht gestartet werden: Es läuft bereits eine Aktion"); + } + try { + JobExecutionStatus es = componentApi.getStatus(jobStartStatus); + if ("FAILED".equals(es.exitStatus.getExitCode())) { + EtlJobApi.outputErrorSummary(es, System.err); + } + } catch (Exception e) { + System.err.println(("Beim Ausführen der Aktion ist ein Fehler aufgetreten:")); + e.printStackTrace(); + } + } + + private static void handleJobException(Exception e, String jobName) { + System.err.println("error while executing the job '" + jobName + "'"); + e.printStackTrace(); + } + + private static void handleBeansException(BeansException be) { + System.err.println("configuration error or error with resolving the bean '" + EtlJobApi.class.getCanonicalName() + "'"); + be.printStackTrace(); + } + + private static boolean isHauptladeroutine(String comp, EtlJobApi etlJob) { + List installJobs = etlJob.getEtlJobs(); + for (Component comp_meta : installJobs) { + if (comp_meta != null && comp_meta.getAbbreviation().equals(comp) && comp_meta.isDatabaseConnected()) { + return true; + } + } + return false; + } + + private static boolean isLoadTransform(String comp, EtlJobApi etlJob) { + List installJobs = etlJob.getEtlJobs(); + for (Component comp_meta : installJobs) { + if (comp_meta != null && comp_meta.getAbbreviation().equals(comp) && !comp_meta.isDatabaseConnected()) { + return true; + } + } + return false; + } + + private static void printAllJobs() { + try (GenericApplicationContext context = createContext()) { + EtlJobApi etlJob = context.getBean(EtlJobApi.class); + List allJobs = etlJob.getAllJobs(); + for (ContainerNode cn : allJobs) { + printDetails(cn); + System.out.println(); + } + } + } + + private static GenericApplicationContext createContext() { + /* + * https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/context/annotation/AnnotationConfigApplicationContext.html + * quote: + * "In case of multiple @Configuration classes, @Bean methods defined in later classes will override those defined in earlier classes. + * This can be leveraged to deliberately override certain bean definitions via an extra @Configuration class." + * - so it's alright to override some beans via "CLIConfig" + */ + if (APPLICATION_CONTEXT == null) { + APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(BatchConfig.class, DataJdbcConfiguration.class, CLIConfig.class, ServiceConfig.class); + HisInOneConfiguration.configSuperXDbformsXML("pg", SuperXManager.getWEB_INFPfad() + File.separator + ".."); + } + // Override the JobDescriptionSource Bean if the -f flag is passed. + if(FILESYSTEM) { + EtlJobApi etlJob = APPLICATION_CONTEXT.getBean(EtlJobApi.class); + EntityJobDescriptionSource entityJobDescriptionSource = APPLICATION_CONTEXT.getBean(EntityJobDescriptionSource.class); + etlJob.setJobDescriptionSource(entityJobDescriptionSource); + } + return APPLICATION_CONTEXT; + } + + private static void printDetails(ContainerNode cn) { + System.out.println("id: " + cn.id); + System.out.println("name: " + cn.name); + System.out.println("systeminfo_id: " + cn.systemInfoId); + System.out.println("tid: " + cn.tid); + System.out.println("type: " + cn.type); + } + + private static Options createOptions() { + Options options = new Options(); + Option opt; + opt = new Option("h", "help", false, "get help"); + options.addOption(opt); + opt = new Option("f", "filesystem", false, "load jobs from filesystem"); + options.addOption(opt); + opt = new Option("s", "setup", false, "setup minimal list of tables necessary for kern install"); + options.addOption(opt); + opt = new Option("la", "list-all", false, "list all available components"); + options.addOption(opt); + opt = new Option("li", "list-installables", false, "list all installable components"); + options.addOption(opt); + opt = new Option("le", "list-etl", false, "list all etl components"); + options.addOption(opt); + opt = new Option("i", "install", true, "install components"); + opt.setArgs(Option.UNLIMITED_VALUES); + options.addOption(opt); + opt = new Option("if", "install-functions", true, "install database functions for components (use 'all' to install functions for all installed components)"); + options.addOption(opt); + opt.setArgs(Option.UNLIMITED_VALUES); + opt = new Option("d", "deinstall", true, "de-/uninstall component"); + options.addOption(opt); + opt = new Option("u", "upgrade", true, "upgrade components"); + opt.setArgs(Option.UNLIMITED_VALUES); + options.addOption(opt); + opt = new Option("ua", "upgrade-all", false, "upgrade all installed components"); + options.addOption(opt); + opt = new Option("e", "etl", true, "run etl jobs"); + opt.setArgs(Option.UNLIMITED_VALUES); + options.addOption(opt); + opt = new Option("r", "reload-modules", false, "reload modules"); + options.addOption(opt); + //opt = new Option("re", "reload-module-etl", true, "reload module etl"); + //options.addOption(opt); + opt = new Option("db", "database", true, "database system"); + options.addOption(opt); + opt = new Option("lg", "log-to-stdout", false, "log only to stdout and not to individual job files"); + options.addOption(opt); + return options; + } + + private static CommandLine parseArgs(String[] args, Options options) { + CommandLineParser parser = new GnuParser(); + try { + return parser.parse(options, args, false); + } catch (ParseException e) { + System.out.println("error while reading the command line parameters:"); + e.printStackTrace(); + System.exit(1); + } + return null; + } + + // some actions require the SuperXManager, this is the place for initializing its static class attributes when needed + private static void initSuperXManager() { + try { + SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath()); + SuperXManager.setModuleDir(PathAndFileUtils.getWebinfPath() + File.separator + PathAndFileUtils.MODULE_PATH); + } catch(Exception e) { + System.out.println("error while initialising the SuperXManger:"); + e.printStackTrace(); + System.exit(1); + } + } + + // sxPools need to be initialized, because spring batch ETL uses them to look up the database connections + private static void initSxPools() { + try { + List mandantenNamen = new LinkedList(); + mandantenNamen.add(DEFAULT_MANDANTEN_ID); + SxPools.closeAll(); + SxPools.init(mandantenNamen); + SxPools.get(DEFAULT_MANDANTEN_ID).init(); + SxPools.get(DEFAULT_MANDANTEN_ID).initLogging(true, Level.DEBUG); + // also init kettle env, set plugin dir + SuperXManager.initKettleEnv(APPLICATION_CONTEXT); + } catch (Exception e) { + System.out.println("error while initialising the SuperX pools:"); + e.printStackTrace(); + System.exit(1); + } + } + +} diff --git a/src/de/superx/bin/DataProfiler.java b/src/de/superx/bin/DataProfiler.java new file mode 100644 index 0000000..c37c429 --- /dev/null +++ b/src/de/superx/bin/DataProfiler.java @@ -0,0 +1,674 @@ +package de.superx.bin; + +import static de.superx.servlet.SxSQL_Server.DEFAULT_MANDANTEN_ID; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +import javax.sql.DataSource; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFDataFormat; +import org.apache.poi.xssf.usermodel.XSSFFont; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.jdbc.core.RowMapper; + +import de.superx.servlet.SuperXManager; +import de.superx.servlet.SxPools; +import de.superx.spring.batch.His1DataSources; +import de.superx.spring.cli.config.CLIConfig; +import de.superx.spring.config.BatchConfig; +import de.superx.spring.config.DataJdbcConfiguration; +import de.superx.spring.config.ServiceConfig; + +/** + * A utility for creating data profiling statistics for tables in a database + * to be used in Data Warehouse Design. + * This can be used as a command line utility or embedded in an application. + */ +public class DataProfiler { + + private final static String SQL_COUNT_NULL = "select count(*) from %s where %s is null"; + private final static String SQL_PERCENT_UNIQUE = "select\n" + + " count(distinct %s) as unique_anz,\n" + + " count(distinct %s)::float / count(*) * 100 as unique_percentage\n" + + "from\n" + + " %s;"; + private final static String SQL_RANKING = "select %s, count(*) as anz from %s where %s is not null group by %s order by 2 desc limit 10;"; + private final static String SQL_MIN_MAX_LEN = "select min(length(%s)) as min_length, max(length(%s)) as max_length\n" + + "from %s\n" + + "where %s is not null"; + private final static String SQL_COUNT_VALUE = "select count(*) from %s where %s = %s"; + private final static String SQL_MIN_MAX_AVG = "select min(%s), max(%s), avg(%s) from %s where %s is not null;"; + private final static String SQL_START_END = "select min(%s), max(%s) from %s where %s is not null"; + + private static GenericApplicationContext APPLICATION_CONTEXT = null; + + private static String HELP_STRING = "Use this tool to profile database tables for dwh design. " + + "It needs the config file 'his1_databases.properties' inside the classpath; " + + "this file gets written automatically when starting the web application."; + + + static Logger logger = Logger.getLogger(DataProfiler.class); + + private DataSource dataSource; + private String database; + private String schema; + private String[] tables; + + /** + * Instantiate a new DataProfiler. + * @param dataSource The DataSource from which to read the table statistics. + * @param schema The schema from which to read. If null public is assumed. + * @param tables An Array of the names of the tables for which statistics should be created. + */ + public DataProfiler(DataSource dataSource, String schema, String[] tables) { + try (Connection con = dataSource.getConnection()) { + this.database = con.getCatalog(); + } catch (SQLException e) { + logger.error("Couldn't read catalog", e); + } + this.schema = schema != null ? schema : "public"; + List tableList = Arrays.asList(tables); + // TODO: Make sorting configurable? + tableList.sort(null); + this.tables = tableList.toArray(new String[] {}); + this.dataSource = dataSource; + + } + + public static void main(String[] args) { + System.setProperty(SuperXManager.SUPER_X_HISINONE_VERSION, "non-empty-value"); + Options options = createOptions(); + CommandLine parsedArgs = parseArgs(args, options); + if (parsedArgs.hasOption("h")) { + printHelp(options); + System.exit(0); + } + + String database = null; + String schema = "public"; + String[] tables = null; + + if (parsedArgs.hasOption("d")) { + database = parsedArgs.getOptionValue('d'); + } + if (parsedArgs.hasOption("s")) { + schema = parsedArgs.getOptionValue('s'); + } + if (parsedArgs.hasOption("t")) { + tables = parsedArgs.getOptionValues('t'); + } + if (!parsedArgs.hasOption('d') || !parsedArgs.hasOption('t')) { + printHelp(options); + System.exit(0); + } + try (GenericApplicationContext context = createContext()) { + initSxPools(); + DataSource dataSource = context.getBean(His1DataSources.class).get(database); + DataProfiler profiler = new DataProfiler(dataSource, schema, tables); + profiler.outputExcel(profiler.createStatistics(), null); + } + } + + /** + * Create the List of TableStatistics. + * @return List of TableStatistics. + */ + public List createStatistics() { + List tableStats = new ArrayList<>(); + try { + JdbcTemplate jt = new JdbcTemplate(dataSource); + logger.info("Database: " + database); + logger.info("Schema: " + schema); + logger.info("Tables: " + Arrays.asList(tables)); + try (Connection con = dataSource.getConnection()) { + jt.execute("set search_path to " + schema); + DatabaseMetaData meta = con.getMetaData(); + for (String table : tables) { + long rowCount = jt.queryForObject("select count(*) from " + table, Long.class).longValue(); + TableStatistic tableStat = new TableStatistic(table, rowCount); + logger.info("Table " + table); + try(ResultSet columns = meta.getColumns(null, schema, table, null); + ResultSet exported = meta.getExportedKeys(null, schema, table); + ResultSet imported = meta.getImportedKeys(null, schema, table); + ResultSet pks = meta.getPrimaryKeys(null, schema, table)) { + while(columns.next()) { + ColumnStatistic columnStat = new ColumnStatistic(); + columnStat.name = columns.getString("COLUMN_NAME"); + columnStat.size = columns.getInt("COLUMN_SIZE"); + columnStat.decimalDigits = columns.getInt("DECIMAL_DIGITS"); + columnStat.type = JDBCType.valueOf(columns.getInt("DATA_TYPE")); + columnStat.comment = columns.getString("REMARKS"); + columnStat.isNullable = columns.getString("IS_NULLABLE").equalsIgnoreCase("yes"); + columnStat.isAutoincrement = columns.getString("IS_AUTOINCREMENT").equalsIgnoreCase("yes"); + columnStat.countNull = jt.queryForObject(String.format(SQL_COUNT_NULL, tableStat.name, columnStat.name), Long.class).longValue(); + columnStat.percentNull = (double) columnStat.countNull / (double) tableStat.rowCount * 100.0; + jt.query(String.format(SQL_PERCENT_UNIQUE, columnStat.name, columnStat.name, tableStat.name), new RowCallbackHandler() { + + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.uniqueCount = rs.getLong(1); + columnStat.uniquePercent = rs.getDouble(2); + + } + }); + columnStat.ranking = jt.query(String.format(SQL_RANKING, columnStat.name, tableStat.name, columnStat.name, columnStat.name), new RowMapper() { + @Override + public RankingEntry mapRow(ResultSet rs, int rowNum) throws SQLException { + return new RankingEntry(rs.getString(1), rs.getInt(2)); + } + + }); + switch (columnStat.type) { + case CHAR: + case NCHAR: + case VARCHAR: + case NVARCHAR: + case LONGVARCHAR: + case LONGNVARCHAR: + jt.query(String.format(SQL_MIN_MAX_LEN, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() { + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.minLen = Optional.of(Integer.valueOf(rs.getInt(1))); + columnStat.maxLen = Optional.of(Integer.valueOf(rs.getInt(2))); + } + }); + columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, + "length(" + columnStat.name + ")", columnStat.minLen.get()), Integer.class)); + columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, + "length(" + columnStat.name + ")", columnStat.maxLen.get()), Integer.class)); + break; + case BIGINT: + case DECIMAL: + case DOUBLE: + case FLOAT: + case INTEGER: + case REAL: + case NUMERIC: + case SMALLINT: + case TINYINT: + jt.query(String.format(SQL_MIN_MAX_AVG, columnStat.name, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() { + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.min = Optional.of(Double.valueOf(rs.getDouble(1))); + columnStat.max = Optional.of(Double.valueOf(rs.getDouble(2))); + columnStat.avg = Optional.of(Double.valueOf(rs.getDouble(3))); + } + }); + columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, columnStat.min.get()), Integer.class)); + columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, columnStat.max.get()), Integer.class)); + break; + case DATE: + jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() { + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.earliestDate = Optional.ofNullable(rs.getDate(1)); + columnStat.latestDate = Optional.ofNullable(rs.getDate(2)); + } + }); + if (columnStat.earliestDate.isPresent()) { + columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestDate.get())), Integer.class)); + } + if (columnStat.latestDate.isPresent()) { + columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestDate.get())), Integer.class)); + } + break; + case TIMESTAMP: + case TIMESTAMP_WITH_TIMEZONE: + jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() { + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.earliestTimestamp = Optional.ofNullable(rs.getTimestamp(1)); + columnStat.latestTimestamp = Optional.ofNullable(rs.getTimestamp(2)); + } + }); + if (columnStat.earliestTimestamp.isPresent()) { + columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestTimestamp.get())), Integer.class)); + } + if (columnStat.latestTimestamp.isPresent()) { + columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestTimestamp.get())), Integer.class)); + } + break; + case TIME: + case TIME_WITH_TIMEZONE: + jt.query(String.format(SQL_START_END, columnStat.name, columnStat.name, tableStat.name, columnStat.name), new RowCallbackHandler() { + @Override + public void processRow(ResultSet rs) throws SQLException { + columnStat.earliestTime = Optional.ofNullable(rs.getTime(1)); + columnStat.latestTime = Optional.ofNullable(rs.getTime(2)); + } + }); + if (columnStat.earliestTime.isPresent()) { + columnStat.min_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.earliestTime.get())), Integer.class)); + } + if (columnStat.latestTime.isPresent()) { + columnStat.max_count = Optional.of(jt.queryForObject(String.format(SQL_COUNT_VALUE, tableStat.name, columnStat.name, quote(columnStat.latestTime.get())), Integer.class)); + } + break; + default: + } + tableStat.columns.add(columnStat); + } + while (exported.next()) { + String fromColumn = exported.getString("PKCOLUMN_NAME"); + String toTable = exported.getString("FKTABLE_NAME"); + String toColumn = exported .getString("FKCOLUMN_NAME"); + tableStat.exportedKeys.add(new ForeignKey(fromColumn, toTable, toColumn)); + + } + while (imported.next()) { + String fromColumn = imported.getString("FKCOLUMN_NAME"); + String toTable = imported.getString("PKTABLE_NAME"); + String toColumn = imported .getString("PKCOLUMN_NAME"); + tableStat.importedKeys.add(new ForeignKey(fromColumn, toTable, toColumn)); + + } + while (pks.next()) { + String column = pks.getString("COLUMN_NAME"); + tableStat.primaryKeys.add(column); + } + + } + tableStats.add(tableStat); + } + } + } catch (SQLException e) { + logger.error("SQL Fehler", e); + } + return tableStats; + } + + + /** + * Output statistic for a list of tables to an Excel file. + * The statistics of each table are written to a separate sheet. + * @param tableStats The list of TableStats + * @param outputFile The File to output to. If null output to current dir with a default file name. + */ + public void outputExcel(List tableStats, File outputFile) { + XSSFWorkbook workbook = new XSSFWorkbook(); + XSSFDataFormat dataFormat = workbook.createDataFormat(); + XSSFCellStyle cellStyleDouble = workbook.createCellStyle(); + cellStyleDouble.setDataFormat(dataFormat.getFormat("0.##")); + XSSFCellStyle headerStyle = workbook.createCellStyle(); + XSSFFont bold = workbook.createFont(); + bold.setBold(true); + headerStyle.setFont(bold); + for (TableStatistic tableStat : tableStats) { + XSSFSheet sheet = workbook.createSheet("Table " + tableStat.name); + XSSFRow header = sheet.createRow(0); + XSSFCell cell = header.createCell(0); + cell.setCellValue("Database: " + database); + cell.setCellStyle(headerStyle); + cell = header.createCell(1); + cell.setCellStyle(headerStyle); + cell.setCellValue("Schema: " + schema); + XSSFRow first = sheet.createRow(2); + first.createCell(0).setCellValue("Table"); + first.getCell(0).setCellStyle(headerStyle); + first.createCell(1).setCellValue("Row Count"); + first.getCell(1).setCellStyle(headerStyle); + first.createCell(2).setCellValue("Column"); + first.getCell(2).setCellStyle(headerStyle); + first.createCell(3).setCellValue("Type"); + first.getCell(3).setCellStyle(headerStyle); + first.createCell(4).setCellValue("Size"); + first.getCell(4).setCellStyle(headerStyle); + first.createCell(5).setCellValue("Not Null"); + first.getCell(5).setCellStyle(headerStyle); + first.createCell(6).setCellValue("Autoincrement"); + first.getCell(6).setCellStyle(headerStyle); + first.createCell(7).setCellValue("Count NULL"); + first.getCell(7).setCellStyle(headerStyle); + first.createCell(8).setCellValue("% NULL"); + first.getCell(8).setCellStyle(headerStyle); + first.createCell(9).setCellValue("Count Unique"); + first.getCell(9).setCellStyle(headerStyle); + first.createCell(10).setCellValue("% Unique"); + first.getCell(10).setCellStyle(headerStyle); + first.createCell(11).setCellValue("Min Len"); + first.getCell(11).setCellStyle(headerStyle); + first.createCell(12).setCellValue("Max Len"); + first.getCell(12).setCellStyle(headerStyle); + first.createCell(13).setCellValue("Min"); + first.getCell(13).setCellStyle(headerStyle); + first.createCell(14).setCellValue("Max"); + first.getCell(14).setCellStyle(headerStyle); + first.createCell(15).setCellValue("Avg"); + first.getCell(15).setCellStyle(headerStyle); + first.createCell(16).setCellValue("Min Count"); + first.getCell(16).setCellStyle(headerStyle); + first.createCell(17).setCellValue("Max Count"); + first.getCell(17).setCellStyle(headerStyle); + first.createCell(18).setCellValue("Earliest"); + first.getCell(18).setCellStyle(headerStyle); + first.createCell(19).setCellValue("Latest"); + first.getCell(19).setCellStyle(headerStyle); + first.createCell(20).setCellValue("Comment"); + first.getCell(20).setCellStyle(headerStyle); + int row = 3; + XSSFRow tableRow = sheet.createRow(row); + tableRow.createCell(0).setCellValue(tableStat.name); + tableRow.getCell(0).setCellStyle(headerStyle); + tableRow.createCell(1).setCellValue(tableStat.rowCount); + tableRow.getCell(1).setCellStyle(headerStyle); + for (ColumnStatistic columnStat : tableStat.columns) { + row += 1; + XSSFRow descRow = sheet.createRow(row); + descRow.createCell(2).setCellValue(columnStat.name); + if (tableStat.primaryKeys.contains(columnStat.name)) { + descRow.getCell(2).setCellValue(columnStat.name + " (PK)"); + descRow.getCell(2).setCellStyle(headerStyle); + } + descRow.createCell(3).setCellValue(columnStat.type.getName()); + descRow.createCell(4).setCellValue(columnStat.size); + if (columnStat.decimalDigits != 0) { + descRow.getCell(4).setCellValue( + Double.valueOf(columnStat.size + "." + columnStat.decimalDigits).doubleValue() + ); + descRow.getCell(4).setCellStyle(cellStyleDouble); + } + descRow.createCell(5).setCellValue(!columnStat.isNullable); + descRow.createCell(6).setCellValue(columnStat.isAutoincrement); + descRow.createCell(7).setCellValue(columnStat.countNull); + descRow.createCell(8).setCellValue(columnStat.percentNull); + descRow.getCell(8).setCellStyle(cellStyleDouble); + descRow.createCell(9).setCellValue(columnStat.uniqueCount); + descRow.createCell(10).setCellValue(columnStat.uniquePercent); + descRow.getCell(10).setCellStyle(cellStyleDouble); + if (columnStat.minLen.isPresent()) { + descRow.createCell(11).setCellValue(columnStat.minLen.get().doubleValue()); + } + if (columnStat.maxLen.isPresent()) { + descRow.createCell(12).setCellValue(columnStat.maxLen.get().doubleValue()); + } + if (columnStat.min.isPresent()) { + descRow.createCell(13).setCellValue(columnStat.min.get().doubleValue()); + descRow.getCell(13).setCellStyle(cellStyleDouble); + } + if (columnStat.max.isPresent()) { + descRow.createCell(14).setCellValue(columnStat.max.get().doubleValue()); + descRow.getCell(14).setCellStyle(cellStyleDouble); + } + if (columnStat.avg.isPresent()) { + descRow.createCell(15).setCellValue(columnStat.avg.get().doubleValue()); + descRow.getCell(15).setCellStyle(cellStyleDouble); + } + if (columnStat.min_count.isPresent()) { + descRow.createCell(16).setCellValue(columnStat.min_count.get().doubleValue()); + descRow.getCell(16).setCellStyle(cellStyleDouble); + } + if (columnStat.max_count.isPresent()) { + descRow.createCell(17).setCellValue(columnStat.max_count.get().doubleValue()); + descRow.getCell(17).setCellStyle(cellStyleDouble); + } + if (columnStat.earliestDate.isPresent()) { + descRow.createCell(18).setCellValue(columnStat.earliestDate.get().toString()); + } + if (columnStat.latestDate.isPresent()) { + descRow.createCell(19).setCellValue(columnStat.latestDate.get().toString()); + } + if (columnStat.earliestTime.isPresent()) { + descRow.createCell(18).setCellValue(columnStat.earliestTime.get().toString()); + } + if (columnStat.latestTime.isPresent()) { + descRow.createCell(19).setCellValue(columnStat.latestTime.get().toString()); + } + if (columnStat.earliestTimestamp.isPresent()) { + descRow.createCell(18).setCellValue(columnStat.earliestTimestamp.get().toString()); + } + if (columnStat.latestTimestamp.isPresent()) { + descRow.createCell(19).setCellValue(columnStat.latestTimestamp.get().toString()); + } + descRow.createCell(20).setCellValue(columnStat.comment); + } + for (int i = 0; i < 20; i++) { + sheet.autoSizeColumn(i); + } + XSSFRow frequHeader1 = sheet.createRow(row + 2); + XSSFRow frequHeader2 = sheet.createRow(row + 3); + frequHeader1.createCell(0).setCellValue("Frequency"); + frequHeader1.getCell(0).setCellStyle(headerStyle); + frequHeader2.createCell(0).setCellValue("Column"); + frequHeader2.getCell(0).setCellStyle(headerStyle); + for (int n = 1; n <= 10; n++) { + frequHeader1.createCell(n).setCellValue(n); + frequHeader1.getCell(n).setCellStyle(headerStyle); + } + for (int colNr = 0; colNr < tableStat.columns.size(); colNr++) { + XSSFRow frequRowLabel = sheet.createRow(row + 4 + 2 * colNr); + XSSFRow frequRowCount = sheet.createRow(row + 5 + 2 * colNr); + frequRowLabel.createCell(0).setCellValue(tableStat.columns.get(colNr).name); + for (int rankNr = 0; rankNr < tableStat.columns.get(colNr).ranking.size(); rankNr++) { + frequRowLabel.createCell(rankNr + 1).setCellValue(tableStat.columns.get(colNr).ranking.get(rankNr).label); + frequRowCount.createCell(rankNr + 1).setCellValue(tableStat.columns.get(colNr).ranking.get(rankNr).count); + } + } + + row = row + 2 * tableStat.columns.size() + 5; + XSSFRow exHeader1 = sheet.createRow(row); + exHeader1.createCell(0).setCellValue("Exported Keys"); + exHeader1.getCell(0).setCellStyle(headerStyle); + XSSFRow exHeader2 = sheet.createRow(row + 1); + exHeader2.createCell(0).setCellValue("From Column"); + exHeader2.getCell(0).setCellStyle(headerStyle); + exHeader2.createCell(1).setCellValue("To Table"); + exHeader2.getCell(1).setCellStyle(headerStyle); + exHeader2.createCell(2).setCellValue("To Column"); + exHeader2.getCell(2).setCellStyle(headerStyle); + for (int fkNr = 0; fkNr < tableStat.exportedKeys.size(); fkNr++) { + XSSFRow fkRow = sheet.createRow(row + 2 + fkNr); + fkRow.createCell(0).setCellValue(tableStat.exportedKeys.get(fkNr).fromColumn); + fkRow.createCell(1).setCellValue(tableStat.exportedKeys.get(fkNr).toTable); + fkRow.createCell(2).setCellValue(tableStat.exportedKeys.get(fkNr).toColumn); + } + row = row + 3 + tableStat.exportedKeys.size(); + XSSFRow imHeader1 = sheet.createRow(row); + imHeader1.createCell(0).setCellValue("Imported Keys"); + imHeader1.getCell(0).setCellStyle(headerStyle); + XSSFRow imHeader2 = sheet.createRow(row + 1); + imHeader2.createCell(0).setCellValue("From Table"); + imHeader2.getCell(0).setCellStyle(headerStyle); + imHeader2.createCell(1).setCellValue("From Column"); + imHeader2.getCell(1).setCellStyle(headerStyle); + imHeader2.createCell(2).setCellValue("To Column"); + imHeader2.getCell(2).setCellStyle(headerStyle); + for (int fkNr = 0; fkNr < tableStat.importedKeys.size(); fkNr++) { + XSSFRow fkRow = sheet.createRow(row + 2 + fkNr); + fkRow.createCell(0).setCellValue(tableStat.importedKeys.get(fkNr).toTable); + fkRow.createCell(1).setCellValue(tableStat.importedKeys.get(fkNr).toColumn); + fkRow.createCell(2).setCellValue(tableStat.importedKeys.get(fkNr).fromColumn); + } + } + + File currDir = new File("."); + String path = currDir.getAbsolutePath(); + String fileLocation = path.substring(0, path.length() - 1) + "db_profile_" + database + ".xlsx"; + if (outputFile != null) { + fileLocation = outputFile.getAbsolutePath(); + } + logger.info("Writing to " + fileLocation); + + FileOutputStream outputStream; + try { + outputStream = new FileOutputStream(fileLocation); + workbook.write(outputStream); + workbook.close(); + } catch (IOException e) { + logger.error("Couldn't write excel file", e); + } + } + + + private static Options createOptions() { + Options options = new Options(); + Option opt; + opt = new Option("h", "help", false, "get help"); + options.addOption(opt); + opt = new Option("t", "tables", true, "tables"); + opt.setArgs(Option.UNLIMITED_VALUES); +// opt.setRequired(true); + options.addOption(opt); + opt = new Option("s", "schema", true, "schema"); + options.addOption(opt); + opt = new Option("d", "database", true, "database"); +// opt.setRequired(true); + options.addOption(opt); + return options; + } + + private static CommandLine parseArgs(String[] args, Options options) { + CommandLineParser parser = new GnuParser(); + try { + return parser.parse(options, args, false); + } catch (ParseException e) { + System.out.println("error while reading the command line parameters:"); + e.printStackTrace(); + System.exit(1); + } + return null; + } + + private static void initSxPools() { + try { + List mandantenNamen = new LinkedList(); + mandantenNamen.add(DEFAULT_MANDANTEN_ID); + SxPools.closeAll(); + SxPools.init(mandantenNamen); + SxPools.get(DEFAULT_MANDANTEN_ID).init(); + SxPools.get(DEFAULT_MANDANTEN_ID).initLogging(true, Level.DEBUG); + // also init kettle env, set plugin dir + SuperXManager.initKettleEnv(APPLICATION_CONTEXT); + } catch (Exception e) { + System.out.println("error while initialising the SuperX pools:"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void printHelp(Options options) { + HelpFormatter help = new HelpFormatter(); + help.printHelp(HELP_STRING, options); + } + + private static GenericApplicationContext createContext() { + /* + * https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/context/annotation/AnnotationConfigApplicationContext.html + * quote: + * "In case of multiple @Configuration classes, @Bean methods defined in later classes will override those defined in earlier classes. + * This can be leveraged to deliberately override certain bean definitions via an extra @Configuration class." + * - so it's alright to override some beans via "CLIConfig" + */ + if (APPLICATION_CONTEXT == null) { + APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(BatchConfig.class, DataJdbcConfiguration.class, CLIConfig.class, ServiceConfig.class); + } + return APPLICATION_CONTEXT; + } + + private static String quote(Object o) { + return "'" + o + "'"; + } +} + +class TableStatistic { + + public String name; + public long rowCount; + public List columns; + public List exportedKeys; + public List importedKeys; + public List primaryKeys; + + public TableStatistic(String name, long rowCount) { + this.name = name; + this.rowCount = rowCount; + this.columns = new ArrayList<>(); + this.exportedKeys = new ArrayList<>(); + this.importedKeys = new ArrayList<>(); + this.primaryKeys = new ArrayList<>(); + } +} + +class ColumnStatistic { + public String name; + public JDBCType type; + public int size; + public int decimalDigits; + public boolean isNullable; + public boolean isAutoincrement; + public String comment; + public long countNull; + public double percentNull; + public long uniqueCount; + public double uniquePercent; + public Optional max_count = Optional.empty(); + public Optional min_count = Optional.empty(); + public List ranking; + public Optional minLen = Optional.empty(); + public Optional maxLen = Optional.empty(); + public Optional min = Optional.empty(); + public Optional max = Optional.empty(); + public Optional avg = Optional.empty(); + public Optional earliestDate = Optional.empty(); + public Optional latestDate = Optional.empty(); + public Optional