package de.superx; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; import java.sql.SQLException; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.sql.DataSource; import org.apache.commons.lang.SystemUtils; import org.h2.jdbc.JdbcClob; import org.junit.Ignore; import org.postgresql.shaded.com.ongres.scram.common.bouncycastle.pbkdf2.RuntimeCryptoException; import org.springframework.core.io.FileSystemResource; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.init.ScriptUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.json.JsonMapper; import de.superx.bianalysis.ReportDefinition; import de.superx.bianalysis.ReportMetadata; import de.superx.bianalysis.StoredReport; import de.superx.bianalysis.metadata.MetadataImporter; import de.superx.bianalysis.service.BiAnalysisManager; import de.superx.bianalysis.service.DbMetaAdapter; import de.superx.dbt.DbtWrapper; import de.superx.dbt.DbtUtils; import de.superx.rest.Report; import de.superx.rest.model.Column; import de.superx.rest.model.Result; import de.superx.rest.model.Row; import de.superx.servlet.SuperXManager; import de.superx.util.PathAndFileUtils; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statement; @Ignore public class BIATestUtils { public static String RW_TEST_PATH = String.join(File.separator, new String[] {"test", "resources", "db", "fixtures", "bianalysis" }); public static boolean setupDone = false; public static List readInsertsFromFile(String file) { List expected = new ArrayList<>(); Path path = Path.of(String.join(File.separator, file)); try { expected = Files.readAllLines(path); } catch (IOException e) { throw new RuntimeException(e); } return expected; } public static String formatSQL(String sql) { return sql.trim() .replaceAll("\n", "") .replaceAll("\r", "") .replaceAll(" +", " "); } public static Result readResultFromJson(String response) { ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); Result result = null; try { result = mapper.readValue(new File(response), Result.class); } catch (Exception e) { throw new RuntimeException(e); } return result; } public static String getStringFromStream(Reader reader) throws IOException { char[] buffer = new char[4096]; StringBuilder builder = new StringBuilder(); int numChars; while ((numChars = reader.read(buffer)) >= 0) { builder.append(buffer, 0, numChars); } return builder.toString(); } public static void executeSqlScript(String filePath, DataSource dataSource) throws SQLException { ScriptUtils.executeSqlScript(dataSource.getConnection(), new FileSystemResource(new File(filePath))); } public static void compareCellsForEquality(Result actualResult, Result expectedResult) { HashMap actRowsMap = new HashMap<>(); HashMap expRowsMap = new HashMap<>(); for (Row actRow : actualResult.rows) { actRowsMap.put(actRow.rowKey, actRow); } for (Row expRow : expectedResult.rows) { expRowsMap.put(expRow.rowKey, expRow); } assertEquals(expRowsMap.size(), actRowsMap.size()); for(Row expRow : expectedResult.rows) { Map cellsExp = expRow.cells; assertTrue("Rows are missing Row Key: " + expRow.rowKey, actRowsMap.containsKey(expRow.rowKey)); Map cellsAct = actRowsMap.get(expRow.rowKey).cells; for (String keyExp : cellsExp.keySet()) { Object objExp = cellsExp.get(keyExp); Object objAct = cellsAct.get(keyExp); boolean areCellsEqual = compareTwoCells(objExp, objAct); String failureMessage = String.format( "\n Cell Mismatch Detected!\n" + "------------------------------\n" + " Row Key: %s\n" + " Cell Key: %s\n" + " Expected -> %s\n" + " Actual -> %s\n" + "------------------------------", expRow.rowKey, keyExp, String.valueOf(objExp), String.valueOf(objAct) ); assertTrue(failureMessage, areCellsEqual); } } } public static boolean compareTwoCells(Object objExp, Object objAct) { // Handle strings and nulls first if (objExp instanceof String && objAct instanceof String) { return objExp.equals(objAct); } if (objExp == null && objAct == null) { return true; } if((objExp == null && objAct != null) || (objExp != null && objAct == null)) { return false; } // Convert both to rounded doubles and compare double expDouble = toDoubleRounded(objExp); double actDouble = toDoubleRounded(objAct); return Math.abs(expDouble - actDouble) < 0.001; // Fuzzy comparison } private static double toDoubleRounded(Object obj) { if (obj == null) { throw new IllegalArgumentException("Cannot convert null to double"); } // Handle LinkedHashMap if (obj instanceof LinkedHashMap) { LinkedHashMap map = (LinkedHashMap) obj; Object value = map.get("parsedValue"); if (value == null) { value = map.get("source"); } if (value == null) { throw new IllegalArgumentException("No numeric value in map: " + map); } return BigDecimal.valueOf(toPrimitiveDouble(value)) .setScale(2, RoundingMode.HALF_UP) .doubleValue(); } // Handle all numbers uniformly if (obj instanceof Number num) { return BigDecimal.valueOf(num.doubleValue()) .setScale(2, RoundingMode.HALF_UP) .doubleValue(); } throw new IllegalArgumentException("Unsupported type: " + obj.getClass() + " - " + obj); } private static double toPrimitiveDouble(Object value) { if (value instanceof Number num) { return num.doubleValue(); } return Double.parseDouble(value.toString().trim()); } public static String getSQLFormatPath() { boolean isLinux = SystemUtils.IS_OS_LINUX; String version = DbtUtils.getPythonVersion(); SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath()); Path dbtInstallDir = Path .of(SuperXManager.getWEB_INFPfad(), "..", "dbt", "dbt_" + (isLinux ? "linux" : "windows")) .toAbsolutePath().normalize(); Path sl = Path.of(dbtInstallDir.toString(), "bin", "dbt"); if (!isLinux) { String[] majorMinor=version.split("\\."); sl = Path.of(dbtInstallDir.toString(), "Python" + majorMinor[0] + majorMinor[1], "Scripts", "sqlformat.exe").toAbsolutePath(); } return sl.toString(); } public static String getFormattedSQL(String sql) { try { File tempFile = File.createTempFile("rw_test_utils-", ".tmp"); Files.write(Paths.get(tempFile.getAbsolutePath()), sql.getBytes()); ProcessBuilder builder = new ProcessBuilder(DbtUtils.getPython3ExecutableName(), getSQLFormatPath(),"-a", "-k", "upper", tempFile.getAbsolutePath()); Process last = builder.start(); BufferedReader error = new BufferedReader(new InputStreamReader(last.getErrorStream())); BufferedReader output = new BufferedReader(new InputStreamReader(last.getInputStream())); List results = output.lines().collect(Collectors.toList()); List errors = error.lines().collect(Collectors.toList()); tempFile.delete(); String result = ""; for (String string : results) { result += string + "\n"; } if(!errors.isEmpty()) { for (String string : errors) { System.out.println(string); } return sql; } return result; } catch (Exception e) { throw new RuntimeException(e); } } public static ReportDefinition readReportDefinitionFromJsonInTestDir(String name) { String path = String.join(File.separator, new String[] {RW_TEST_PATH, "req", name + ".json"}); ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); ReportDefinition reportDefinition = null; try { reportDefinition = mapper.readValue(new File(path), ReportDefinition.class); } catch (Exception e) { throw new RuntimeException(e); } return reportDefinition; } public static ReportDefinition getReportDefinitionFromFile(String name) { ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); ReportDefinition reportDefinition = null; try { reportDefinition = mapper.readValue(new File(name), ReportDefinition.class); } catch (Exception e) { throw new RuntimeException(e); } return reportDefinition; } public static StoredReport readStoredReportFromJson(String name) { String path = String.join(File.separator, new String[] {RW_TEST_PATH, "req", name + ".json"}); ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build(); StoredReport storedReport = null; try { storedReport = mapper.readValue(new File(path), StoredReport.class); } catch (Exception e) { throw new RuntimeException(e); } return storedReport; } public static String getStoredReportDefinition(StoredReport storedReport) { ObjectWriter ow = new ObjectMapper().writer(); String reportDefinitionJson = null; try { reportDefinitionJson = ow.writeValueAsString(storedReport.reportDefinition); } catch (JsonProcessingException e) { throw new RuntimeException(e); } return reportDefinitionJson; } public static void compareResultColumns(Result resultActual, Result result) { assertEquals(resultActual.columns.size(), result.columns.size()); Map actColMap = new HashMap<>(); for(Column actCol : resultActual.columns) { actColMap.put(actCol.field, actCol); } for (Column expCol : result.columns) { Column actCol = actColMap.get(expCol.field); if(actCol == null) { throw new RuntimeException("Expected to find column with field: " + expCol.field); } assertEquals(actCol.aggregation, expCol.aggregation); assertEquals(Boolean.valueOf(actCol.isTotalColumn), Boolean.valueOf(expCol.isTotalColumn)); assertEquals(Boolean.valueOf(actCol.groupable), Boolean.valueOf(expCol.groupable)); assertEquals(actCol.type, expCol.type); } } public static void writeReportResultToFile(Result result, File file) { ObjectMapper mapper = new ObjectMapper(); DefaultPrettyPrinter printer = new DefaultPrettyPrinter(); try { mapper.writer(printer).writeValue(file, result); } catch (Exception e) { throw new RuntimeException(e); } } public static String formatSQLIfEclipse(String sql) { String inEclipseStr = System.getProperty("runInEclipse"); if (inEclipseStr != null && inEclipseStr.equals("true")) { return BIATestUtils.getFormattedSQL(sql); } return formatSQL(sql); } public static String getTotalColumnSqlFromReportDefinition(String json, DbMetaAdapter dbAdapter) { ObjectMapper mapper = new ObjectMapper(); try { ReportDefinition definition = mapper.readValue(json, ReportDefinition.class); String sql = BiAnalysisManager.getTotalsColumnSqlStatement(definition, dbAdapter); Statement sqlStatement = CCJSqlParserUtil.parse(sql); return sqlStatement.toString(); } catch (Exception e) { throw new RuntimeException(e); } } public static List getResultColumnsFromReport(String json, DbMetaAdapter dbAdapter, BiAnalysisManager biAnalysisManager) { ObjectMapper mapper = new ObjectMapper(); try { ReportDefinition definition = mapper.readValue(json, ReportDefinition.class); ReportMetadata metadata = new ReportMetadata(definition, definition.factTableId, dbAdapter); Result result = biAnalysisManager.getReportData(metadata, dbAdapter, definition); return result.columns; } catch (Exception e) { throw new RuntimeException(e); } } public static ReportDefinition getReportDefinitionFromString(String json) { ObjectMapper mapper = new ObjectMapper(); try { ReportDefinition definition = mapper.readValue(json, ReportDefinition.class); return definition; } catch (Exception e) { throw new RuntimeException(e); } } public static String getSqlFromReportDefinition(String json, DbMetaAdapter dbAdapter) { ObjectMapper mapper = new ObjectMapper(); try { ReportDefinition definition = mapper.readValue(json, ReportDefinition.class); ReportMetadata metadata = new ReportMetadata(definition, definition.factTableId, dbAdapter); String sql = BiAnalysisManager.getSqlStatement(metadata); Statement sqlStatement = CCJSqlParserUtil.parse(sql); return sqlStatement.toString(); } catch (Exception e) { throw new RuntimeException(e); } } public static String getParsedSqlFromString(String expectedSql) { try { Statement sqlStatement = CCJSqlParserUtil.parse(expectedSql); return sqlStatement.toString(); } catch (Exception e) { throw new RuntimeException(e); } } public static List parseColumns(String json) throws Exception { ObjectMapper objectMapper = new ObjectMapper(); JsonNode root = objectMapper.readTree(json); JsonNode columnsNode = root.get("columns"); if (columnsNode == null || !columnsNode.isArray()) { throw new IllegalArgumentException("JSON must contain an array field named 'columns'"); } List result = new ArrayList<>(); for (JsonNode node : columnsNode) { String field = node.has("field") ? node.get("field").asText() : null; String header = node.has("header") ? node.get("header").asText() : null; result.add(new Column(field, header)); } return result; } }