kern/test-src/de/superx/BIATestUtils.java

package de.superx;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.sql.DataSource;

import org.apache.commons.lang.SystemUtils;
import org.h2.jdbc.JdbcClob;
import org.junit.Ignore;
import org.postgresql.shaded.com.ongres.scram.common.bouncycastle.pbkdf2.RuntimeCryptoException;
import org.springframework.core.io.FileSystemResource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.init.ScriptUtils;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.json.JsonMapper;

import de.superx.bianalysis.ReportDefinition;
import de.superx.bianalysis.ReportMetadata;
import de.superx.bianalysis.StoredReport;
import de.superx.bianalysis.metadata.MetadataImporter;
import de.superx.bianalysis.service.BiAnalysisManager;
import de.superx.bianalysis.service.DbMetaAdapter;
import de.superx.dbt.DbtWrapper;
import de.superx.dbt.DbtUtils;
import de.superx.rest.Report;
import de.superx.rest.model.Column;
import de.superx.rest.model.Result;
import de.superx.rest.model.Row;
import de.superx.servlet.SuperXManager;
import de.superx.util.PathAndFileUtils;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;

@Ignore
public class BIATestUtils {

    public static String RW_TEST_PATH = String.join(File.separator, new String[] {"test", "resources", "db", "fixtures", "bianalysis" }); 

    public static boolean setupDone = false;

	public static List<String> readInsertsFromFile(String file) {
		List<String> expected = new ArrayList<>();
		Path path = Path.of(String.join(File.separator, file));
		try {
			expected = Files.readAllLines(path);
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
		return expected;
	}
	
	public static String formatSQL(String sql) {
		return sql.trim()
		          .replaceAll("\n", "")
		          .replaceAll("\r", "")
		          .replaceAll(" +", " ");
	}

    public static Result readResultFromJson(String response) {
    	ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
    	Result result = null;
		try {
			result = mapper.readValue(new File(response), Result.class);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
		return result;
	}

    public static String getStringFromStream(Reader reader) throws IOException {
        char[] buffer = new char[4096];
        StringBuilder builder = new StringBuilder();
        int numChars;
        while ((numChars = reader.read(buffer)) >= 0) {
            builder.append(buffer, 0, numChars);
        }
        return builder.toString();
	}

	public static void executeSqlScript(String filePath, DataSource dataSource) throws SQLException {
	    ScriptUtils.executeSqlScript(dataSource.getConnection(), new FileSystemResource(new File(filePath)));
	}


    public static void compareCellsForEquality(Result actualResult, Result expectedResult) {
        HashMap<String, Row> actRowsMap = new HashMap<>(); 
        HashMap<String, Row> expRowsMap = new HashMap<>(); 
        for (Row actRow : actualResult.rows) { actRowsMap.put(actRow.rowKey, actRow); }
        for (Row expRow : expectedResult.rows) { expRowsMap.put(expRow.rowKey, expRow); }
        
        assertEquals(expRowsMap.size(), actRowsMap.size());
        
        for(Row expRow : expectedResult.rows) {

           Map<String, Object> cellsExp = expRow.cells;
           assertTrue("Rows are missing Row Key: " + expRow.rowKey, actRowsMap.containsKey(expRow.rowKey));

           Map<String, Object> cellsAct = actRowsMap.get(expRow.rowKey).cells;

           for (String keyExp : cellsExp.keySet()) {
               Object objExp = cellsExp.get(keyExp);
               Object objAct = cellsAct.get(keyExp);
               boolean areCellsEqual = compareTwoCells(objExp, objAct);

               String failureMessage = String.format(
                        "\n Cell Mismatch Detected!\n" +
                        "------------------------------\n" +
                        "  Row Key: %s\n" +
                        "  Cell Key: %s\n" +
                        "  Expected -> %s\n" +
                        "  Actual -> %s\n" +
                        "------------------------------",
                        expRow.rowKey,
                        keyExp,
                        String.valueOf(objExp),
                        String.valueOf(objAct)
                       );

               assertTrue(failureMessage, areCellsEqual);
           }
        }
    }
    
    

    public static boolean compareTwoCells(Object objExp, Object objAct) {
        // Handle strings and nulls first
        if (objExp instanceof String && objAct instanceof String) {
            return objExp.equals(objAct);
        }
        if (objExp == null && objAct == null) {
            return true;
        }
        if((objExp == null && objAct != null) || (objExp != null && objAct == null)) {
           return false; 
        }

        // Convert both to rounded doubles and compare
        double expDouble = toDoubleRounded(objExp);
        double actDouble = toDoubleRounded(objAct);

        return Math.abs(expDouble - actDouble) < 0.001; // Fuzzy comparison
    }

    private static double toDoubleRounded(Object obj) {
        if (obj == null) {
            throw new IllegalArgumentException("Cannot convert null to double");
        }

        // Handle LinkedHashMap 
        if (obj instanceof LinkedHashMap<?, ?>) {
            LinkedHashMap<String, Object> map = (LinkedHashMap<String, Object>) obj;
            Object value = map.get("parsedValue");
            if (value == null) {
                value = map.get("source");
            }
            if (value == null) {
                throw new IllegalArgumentException("No numeric value in map: " + map);
            }

            return BigDecimal.valueOf(toPrimitiveDouble(value))
                    .setScale(2, RoundingMode.HALF_UP)
                    .doubleValue();
        }

        // Handle all numbers uniformly
        if (obj instanceof Number num) {
            return BigDecimal.valueOf(num.doubleValue())
                    .setScale(2, RoundingMode.HALF_UP)
                    .doubleValue();
        }

        throw new IllegalArgumentException("Unsupported type: " + obj.getClass() + " - " + obj);
    }

    private static double toPrimitiveDouble(Object value) {
        if (value instanceof Number num) {
            return num.doubleValue();
        }
        return Double.parseDouble(value.toString().trim());
    }

    public static String getSQLFormatPath() {
        boolean isLinux = SystemUtils.IS_OS_LINUX;
        String version = DbtUtils.getPythonVersion();
        SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath());
        Path dbtInstallDir = Path
                .of(SuperXManager.getWEB_INFPfad(), "..", "dbt", "dbt_" + (isLinux ? "linux" : "windows"))
                .toAbsolutePath().normalize();
        Path sl = Path.of(dbtInstallDir.toString(), "bin", "dbt");
        if (!isLinux) {
            String[] majorMinor=version.split("\\.");
            sl = Path.of(dbtInstallDir.toString(), "Python" + majorMinor[0] + majorMinor[1], "Scripts", "sqlformat.exe").toAbsolutePath();
        }
        return sl.toString();
    }

    public static String getFormattedSQL(String sql) {
        try {
            File tempFile = File.createTempFile("rw_test_utils-", ".tmp");
            Files.write(Paths.get(tempFile.getAbsolutePath()), sql.getBytes());
            ProcessBuilder builder = new ProcessBuilder(DbtUtils.getPython3ExecutableName(), getSQLFormatPath(),"-a", "-k", "upper", tempFile.getAbsolutePath());
            Process last = builder.start();
            BufferedReader error = new BufferedReader(new InputStreamReader(last.getErrorStream()));
            BufferedReader output = new BufferedReader(new InputStreamReader(last.getInputStream()));
            List<String> results = output.lines().collect(Collectors.toList());
            List<String> errors = error.lines().collect(Collectors.toList());
            tempFile.delete();
            String result = "";
            for (String string : results) {
                result += string + "\n";
            }
            if(!errors.isEmpty()) {
                for (String string : errors) {
                    System.out.println(string); 
                }
                return sql;
            }
            return result;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static ReportDefinition readReportDefinitionFromJsonInTestDir(String name) {
        String path = String.join(File.separator, new String[] {RW_TEST_PATH, "req", name + ".json"});
        ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
        ReportDefinition reportDefinition = null;
        try {
            reportDefinition = mapper.readValue(new File(path), ReportDefinition.class);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return reportDefinition;
    }

    public static ReportDefinition getReportDefinitionFromFile(String name) {
        ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
        ReportDefinition reportDefinition = null;
        try {
            reportDefinition = mapper.readValue(new File(name), ReportDefinition.class);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return reportDefinition;
    }

    public static StoredReport readStoredReportFromJson(String name) {
        String path = String.join(File.separator, new String[] {RW_TEST_PATH, "req", name + ".json"});
        ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
        StoredReport storedReport = null;
        try {
            storedReport = mapper.readValue(new File(path), StoredReport.class);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return storedReport;
    }

    public static String getStoredReportDefinition(StoredReport storedReport) {
        ObjectWriter ow = new ObjectMapper().writer();
        String reportDefinitionJson = null;
        try {
            reportDefinitionJson =  ow.writeValueAsString(storedReport.reportDefinition);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
        return reportDefinitionJson;
    }

    public static void compareResultColumns(Result resultActual, Result result) {
        assertEquals(resultActual.columns.size(), result.columns.size());

        Map<String, Column> actColMap = new HashMap<>();
        for(Column actCol : resultActual.columns) {
            actColMap.put(actCol.field, actCol);
        }

        for (Column expCol : result.columns) {
            Column actCol = actColMap.get(expCol.field);
            if(actCol == null) {
                throw new RuntimeException("Expected to find column with field: " + expCol.field);
            }
            assertEquals(actCol.aggregation, expCol.aggregation);
            assertEquals(Boolean.valueOf(actCol.isTotalColumn), Boolean.valueOf(expCol.isTotalColumn));
            assertEquals(Boolean.valueOf(actCol.groupable), Boolean.valueOf(expCol.groupable));
            assertEquals(actCol.type, expCol.type);
        }
    }

    public static void writeReportResultToFile(Result result, File file) {
        ObjectMapper mapper = new ObjectMapper();
        DefaultPrettyPrinter printer = new DefaultPrettyPrinter();
        try {
            mapper.writer(printer).writeValue(file, result);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static String formatSQLIfEclipse(String sql) {
        String inEclipseStr = System.getProperty("runInEclipse");
        if (inEclipseStr != null && inEclipseStr.equals("true")) {
            return BIATestUtils.getFormattedSQL(sql);
        }
        return formatSQL(sql);
    }

    public static String getTotalColumnSqlFromReportDefinition(String json, DbMetaAdapter dbAdapter) {
        ObjectMapper mapper = new ObjectMapper();
        try {
            ReportDefinition definition = mapper.readValue(json, ReportDefinition.class);
            String sql = BiAnalysisManager.getTotalsColumnSqlStatement(definition, dbAdapter);
            Statement sqlStatement = CCJSqlParserUtil.parse(sql);
            return sqlStatement.toString();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
    public static List<Column> getResultColumnsFromReport(String json, DbMetaAdapter dbAdapter, BiAnalysisManager biAnalysisManager) {
        ObjectMapper mapper = new ObjectMapper();
        try {
            ReportDefinition definition = mapper.readValue(json, ReportDefinition.class);
            ReportMetadata metadata = new ReportMetadata(definition, definition.factTableId, dbAdapter);
            Result result = biAnalysisManager.getReportData(metadata, dbAdapter, definition);
            return result.columns;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
    
    public static ReportDefinition getReportDefinitionFromString(String json) {
        ObjectMapper mapper = new ObjectMapper();
        try {
            ReportDefinition definition = mapper.readValue(json, ReportDefinition.class);
            return definition;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static String getSqlFromReportDefinition(String json, DbMetaAdapter dbAdapter) {
        ObjectMapper mapper = new ObjectMapper();
        try {
            ReportDefinition definition = mapper.readValue(json, ReportDefinition.class);
            ReportMetadata metadata = new ReportMetadata(definition, definition.factTableId, dbAdapter);
            String sql = BiAnalysisManager.getSqlStatement(metadata);
            Statement sqlStatement = CCJSqlParserUtil.parse(sql);
            return sqlStatement.toString();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static String getParsedSqlFromString(String expectedSql) {
        try {
            Statement sqlStatement = CCJSqlParserUtil.parse(expectedSql);
            return sqlStatement.toString();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    
    public static List<Column> parseColumns(String json) throws Exception {
        ObjectMapper objectMapper = new ObjectMapper();
        JsonNode root = objectMapper.readTree(json);

        JsonNode columnsNode = root.get("columns");
        if (columnsNode == null || !columnsNode.isArray()) {
            throw new IllegalArgumentException("JSON must contain an array field named 'columns'");
        }

        List<Column> result = new ArrayList<>();
        for (JsonNode node : columnsNode) {
            String field = node.has("field") ? node.get("field").asText() : null;
            String header = node.has("header") ? node.get("header").asText() : null;
            result.add(new Column(field, header));
        }

        return result;

    }

}