kern/test-src/de/superx/TestDbManagerPg.java

package de.superx;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.sql.DataSource;

import org.junit.Ignore;

import com.zaxxer.hikari.HikariDataSource;

@Ignore
public class TestDbManagerPg implements TestDbManager {
    // TODO: Hier sollten ALLE Test-DBs definiert werden
    // Momentan wird aber nur eduetl für die Tests verwendet
    protected static final String[] TEST_DBS = new String[] { EDUETL_DB, HISINONE_DB };
    protected static final String LOCAL = "localhost";
    protected static final String USER = "postgres";
    protected static final String PASSWORD = "postgres";
    private static final String QISDEV_HIS1_DUMP_URL = "http://qisdev.his.de/databaseBackups/hio_%s-stable,audit,develop,tests_qamuster.backup";
    private static final String QISDEV_HIS1_DUMP_URL_BRANCH = "http://qisdev.his.de/databaseBackups/hio_%s-stable,audit_qamuster.backup";
    private static final String QISDEV_EDUETL_DUMP_URL = "http://qisdev.his.de/Distributionen/BI-Musterdaten/%s/eduetl_muster_update.backup";
    private static Pattern BRANCH_NAME_PATTERN = Pattern.compile("^(release\\_\\d\\d\\d\\d\\_\\d\\d)|(master)$");
    private static final String VMPOSTGRES = "vmpostgres17.his.de";
    private static final String SERVER = VMPOSTGRES;
    private static final String CONNECTION_STRING_PREFIX = String.format(
            "jdbc:postgresql://%s/", SERVER);

    private static final String PREFIX = "bi_tests_";

    protected static final String CONNECTION_STRING = String.format("%spostgres", CONNECTION_STRING_PREFIX);
    
    private String branchName;
    private Map<String,String> testDbNames = new HashMap<>();
    
    public TestDbManagerPg() {
        String branch = getBranchName();
        System.out.println(String.format("Branch name über git gefunden: %s", branch));
    }
    
    @Override
    public
    void setupTestDbs() {
        String connectionString = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement stmt = conn.createStatement();
                Statement deleteStmt = conn.createStatement();
            ) {
            
            // "Legacy": Dumps von den Builds auf
            // qa.his.de und qisdev.his.de holen:
            // ensureLatestQaDumps();
            
            // "Neu": Dumps von qisdev direkt herunterladen:
            LocalDateTime ld = LocalDateTime.now().minusHours(19).minusMinutes(30);
            String dateStr = ld.format(DateTimeFormatter.BASIC_ISO_DATE);
            ensureLatestQisdevDumps(dateStr);
            // TODO: Hier sollte über ALLE Test-DBs iteriert werden
            // Momentan wird aber nur eduetl für die Tests verwendet
            for (String dbName : TEST_DBS) {
                String lookupSql = String.format(
                                                 "SELECT datname FROM pg_database WHERE datname = '" + getTemplateName(dbName, branchName, dateStr) + "';");
                ResultSet result = stmt.executeQuery(lookupSql);
                if (!result.next()) {
                    throw new RuntimeException("could not set up test database, because no db template was found :´(");
                }
                String templateName = result.getString("datname");
                String databaseName = getTestDbName(dbName);
                String createSql = String.format("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s TEMPLATE %s;", databaseName, databaseName, templateName);
                stmt.execute(createSql);
            }
        } catch (SQLException e) {
            System.out.println("Schade, etwas ging bei der Datenbankanlage schief:");
            e.printStackTrace();
        }
        /*
         * workaround wegen vorübergehend "schräger" Musterdaten - zum Vergleich:
         * https://hiszilla.his.de/hiszilla/show_bug.cgi?id=364365
         * wenn das Erfassungssystem auf Version 2026.06 ist und die Daten dort
         * angepasst wurden, _sollte_ der workaround entfernt werden können...
         */
        String eduetlName = getTestDbName(EDUETL_DB);
        connectionString = String.format("%s%s", CONNECTION_STRING_PREFIX, eduetlName);
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement stmt = conn.createStatement();
            ) {
            String mySql = "UPDATE sos_stg_aggr SET fach_sem_zahl = abs(fach_sem_zahl);";
            stmt.execute(mySql);
        } catch (SQLException e) {
            System.out.println("Beim Anpassungen der eduetl-Datenbank ging etwas schief:");
            e.printStackTrace();
        }
        // workaround Ende
    }
    
    @Override
    public DataSource getDataSource(String logicalDatabaseName) {
	    HikariDataSource ds = new HikariDataSource();
	    ds.setJdbcUrl(getJdbcUrl(logicalDatabaseName));
	    ds.setUsername(TestDbManagerPg.USER);
	    ds.setPassword(TestDbManagerPg.PASSWORD);
	    ds.setConnectionInitSql("SET standard_conforming_strings TO 'off'; SET datestyle TO 'ISO, DMY';");
	    if ("hisinone".equals(logicalDatabaseName)) {
	        ds.setSchema("hisinone");
	    }
        return ds;
    }
    
    @Override
    public void removeTestDbs() {
        String connectionString = TestDbManagerPg.CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement stmt = conn.createStatement();
            ) {
            for (String dbName : TEST_DBS) {
                String databaseName = getTestDbName(dbName);
                String dropSql = String.format("DROP DATABASE IF EXISTS %s;", databaseName);
                stmt.execute(dropSql);
            }
        } catch (SQLException e) {
            System.out.println("Problem bei Bereinigung der Testdatenbank:");
            e.printStackTrace();
        }
    }

    protected void ensureLatestQisdevDumps(String dateStr) throws SQLException {
        //System.out.println("entering ensureLatestQisdevDumps()");
        /* Warum werden hier 19:30 abgezogen?
         * Die Klasse spielt Dumps ein und macht daraus template-Datenbanken,
         * aus welchen dann Testdatenbanken erzeugt werden.
         * Das sollte möglichst während der nächtlichen Tests passieren, weil es lange dauert,
         * beim Entwickeln will man nicht, dass sowas passiert.
         * Um nicht auf veralteten Daten zu arbeiten, kommt ein Timestamp des Erzeugungszeitpunktes
         * in den Templatenamen - Bsp. "2025_09_04".
         * Man weiss aber nicht sicher, ob das Template während des Builds vor oder nach Mitternacht
         * erzeugt wird. Wenn das vor Mitternacht passiert und dann nach Mitternacht plötzlich den
         * Timestamp des Vortages hat, würde es bereits als veraltet gelten und bei der ersten
         * Ausführung eines Testfalls müsste man auf den Neuaufbau warten.
         * Daher die Lösung:
         * Man zieht bei der Erzeugung des Timestampt noch 19:30 ab; so wechselt der Timestamp erst
         * nach Feierabend auf den nächsten Tag.
         * */
        if( !templateExists(EDUETL_DB, branchName, dateStr)) {
            System.out.println(String.format("kein %s template vorhanden...erstelle", EDUETL_DB));
            restoreDump(EDUETL_DB, branchName, dateStr);
            createHis1UnitTestDb(EDUETL_DB, branchName, dateStr);
        } else {
            System.out.println(String.format("%s template bereits vorhanden...", EDUETL_DB));
        }
        if( !templateExists(HISINONE_DB, branchName, dateStr)) {
            System.out.println(String.format("kein %s template vorhanden...erstelle", HISINONE_DB));
            restoreDump(HISINONE_DB, branchName, dateStr);
            createHis1UnitTestDb(HISINONE_DB, branchName, dateStr);
        } else {
            System.out.println(String.format("%s template bereits vorhanden...", HISINONE_DB));
        }
    }

    protected String getTestDbName(String name) {
        if ( !this.testDbNames.containsKey(name) ) {
            SimpleDateFormat dateformat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss");
            String timestamp = dateformat.format(new Date());
            try {
                this.testDbNames.put(name, String.format(PREFIX + "%s_%s_%s_%s",
                        name,
                        InetAddress.getLocalHost().getHostName().replace("-", "_"),
                        branchName,
                        timestamp));
            } catch (UnknownHostException e) {
                e.printStackTrace();
            }
        }
        return this.testDbNames.get(name);
    }

    protected String getJdbcUrl(String name) {
        return CONNECTION_STRING_PREFIX + getTestDbName(name);
    }
    
    protected static Properties dbConnectionProperties() {
        //System.out.println("entering dbConnectionProperties()");
        Properties props = new Properties();
        props.setProperty("user", TestDbManagerPg.USER);
        props.setProperty("password", TestDbManagerPg.PASSWORD);
        props.setProperty("ssl", "false");
        return props;
    }

    private String getBranchName() {
        if ( this.branchName == null ) {
            String[] branch_name_command = {"git", "branch", "--show-current"};
            ProcessBuilder branch_name = new ProcessBuilder(branch_name_command);
            try {
                Process branch_name_proc;
                branch_name_proc = branch_name.start();
                try (InputStreamReader branch_name_proc_stderr = new InputStreamReader(branch_name_proc.getErrorStream());
                                InputStreamReader branch_name_proc_stdout = new InputStreamReader(branch_name_proc.getInputStream())) {
                    String err0 = new BufferedReader(branch_name_proc_stderr).lines().collect(Collectors.joining("\n"));
                    String output = new BufferedReader(branch_name_proc_stdout).lines().collect(Collectors.joining("\n"));
                    String branchNameFound = output.trim().toLowerCase();
                    Matcher matcher = BRANCH_NAME_PATTERN.matcher(branchNameFound);
                    if ( matcher.find() ) {
                        System.out.println(String.format("git branch '%s' für Auswahl der Test DB gefunden", branchNameFound));
                        this.branchName = branchNameFound;
                    } else {
                        System.out.println("konnte keinen zu db dumps passenden git branch finden - benutze 'master'");
                        this.branchName = "master";
                    }
                }
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        return this.branchName;
    }

    private static boolean templateExists(String database, String branch, String dateStr) throws SQLException {
        //System.out.println("entering templateExists()");
        String templateName = getTemplateName(database, branch, dateStr);
        String url = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (Connection conn = DriverManager.getConnection(url, props); Statement stmt = conn.createStatement();) {
            String sql = String.format("SELECT datname FROM pg_database WHERE datname = '%s';", templateName);
            ResultSet result = stmt.executeQuery(sql);
            return result.next();
        } catch (SQLException e) {
            System.out.println("Konnte die Buildnummer des Templates von '" + database + "' nicht prüfen:");
            e.printStackTrace();
            throw e;
        }
    }

    private static String getTemplateName(String database, String branch, String dateStr) {
        return String.format(PREFIX + "template_%s_%s_%s", database, branch, dateStr);
    }

    private void restoreDump(String database, String branch, String dateStr) {
        //System.out.println("entering restoreDump()");
        String templateName = getTemplateName(database, branch, dateStr);
        String connectionString = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement createStmt = conn.createStatement();
            ) {
            String createSql = String.format("CREATE DATABASE %s;", templateName);
            createStmt.execute(createSql);
            String url = String.format(branch.equals("master") ? QISDEV_HIS1_DUMP_URL : QISDEV_HIS1_DUMP_URL_BRANCH, branch.equals("master") ? "HEAD" : branch.toUpperCase());
            if ("eduetl".equals(database)) {
                url = String.format(QISDEV_EDUETL_DUMP_URL, "master".equals(branchName) ? "HEAD" : branchName.toUpperCase());
            }
            String[] dump_dl_command = {"curl", "-o", database + ".backup", url};
            ProcessBuilder dump_download = new ProcessBuilder(dump_dl_command);
            String[] dump_restore_command = {"pg_restore", "-h", SERVER, "-U", USER, "-d", templateName, database + ".backup"};
            ProcessBuilder dump_restore = new ProcessBuilder(dump_restore_command);
            dump_restore.environment().put("PGPASSWORD", PASSWORD);
            Process dl_proc = dump_download.start();
            try (InputStreamReader dl_proc_stdin = new InputStreamReader(dl_proc.getInputStream());
                            InputStreamReader dl_proc_stderr = new InputStreamReader(dl_proc.getErrorStream())) {
                String output = new BufferedReader(dl_proc_stdin).lines().collect(Collectors.joining("\n"));
                String err0 = new BufferedReader(dl_proc_stderr).lines().collect(Collectors.joining("\n"));
                System.out.println(output);
                System.out.println(err0);
            }
            Process rest_proc = dump_restore.start();
            try (InputStreamReader rest_proc_stderr = new InputStreamReader(rest_proc.getErrorStream())) {
                String err1 = new BufferedReader(rest_proc_stderr).lines().collect(Collectors.joining("\n"));
                System.out.println(err1);
            }
            String[] dump_rm_command = {"rm", database + ".backup"};
            ProcessBuilder dump_remove = new ProcessBuilder(dump_rm_command);
            dump_remove.start();
        } catch (SQLException e) {
            System.out.println("Schade, etwas ging bei der Datenbankanlage schief:");
            e.printStackTrace();
        } catch (IOException e) {
            System.out.println("Schade, etwas ging beim Datenbankeinspielen schief:");
            e.printStackTrace();
        }
        cleanupObsoleteTemplates(database, branch, dateStr);
        cleanupOldUserDbs(database);
    }
    
    private static void createHis1UnitTestDb(String database, String branch, String dateStr) {
        String connectionString = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement stmt = conn.createStatement();
            ) {
            String templateName = getTemplateName(database, branch, dateStr);
            String databaseName = "bi_" + database + "_" + branch;
            System.out.println(String.format("Erstelle %s ", databaseName));
            String createSql = String.format("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s TEMPLATE %s;", databaseName, databaseName, templateName);
            stmt.execute(createSql);
        } catch (SQLException e) {
            System.out.println("Schade, etwas ging bei der Datenbankanlage schief:");
            e.printStackTrace();
        }
    }

    private static void cleanupObsoleteTemplates(String database, String branch, String dateStr) {
        String connectionString = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement lookupStmt = conn.createStatement();
                Statement deleteStmt = conn.createStatement();
            ) {
                String lookupSql = String.format(
                                             "SELECT datname FROM pg_database WHERE datname LIKE '" + PREFIX + "template_%s_%s%%' AND NOT datname = '" + PREFIX + "template_%s_%s_%s'",
                                             database, branch, database, branch, dateStr );
                ResultSet result = lookupStmt.executeQuery(lookupSql);
                StringBuilder sb = new StringBuilder();
                while(result.next()) {
                    sb.append(String.format("DROP DATABASE %s;", result.getString("datname")));
                }
                String dropSql = sb.toString();
                deleteStmt.execute(dropSql);
        } catch (SQLException e) {
            System.out.println("Schade, etwas ging bei der Datenbankanlage schief:");
            e.printStackTrace();
        }
    }

    private static void cleanupOldUserDbs(String database) {
        String connectionString = CONNECTION_STRING;
        Properties props = dbConnectionProperties();
        try (
                Connection conn = DriverManager.getConnection(connectionString, props);
                Statement lookupStmt = conn.createStatement();
                Statement deleteStmt = conn.createStatement();
            ) {
            String lookupSql = String.format(
                                             "SELECT datname FROM pg_database WHERE datname LIKE '" + PREFIX + "%s_%%'",
                                             database);
                ResultSet result = lookupStmt.executeQuery(lookupSql);
                StringBuilder sb = new StringBuilder();
                while(result.next()) {
                    String dbName = result.getString("datname");
                    String[] nameParts = dbName.split("_");
                    String year = nameParts[nameParts.length - 6];
                    String month = nameParts[nameParts.length - 5];
                    String day = nameParts[nameParts.length - 4];
                    LocalDateTime today = LocalDateTime.now();
                    LocalDateTime then = LocalDate.parse(year + month + day, DateTimeFormatter.BASIC_ISO_DATE).atStartOfDay();
                    if (today.isAfter(then)) {
                        sb.append(String.format("DROP DATABASE %s;", dbName));
                    }
                }
                String dropSql = sb.toString();
                deleteStmt.execute(dropSql);
        } catch (SQLException e) {
            System.out.println("Schade, etwas ging bei der Bereinigung der alten Benutzer-Datenbanken schief:");
            e.printStackTrace();
        }
    }

    @Override
    public Map<String, String> dbtProfile() {
        Map<String, String> attributes = new HashMap<>();
        attributes.put("DBT_HOST", SERVER);
        attributes.put("DBT_PORT", "5432");
        attributes.put("DBT_USER", USER);
        attributes.put("DBT_ENV_SECRET_PW", PASSWORD);
        attributes.put("DBT_DB", getTestDbName(EDUETL_DB));
        return attributes;
    }

}