package de.superx; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.InetAddress; import java.net.UnknownHostException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.sql.DataSource; import org.junit.Ignore; import com.zaxxer.hikari.HikariDataSource; @Ignore public class TestDbManagerPg implements TestDbManager { // TODO: Hier sollten ALLE Test-DBs definiert werden // Momentan wird aber nur eduetl für die Tests verwendet protected static final String[] TEST_DBS = new String[] { EDUETL_DB, HISINONE_DB }; protected static final String LOCAL = "localhost"; protected static final String USER = "postgres"; protected static final String PASSWORD = "postgres"; private static final String QISDEV_HIS1_DUMP_URL = "http://qisdev.his.de/databaseBackups/hio_%s-stable,audit,develop,tests_qamuster.backup"; private static final String QISDEV_HIS1_DUMP_URL_BRANCH = "http://qisdev.his.de/databaseBackups/hio_%s-stable,audit_qamuster.backup"; private static final String QISDEV_EDUETL_DUMP_URL = "http://qisdev.his.de/Distributionen/BI-Musterdaten/%s/eduetl_muster_update.backup"; private static Pattern BRANCH_NAME_PATTERN = Pattern.compile("^(release\\_\\d\\d\\d\\d\\_\\d\\d)|(master)$"); private static final String VMPOSTGRES = "vmpostgres17.his.de"; private static final String SERVER = VMPOSTGRES; private static final String CONNECTION_STRING_PREFIX = String.format( "jdbc:postgresql://%s/", SERVER); private static final String PREFIX = "bi_tests_"; protected static final String CONNECTION_STRING = String.format("%spostgres", CONNECTION_STRING_PREFIX); private String branchName; private Map testDbNames = new HashMap<>(); public TestDbManagerPg() { String branch = getBranchName(); System.out.println(String.format("Branch name über git gefunden: %s", branch)); } @Override public void setupTestDbs() { String connectionString = CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement stmt = conn.createStatement(); Statement deleteStmt = conn.createStatement(); ) { // "Legacy": Dumps von den Builds auf // qa.his.de und qisdev.his.de holen: // ensureLatestQaDumps(); // "Neu": Dumps von qisdev direkt herunterladen: LocalDateTime ld = LocalDateTime.now().minusHours(19).minusMinutes(30); String dateStr = ld.format(DateTimeFormatter.BASIC_ISO_DATE); ensureLatestQisdevDumps(dateStr); // TODO: Hier sollte über ALLE Test-DBs iteriert werden // Momentan wird aber nur eduetl für die Tests verwendet for (String dbName : TEST_DBS) { String lookupSql = String.format( "SELECT datname FROM pg_database WHERE datname = '" + getTemplateName(dbName, branchName, dateStr) + "';"); ResultSet result = stmt.executeQuery(lookupSql); if (!result.next()) { throw new RuntimeException("could not set up test database, because no db template was found :´("); } String templateName = result.getString("datname"); String databaseName = getTestDbName(dbName); String createSql = String.format("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s TEMPLATE %s;", databaseName, databaseName, templateName); stmt.execute(createSql); } } catch (SQLException e) { System.out.println("Schade, etwas ging bei der Datenbankanlage schief:"); e.printStackTrace(); } /* * workaround wegen vorübergehend "schräger" Musterdaten - zum Vergleich: * https://hiszilla.his.de/hiszilla/show_bug.cgi?id=364365 * wenn das Erfassungssystem auf Version 2026.06 ist und die Daten dort * angepasst wurden, _sollte_ der workaround entfernt werden können... */ String eduetlName = getTestDbName(EDUETL_DB); connectionString = String.format("%s%s", CONNECTION_STRING_PREFIX, eduetlName); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement stmt = conn.createStatement(); ) { String mySql = "UPDATE sos_stg_aggr SET fach_sem_zahl = abs(fach_sem_zahl);"; stmt.execute(mySql); } catch (SQLException e) { System.out.println("Beim Anpassungen der eduetl-Datenbank ging etwas schief:"); e.printStackTrace(); } // workaround Ende } @Override public DataSource getDataSource(String logicalDatabaseName) { HikariDataSource ds = new HikariDataSource(); ds.setJdbcUrl(getJdbcUrl(logicalDatabaseName)); ds.setUsername(TestDbManagerPg.USER); ds.setPassword(TestDbManagerPg.PASSWORD); ds.setConnectionInitSql("SET standard_conforming_strings TO 'off'; SET datestyle TO 'ISO, DMY';"); if ("hisinone".equals(logicalDatabaseName)) { ds.setSchema("hisinone"); } return ds; } @Override public void removeTestDbs() { String connectionString = TestDbManagerPg.CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement stmt = conn.createStatement(); ) { for (String dbName : TEST_DBS) { String databaseName = getTestDbName(dbName); String dropSql = String.format("DROP DATABASE IF EXISTS %s;", databaseName); stmt.execute(dropSql); } } catch (SQLException e) { System.out.println("Problem bei Bereinigung der Testdatenbank:"); e.printStackTrace(); } } protected void ensureLatestQisdevDumps(String dateStr) throws SQLException { //System.out.println("entering ensureLatestQisdevDumps()"); /* Warum werden hier 19:30 abgezogen? * Die Klasse spielt Dumps ein und macht daraus template-Datenbanken, * aus welchen dann Testdatenbanken erzeugt werden. * Das sollte möglichst während der nächtlichen Tests passieren, weil es lange dauert, * beim Entwickeln will man nicht, dass sowas passiert. * Um nicht auf veralteten Daten zu arbeiten, kommt ein Timestamp des Erzeugungszeitpunktes * in den Templatenamen - Bsp. "2025_09_04". * Man weiss aber nicht sicher, ob das Template während des Builds vor oder nach Mitternacht * erzeugt wird. Wenn das vor Mitternacht passiert und dann nach Mitternacht plötzlich den * Timestamp des Vortages hat, würde es bereits als veraltet gelten und bei der ersten * Ausführung eines Testfalls müsste man auf den Neuaufbau warten. * Daher die Lösung: * Man zieht bei der Erzeugung des Timestampt noch 19:30 ab; so wechselt der Timestamp erst * nach Feierabend auf den nächsten Tag. * */ if( !templateExists(EDUETL_DB, branchName, dateStr)) { System.out.println(String.format("kein %s template vorhanden...erstelle", EDUETL_DB)); restoreDump(EDUETL_DB, branchName, dateStr); createHis1UnitTestDb(EDUETL_DB, branchName, dateStr); } else { System.out.println(String.format("%s template bereits vorhanden...", EDUETL_DB)); } if( !templateExists(HISINONE_DB, branchName, dateStr)) { System.out.println(String.format("kein %s template vorhanden...erstelle", HISINONE_DB)); restoreDump(HISINONE_DB, branchName, dateStr); createHis1UnitTestDb(HISINONE_DB, branchName, dateStr); } else { System.out.println(String.format("%s template bereits vorhanden...", HISINONE_DB)); } } protected String getTestDbName(String name) { if ( !this.testDbNames.containsKey(name) ) { SimpleDateFormat dateformat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss"); String timestamp = dateformat.format(new Date()); try { this.testDbNames.put(name, String.format(PREFIX + "%s_%s_%s_%s", name, InetAddress.getLocalHost().getHostName().replace("-", "_"), branchName, timestamp)); } catch (UnknownHostException e) { e.printStackTrace(); } } return this.testDbNames.get(name); } protected String getJdbcUrl(String name) { return CONNECTION_STRING_PREFIX + getTestDbName(name); } protected static Properties dbConnectionProperties() { //System.out.println("entering dbConnectionProperties()"); Properties props = new Properties(); props.setProperty("user", TestDbManagerPg.USER); props.setProperty("password", TestDbManagerPg.PASSWORD); props.setProperty("ssl", "false"); return props; } private String getBranchName() { if ( this.branchName == null ) { String[] branch_name_command = {"git", "branch", "--show-current"}; ProcessBuilder branch_name = new ProcessBuilder(branch_name_command); try { Process branch_name_proc; branch_name_proc = branch_name.start(); try (InputStreamReader branch_name_proc_stderr = new InputStreamReader(branch_name_proc.getErrorStream()); InputStreamReader branch_name_proc_stdout = new InputStreamReader(branch_name_proc.getInputStream())) { String err0 = new BufferedReader(branch_name_proc_stderr).lines().collect(Collectors.joining("\n")); String output = new BufferedReader(branch_name_proc_stdout).lines().collect(Collectors.joining("\n")); String branchNameFound = output.trim().toLowerCase(); Matcher matcher = BRANCH_NAME_PATTERN.matcher(branchNameFound); if ( matcher.find() ) { System.out.println(String.format("git branch '%s' für Auswahl der Test DB gefunden", branchNameFound)); this.branchName = branchNameFound; } else { System.out.println("konnte keinen zu db dumps passenden git branch finden - benutze 'master'"); this.branchName = "master"; } } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } return this.branchName; } private static boolean templateExists(String database, String branch, String dateStr) throws SQLException { //System.out.println("entering templateExists()"); String templateName = getTemplateName(database, branch, dateStr); String url = CONNECTION_STRING; Properties props = dbConnectionProperties(); try (Connection conn = DriverManager.getConnection(url, props); Statement stmt = conn.createStatement();) { String sql = String.format("SELECT datname FROM pg_database WHERE datname = '%s';", templateName); ResultSet result = stmt.executeQuery(sql); return result.next(); } catch (SQLException e) { System.out.println("Konnte die Buildnummer des Templates von '" + database + "' nicht prüfen:"); e.printStackTrace(); throw e; } } private static String getTemplateName(String database, String branch, String dateStr) { return String.format(PREFIX + "template_%s_%s_%s", database, branch, dateStr); } private void restoreDump(String database, String branch, String dateStr) { //System.out.println("entering restoreDump()"); String templateName = getTemplateName(database, branch, dateStr); String connectionString = CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement createStmt = conn.createStatement(); ) { String createSql = String.format("CREATE DATABASE %s;", templateName); createStmt.execute(createSql); String url = String.format(branch.equals("master") ? QISDEV_HIS1_DUMP_URL : QISDEV_HIS1_DUMP_URL_BRANCH, branch.equals("master") ? "HEAD" : branch.toUpperCase()); if ("eduetl".equals(database)) { url = String.format(QISDEV_EDUETL_DUMP_URL, "master".equals(branchName) ? "HEAD" : branchName.toUpperCase()); } String[] dump_dl_command = {"curl", "-o", database + ".backup", url}; ProcessBuilder dump_download = new ProcessBuilder(dump_dl_command); String[] dump_restore_command = {"pg_restore", "-h", SERVER, "-U", USER, "-d", templateName, database + ".backup"}; ProcessBuilder dump_restore = new ProcessBuilder(dump_restore_command); dump_restore.environment().put("PGPASSWORD", PASSWORD); Process dl_proc = dump_download.start(); try (InputStreamReader dl_proc_stdin = new InputStreamReader(dl_proc.getInputStream()); InputStreamReader dl_proc_stderr = new InputStreamReader(dl_proc.getErrorStream())) { String output = new BufferedReader(dl_proc_stdin).lines().collect(Collectors.joining("\n")); String err0 = new BufferedReader(dl_proc_stderr).lines().collect(Collectors.joining("\n")); System.out.println(output); System.out.println(err0); } Process rest_proc = dump_restore.start(); try (InputStreamReader rest_proc_stderr = new InputStreamReader(rest_proc.getErrorStream())) { String err1 = new BufferedReader(rest_proc_stderr).lines().collect(Collectors.joining("\n")); System.out.println(err1); } String[] dump_rm_command = {"rm", database + ".backup"}; ProcessBuilder dump_remove = new ProcessBuilder(dump_rm_command); dump_remove.start(); } catch (SQLException e) { System.out.println("Schade, etwas ging bei der Datenbankanlage schief:"); e.printStackTrace(); } catch (IOException e) { System.out.println("Schade, etwas ging beim Datenbankeinspielen schief:"); e.printStackTrace(); } cleanupObsoleteTemplates(database, branch, dateStr); cleanupOldUserDbs(database); } private static void createHis1UnitTestDb(String database, String branch, String dateStr) { String connectionString = CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement stmt = conn.createStatement(); ) { String templateName = getTemplateName(database, branch, dateStr); String databaseName = "bi_" + database + "_" + branch; System.out.println(String.format("Erstelle %s ", databaseName)); String createSql = String.format("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s TEMPLATE %s;", databaseName, databaseName, templateName); stmt.execute(createSql); } catch (SQLException e) { System.out.println("Schade, etwas ging bei der Datenbankanlage schief:"); e.printStackTrace(); } } private static void cleanupObsoleteTemplates(String database, String branch, String dateStr) { String connectionString = CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement lookupStmt = conn.createStatement(); Statement deleteStmt = conn.createStatement(); ) { String lookupSql = String.format( "SELECT datname FROM pg_database WHERE datname LIKE '" + PREFIX + "template_%s_%s%%' AND NOT datname = '" + PREFIX + "template_%s_%s_%s'", database, branch, database, branch, dateStr ); ResultSet result = lookupStmt.executeQuery(lookupSql); StringBuilder sb = new StringBuilder(); while(result.next()) { sb.append(String.format("DROP DATABASE %s;", result.getString("datname"))); } String dropSql = sb.toString(); deleteStmt.execute(dropSql); } catch (SQLException e) { System.out.println("Schade, etwas ging bei der Datenbankanlage schief:"); e.printStackTrace(); } } private static void cleanupOldUserDbs(String database) { String connectionString = CONNECTION_STRING; Properties props = dbConnectionProperties(); try ( Connection conn = DriverManager.getConnection(connectionString, props); Statement lookupStmt = conn.createStatement(); Statement deleteStmt = conn.createStatement(); ) { String lookupSql = String.format( "SELECT datname FROM pg_database WHERE datname LIKE '" + PREFIX + "%s_%%'", database); ResultSet result = lookupStmt.executeQuery(lookupSql); StringBuilder sb = new StringBuilder(); while(result.next()) { String dbName = result.getString("datname"); String[] nameParts = dbName.split("_"); String year = nameParts[nameParts.length - 6]; String month = nameParts[nameParts.length - 5]; String day = nameParts[nameParts.length - 4]; LocalDateTime today = LocalDateTime.now(); LocalDateTime then = LocalDate.parse(year + month + day, DateTimeFormatter.BASIC_ISO_DATE).atStartOfDay(); if (today.isAfter(then)) { sb.append(String.format("DROP DATABASE %s;", dbName)); } } String dropSql = sb.toString(); deleteStmt.execute(dropSql); } catch (SQLException e) { System.out.println("Schade, etwas ging bei der Bereinigung der alten Benutzer-Datenbanken schief:"); e.printStackTrace(); } } @Override public Map dbtProfile() { Map attributes = new HashMap<>(); attributes.put("DBT_HOST", SERVER); attributes.put("DBT_PORT", "5432"); attributes.put("DBT_USER", USER); attributes.put("DBT_ENV_SECRET_PW", PASSWORD); attributes.put("DBT_DB", getTestDbName(EDUETL_DB)); return attributes; } }