From 71822974d9c8b11c89b747a30f21695ad77a0dfe Mon Sep 17 00:00:00 2001 From: Andre Knieschewski Date: Wed, 16 Sep 2026 11:36:00 +0200 Subject: [PATCH] Demo Sonderladeroutine --- .../kern_sonderladeroutine_demo.sql | 492 ++++++++++++++++++ webapps_migration/migrate_superx_webapp.sh | 60 ++- 2 files changed, 530 insertions(+), 22 deletions(-) create mode 100644 sonderladeroutine/kern_sonderladeroutine_demo.sql diff --git a/sonderladeroutine/kern_sonderladeroutine_demo.sql b/sonderladeroutine/kern_sonderladeroutine_demo.sql new file mode 100644 index 0000000..e825fe7 --- /dev/null +++ b/sonderladeroutine/kern_sonderladeroutine_demo.sql @@ -0,0 +1,492 @@ +--Freemarker Template + +-- Demo einer SuperX-Sonderladeroutine. +-- +-- Ablauf: +-- 1. Einen kleinen Beispieldatensatz aus eduetl in eine UNL-Datei entladen. +-- 2. Eine einfache Zieltabelle in eduetl anlegen. +-- 3. Die UNL-Datei in diese Tabelle laden. +-- +-- Nach dem Lauf kann das Ergebnis geprüft werden mit: +-- select * from tmp_kern_special_demo; +-- +-- Beim erneuten Lauf leert der LOAD-Schritt die Tabelle wegen refresh=true. + +<#assign etl_jobs = [ +{"uniquename":"kern_special_demo_special","name":"Demo Sonderladeroutine","systeminfo_id":9,"logfile":""} +] /> + +<#assign etl_job_params = [ +] /> + +<#assign etl_steps = [ +{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_unload","name":"Demo-Daten entladen","type":"UNLOAD","parent":""}, +{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_create","name":"Demo-Zieltabelle anlegen","type":"DOQUERY","parent":""}, +{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_load","name":"Demo-Daten laden","type":"LOAD","parent":""} +] /> + +<#assign etl_step_properties = [ + +{"etl_step":"kern_special_demo_unload","prop_name":"datasource","prop_value":"eduetl"}, +{"etl_step":"kern_special_demo_unload","prop_name":"sqlVarSource","prop_value":"eduetl"}, +{"etl_step":"kern_special_demo_unload","prop_name":"sql","prop_value":"select 1 as id, ''Demo'' as name"}, +{"etl_step":"kern_special_demo_unload","prop_name":"unlFile","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"}, + +{"etl_step":"kern_special_demo_create","prop_name":"dataSource","prop_value":"eduetl"}, +{"etl_step":"kern_special_demo_create","prop_name":"sqlVarDataSource","prop_value":"eduetl"}, +{"etl_step":"kern_special_demo_create","prop_name":"sql","prop_value":"create table if not exists tmp_kern_special_demo (id integer, name varchar(20))"}, + +{"etl_step":"kern_special_demo_load","prop_name":"database","prop_value":"eduetl"}, +{"etl_step":"kern_special_demo_load","prop_name":"unlFilePath","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"}, +{"etl_step":"kern_special_demo_load","prop_name":"tableName","prop_value":"tmp_kern_special_demo"}, +{"etl_step":"kern_special_demo_load","prop_name":"delimiter","prop_value":"^"}, +{"etl_step":"kern_special_demo_load","prop_name":"header","prop_value":"false"}, +{"etl_step":"kern_special_demo_load","prop_name":"refresh","prop_value":"true"}, +{"etl_step":"kern_special_demo_load","prop_name":"usePostgresCopyApi","prop_value":"true"} + +] /> + +<#assign testfaelle = [ +] /> + + +<#macro etl_steps_def_check> + <#foreach step in etl_steps> + <#assign found=false/> + <#foreach job in etl_jobs> + <#if job.uniquename == step.etl_job> + <#assign found=true/> + + + <#if !found> + <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der ist aber nicht definiert!"> + + + <#switch step.type> + <#case "LOAD"> + <#case "UNLOAD"> + <#case "DOSQL"> + <#case "DOQUERY"> + <#case "MSG"> + <#break/> + <#default> + <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' hat unbekannten Typ '"+step.type+"'"> + + + + <#foreach step_prop in etl_step_properties> + <#assign found=false/> + <#foreach step in etl_steps> + <#if step.uniquename == step_prop.etl_step> + <#assign found=true/> + + + <#if !found> + <#stop "DEFINITIONSFEHLER: Eine Etl_step_property '"+step_prop.prop_name+ "' verweist auf etl_step '"+step_prop.etl_step+"' der ist aber nicht definiert!"> + + + + +<#function etl_step_lookup_systeminfo job_uniquename> + <#assign val=-1/> + <#foreach job in etl_jobs> + <#if job.uniquename == job_uniquename> + <#assign val=job.systeminfo_id/> + + + <#return val/> + + +<#function etl_properties_contains step propname> + <#assign val=false/> + <#foreach prop in etl_step_properties> + <#if prop.etl_step == step && prop.prop_name == propname> + <#assign val=true/> + + + <#return val/> + + +<#macro check_etl_properties step_uniquename propname defaultvalue> + <#if !etl_properties_contains(step_uniquename,propname)> + <#assign new_etl_step_properties = new_etl_step_properties + [{ + "etl_step": step_uniquename, + "prop_name": propname, + "prop_value": defaultvalue + }] /> + + + +<#macro ETL_STEPS_FUELLEN> + <@etl_steps_def_check/> + + <#assign new_etl_step_properties = etl_step_properties/> + <#foreach step in etl_steps> + <#if etl_step_lookup_systeminfo(step.etl_job) == -1> + <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der liefert aber keine systeminfo_id"> + + + <@check_etl_properties step_uniquename=step.uniquename propname="systemInfoId" defaultvalue=etl_step_lookup_systeminfo(step.etl_job)/> + <@check_etl_properties step_uniquename=step.uniquename propname="active" defaultvalue="true"/> + <@check_etl_properties step_uniquename=step.uniquename propname="custom" defaultvalue="true"/> + + <#switch step.type> + <#case "UNLOAD"> + <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/> + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarSource" defaultvalue="eduetl"/> + <#break> + <#case "LOAD"> + <@check_etl_properties step_uniquename=step.uniquename propname="database" defaultvalue="eduetl"/> + <@check_etl_properties step_uniquename=step.uniquename propname="delimiter" defaultvalue="^"/> + <@check_etl_properties step_uniquename=step.uniquename propname="header" defaultvalue="false"/> + <@check_etl_properties step_uniquename=step.uniquename propname="refresh" defaultvalue="true"/> + <@check_etl_properties step_uniquename=step.uniquename propname="usePostgresCopyApi" defaultvalue="true"/> + <#break> + <#case "DOQUERY"> + <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/> + <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/> + <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/> + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/> + <#break> + <#case "DOSQL"> + <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/> + <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/> + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/> + <@check_etl_properties step_uniquename=step.uniquename propname="limitDebugStmts" defaultvalue="-1"/> + <#break> + <#default> + + + + <#assign etl_step_properties = new_etl_step_properties/> + +--Bezeichnungen aktualisieren +<#foreach etl_job in etl_jobs> +update etl_job set caption='${etl_job.name}' where uniquename='${etl_job.uniquename}'; + +<#foreach etl_step in etl_steps> +update etl_step set caption='${etl_step.name}' where uniquename='${etl_step.uniquename}' and tid in +(select step_id from etl_step_relation where job_id in (select tid from etl_job where uniquename='${etl_step.etl_job}')); + + +create temp table tmp_etl_step( +tid INTEGER, +uniquename VARCHAR(255), +caption VARCHAR(255), +systeminfo_id INTEGER not null, +step_type INTEGER, +step_type_uniquename VARCHAR(255), +sortnr SMALLINT not null, +force_continue SMALLINT, +etl_job_id INTEGER, +parent_step_id INTEGER, +parent_step_uniquename varchar(255), +parent_job_uniquename varchar(255), +logfile varchar(255), +custom_step smallint, +already_exists smallint +); + +create temp table tmp_etl_job( +tid INTEGER, +uniquename VARCHAR(255), +caption VARCHAR(255), +systeminfo_id INTEGER not null, +logfile varchar(255), +already_exists smallint, +custom_job smallint +); + +create temp table tmp_etl_job_param( +tid SERIAL not null, +etl_job_id INTEGER, +uniquename VARCHAR(255) not null, +name VARCHAR(255), +param_default VARCHAR(255) +); + +create temp table tmp_etl_step_property( +tid SERIAL not null, +etl_step_id INTEGER not null, +prop_name VARCHAR(255), +prop_value text +); + +create temp table tmp_etl_step_relation( +tid SERIAL not null, +step_id INTEGER not null, +parent_step_id INTEGER, +job_id INTEGER not null, +force_continue SMALLINT default 1, +step_active SMALLINT default 1, +sortnr SMALLINT default 1, +custom_step SMALLINT default 1 +); + +update etl_job set custom_job=2 +where custom_job=0 +and uniquename in ('sx_select_mask','sx_insert_mask', +'kern_konstanten_update', +'sichten_unload', +'sos_gewichtung_unload'); + +update etl_step set custom_step=2 +where custom_step=0 +and tid in (select R.step_id from etl_step_relation R, etl_job J +where R.job_id=J.tid +and J.uniquename in ('sx_select_mask','sx_insert_mask', +'kern_konstanten_update', +'sichten_unload', +'sos_gewichtung_unload')); + +update etl_step_relation set custom_step=2 +where custom_step=0 +and job_id in (select J.tid +from etl_job J +where J.uniquename in ('sx_select_mask','sx_insert_mask', +'kern_konstanten_update', +'sichten_unload', +'sos_gewichtung_unload')); + +<#foreach etl_job in etl_jobs> + +truncate table tmp_etl_job; +truncate table tmp_etl_step; +truncate table tmp_etl_job_param; +truncate table tmp_etl_step_property; +truncate table tmp_etl_step_relation; + +insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists,custom_job) +values ('${etl_job.uniquename}', +'${etl_job.name}', +${etl_job.systeminfo_id}, +'${etl_job.logfile}',0,2); + +update tmp_etl_job set tid=J.tid, +already_exists=1, +caption=J.caption, +logfile=J.logfile, +custom_job=J.custom_job +from etl_job J +where J.uniquename=tmp_etl_job.uniquename +and J.systeminfo_id=tmp_etl_job.systeminfo_id; + +insert into etl_job(uniquename,caption,systeminfo_id,custom_job) +select uniquename,caption,systeminfo_id,custom_job +from tmp_etl_job +where already_exists=0; + +update tmp_etl_job set tid=(select J.tid +from etl_job J +where J.uniquename=tmp_etl_job.uniquename +and J.systeminfo_id=tmp_etl_job.systeminfo_id) +where already_exists=0; + +<#foreach etl_job_param in etl_job_params> + <#if etl_job_param.etl_job == etl_job.uniquename> +insert into tmp_etl_job_param( +etl_job_id, +uniquename, +name, +param_default) +select J.tid, +'${etl_job_param.param_name}', +'${etl_job_param.name}', +'${etl_job_param.param_default}' +from tmp_etl_job J; + + + +<#assign sortnr=0/> +<#foreach etl_step in etl_steps> + <#if etl_step.etl_job == etl_job.uniquename> + <#assign sortnr=sortnr+1/> + +insert into tmp_etl_step( +uniquename, +caption, +systeminfo_id, +step_type_uniquename, +sortnr, +force_continue, +etl_job_id, +parent_step_uniquename, +parent_job_uniquename, +logfile, +custom_step, +already_exists +) +select '${etl_step.uniquename}', +'${etl_step.name}', +${etl_job.systeminfo_id}, +'${etl_step.type}' as step_type_uniquename, +${sortnr}*10 as sortnr, +0 as force_continue, +J.tid as etl_job_id, +<#if etl_step.parent?exists && etl_step.parent != ""> +'${etl_step.parent}', +<#else> +'' as parent_step_uniquename, + +J.uniquename, +'${etl_job.logfile}' as logfile, +2, +0 +from etl_job J +where J.uniquename='${etl_job.uniquename}' +and J.systeminfo_id=${etl_job.systeminfo_id}; + + + + +delete from etl_job_param +where etl_job_id in ( +select distinct etl_job_id +from tmp_etl_job_param +); + +insert into etl_job_param +( +etl_job_id, +uniquename, +name, +param_default +) +select +etl_job_id, +uniquename, +name, +param_default +from tmp_etl_job_param; + +update tmp_etl_step set step_type=(select T.tid from etl_step_type T +where T.uniquename=tmp_etl_step.step_type_uniquename); + +update tmp_etl_step set tid=S.tid, +already_exists=1, +caption=S.caption, +logfile=S.logfile, +custom_step=S.custom_step +from etl_step S +where S.uniquename=tmp_etl_step.uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id; + +insert into etl_step( +uniquename, +caption, +systeminfo_id, +step_type_id, +logfile, +custom_step) +select +uniquename, +caption, +systeminfo_id, +step_type, +logfile, +custom_step +from tmp_etl_step +where already_exists=0; + +update tmp_etl_step set tid=(select S.tid +from etl_step S +where S.uniquename=tmp_etl_step.uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id) +where already_exists=0; + +update tmp_etl_step set parent_step_id=(select S.tid +from etl_step S +where S.uniquename=tmp_etl_step.parent_step_uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id); + +delete from etl_step_property +where etl_step_id in (select T.tid from tmp_etl_step T); + +delete from etl_step_relation +where job_id in (select J.tid from tmp_etl_job J) +and custom_step=2; + +<#foreach etl_step_property in etl_step_properties> +insert into tmp_etl_step_property( +etl_step_id, +prop_name) +select +T.tid as etl_step_id, +'${etl_step_property.prop_name}' +from tmp_etl_step T +where uniquename='${etl_step_property.etl_step}'; + + +insert into etl_step_property( +etl_step_id, +prop_name) +select +T.etl_step_id, +T.prop_name +from tmp_etl_step_property T; + +insert into tmp_etl_step_relation( +step_id, +parent_step_id, +job_id, +force_continue, +step_active, +sortnr, +custom_step +) +select +tid, +parent_step_id, +etl_job_id, +0 as force_continue, +1 as step_active, +sortnr, +2 +from tmp_etl_step; + +insert into etl_step_relation( +step_id, +parent_step_id, +job_id, +force_continue, +step_active, +sortnr, +custom_step +) +select +step_id, +parent_step_id, +job_id, +force_continue, +step_active, +sortnr, +custom_step +from tmp_etl_step_relation; + + + +drop table tmp_etl_step; +drop table tmp_etl_job; +drop table tmp_etl_job_param; +drop table tmp_etl_step_property; +drop table tmp_etl_step_relation; + +<#foreach testfall in testfaelle> +select 'testfall ${testfall.testcase}: ERFOLG' +from xdummy +where ${testfall.assertion}=(${testfall.sql}); +select 'testfall ${testfall.testcase}: FEHLER bei ${testfall.testcase}' +from xdummy +where ${testfall.assertion}!=(${testfall.sql}); + + +<#foreach etl_step_property in etl_step_properties> +update etl_step_property +set prop_value='${etl_step_property.prop_value}' +where prop_name='${etl_step_property.prop_name}' +and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step}'); + + + + +<@ETL_STEPS_FUELLEN/> diff --git a/webapps_migration/migrate_superx_webapp.sh b/webapps_migration/migrate_superx_webapp.sh index a9b87ee..4e3f4e3 100755 --- a/webapps_migration/migrate_superx_webapp.sh +++ b/webapps_migration/migrate_superx_webapp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # migrate_superx_webapp.sh -# Version 1.5 +# Version 1.7 # # Migriert eine bestehende SuperX-Installation in eine neue Webapp-Struktur. # Standardziel: /home/superx/webapps/superx @@ -9,7 +9,7 @@ set -euo pipefail SCRIPT_NAME="$(basename "$0")" -SCRIPT_VERSION="1.6" +SCRIPT_VERSION="1.7" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_FILE="$SCRIPT_DIR/migrate_superx.conf" @@ -27,6 +27,7 @@ group_exists() { getent group "$1" >/dev/null 2>&1; } user_in_group() { id -nG "$1" 2>/dev/null | tr ' ' '\n' | grep -qx "$2"; } current_user_matches_target() { [ "$(current_user)" = "$WEBAPP_USER" ] && user_in_group "$(current_user)" "$WEBAPP_GROUP"; } canon_path() { if command -v realpath >/dev/null 2>&1; then realpath -m "$1"; else printf '%s\n' "$1"; fi; } +path_exists() { [ -e "$1" ] || [ -L "$1" ]; } run() { if is_true "${DRY_RUN:-false}"; then @@ -52,29 +53,30 @@ USAGE load_config() { [ -f "$CONFIG_FILE" ] || { usage; fail "Config-Datei nicht gefunden: $CONFIG_FILE"; } - # shellcheck disable=SC1090 - - if [ "$SQL_ENV" == "" ] - then - SQL_ENV=`which SQL_ENV` - - fi - if [ "$TARGET_WEBAPP" == "" ] - then - TARGET_WEBAPP=$WEBAPP - - fi echo "Lade $CONFIG_FILE" + # shellcheck disable=SC1090 . "$CONFIG_FILE" - #echo $SQL_ENV - : "${SQL_ENV:?SQL_ENV ist nicht gesetzt}" + + MIGRATION_MODE="${MIGRATION_MODE:-db_only}" + DB_TRANSFER_MODE="${DB_TRANSFER_MODE:-move}" + + if [ -z "${SQL_ENV:-}" ]; then + SQL_ENV="$(command -v SQL_ENV 2>/dev/null || true)" + fi + + if [ -z "${TARGET_WEBAPP:-}" ]; then + if [ "$MIGRATION_MODE" = "db_only" ]; then + TARGET_WEBAPP="auto" + else + TARGET_WEBAPP="/home/superx/webapps/superx" + fi + fi + + : "${SQL_ENV:?SQL_ENV ist nicht gesetzt und konnte nicht im PATH gefunden werden}" : "${TARGET_WEBAPP:?TARGET_WEBAPP ist nicht gesetzt}" : "${WEBAPP_USER:?WEBAPP_USER ist nicht gesetzt}" : "${WEBAPP_GROUP:?WEBAPP_GROUP ist nicht gesetzt}" - - MIGRATION_MODE="${MIGRATION_MODE:-full}" - DB_TRANSFER_MODE="${DB_TRANSFER_MODE:-copy}" STOP_TOMCAT="${STOP_TOMCAT:-false}" START_TOMCAT="${START_TOMCAT:-false}" TOMCAT_SERVICE="${TOMCAT_SERVICE:-tomcat10}" @@ -320,8 +322,18 @@ validate_target() { [ "$SOURCE_DB_REAL" = "$TARGET_DB_REAL" ] && fail "SOURCE_DB und TARGET_DB sind identisch." case "$TARGET_DB_REAL" in "$SOURCE_DB_REAL"/*) fail "TARGET_DB liegt innerhalb von SOURCE_DB.";; esac case "$SOURCE_DB_REAL" in "$TARGET_DB_REAL"/*) fail "SOURCE_DB liegt innerhalb von TARGET_DB.";; esac - [ "$DB_TRANSFER_MODE" = "move" ] && [ -e "$TARGET_DB" ] && fail "DB_TRANSFER_MODE=move, aber TARGET_DB existiert bereits: $TARGET_DB" - [ "$DB_TRANSFER_MODE" = "copy" ] && [ -e "$TARGET_DB" ] && ! is_true "$ALLOW_EXISTING_TARGET_DB" && fail "TARGET_DB existiert bereits: $TARGET_DB" + if path_exists "$TARGET_DB"; then + if [ "$DB_TRANSFER_MODE" = "move" ]; then + ! is_true "$ALLOW_EXISTING_TARGET_DB" && fail "TARGET_DB existiert bereits und ALLOW_EXISTING_TARGET_DB=false: $TARGET_DB" + [ -d "$TARGET_DB" ] || [ -L "$TARGET_DB" ] || fail "TARGET_DB existiert, ist aber weder Verzeichnis noch Symlink: $TARGET_DB" + if command -v mountpoint >/dev/null 2>&1 && mountpoint -q "$TARGET_DB"; then + fail "TARGET_DB ist ein Mountpoint und wird nicht automatisch ersetzt: $TARGET_DB" + fi + warn "TARGET_DB existiert bereits und wird bei DB_TRANSFER_MODE=move vollstaendig ersetzt: $TARGET_DB" + elif ! is_true "$ALLOW_EXISTING_TARGET_DB"; then + fail "TARGET_DB existiert bereits: $TARGET_DB" + fi + fi if is_true "$CREATE_TOMCAT_SYMLINK" && { [ -e "$TOMCAT_SYMLINK_PATH" ] || [ -L "$TOMCAT_SYMLINK_PATH" ]; }; then if [ -L "$TOMCAT_SYMLINK_PATH" ] && is_true "$REPLACE_EXISTING_SYMLINK"; then :; else fail "Tomcat-Symlink/Ziel existiert bereits: $TOMCAT_SYMLINK_PATH" @@ -363,7 +375,11 @@ transfer_db() { else log "Verschiebe DB-Verzeichnis nach WEB-INF/conf/edustore/db..." run "mkdir -p '$(dirname "$TARGET_DB")'" - run "mv '$SOURCE_DB' '$TARGET_DB'" + if path_exists "$TARGET_DB"; then + log "Entferne bestehendes TARGET_DB vor dem Verschieben..." + run "rm -rf -- '$TARGET_DB'" + fi + run "mv -- '$SOURCE_DB' '$TARGET_DB'" fi }