SuperX
/
BI_Maintenance
7
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity
Browse Source

Demo Sonderladeroutine

master
Andre Knieschewski 1 month ago
parent
bf741e8de5
commit
71822974d9
2 changed files with 530 additions and 22 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 492
      sonderladeroutine/kern_sonderladeroutine_demo.sql
  2. 60
      webapps_migration/migrate_superx_webapp.sh

492
sonderladeroutine/kern_sonderladeroutine_demo.sql
Unescape View File

@ -0,0 +1,492 @@ @@ -0,0 +1,492 @@
--Freemarker Template
-- Demo einer SuperX-Sonderladeroutine.
--
-- Ablauf:
-- 1. Einen kleinen Beispieldatensatz aus eduetl in eine UNL-Datei entladen.
-- 2. Eine einfache Zieltabelle in eduetl anlegen.
-- 3. Die UNL-Datei in diese Tabelle laden.
--
-- Nach dem Lauf kann das Ergebnis geprüft werden mit:
-- select * from tmp_kern_special_demo;
--
-- Beim erneuten Lauf leert der LOAD-Schritt die Tabelle wegen refresh=true.
<#assign etl_jobs = [
{"uniquename":"kern_special_demo_special","name":"Demo Sonderladeroutine","systeminfo_id":9,"logfile":""}
] />
<#assign etl_job_params = [
] />
<#assign etl_steps = [
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_unload","name":"Demo-Daten entladen","type":"UNLOAD","parent":""},
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_create","name":"Demo-Zieltabelle anlegen","type":"DOQUERY","parent":""},
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_load","name":"Demo-Daten laden","type":"LOAD","parent":""}
] />
<#assign etl_step_properties = [
{"etl_step":"kern_special_demo_unload","prop_name":"datasource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_unload","prop_name":"sqlVarSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_unload","prop_name":"sql","prop_value":"select 1 as id, ''Demo'' as name"},
{"etl_step":"kern_special_demo_unload","prop_name":"unlFile","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"},
{"etl_step":"kern_special_demo_create","prop_name":"dataSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_create","prop_name":"sqlVarDataSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_create","prop_name":"sql","prop_value":"create table if not exists tmp_kern_special_demo (id integer, name varchar(20))"},
{"etl_step":"kern_special_demo_load","prop_name":"database","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_load","prop_name":"unlFilePath","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"},
{"etl_step":"kern_special_demo_load","prop_name":"tableName","prop_value":"tmp_kern_special_demo"},
{"etl_step":"kern_special_demo_load","prop_name":"delimiter","prop_value":"^"},
{"etl_step":"kern_special_demo_load","prop_name":"header","prop_value":"false"},
{"etl_step":"kern_special_demo_load","prop_name":"refresh","prop_value":"true"},
{"etl_step":"kern_special_demo_load","prop_name":"usePostgresCopyApi","prop_value":"true"}
] />
<#assign testfaelle = [
] />
<#macro etl_steps_def_check>
<#foreach step in etl_steps>
<#assign found=false/>
<#foreach job in etl_jobs>
<#if job.uniquename == step.etl_job>
<#assign found=true/>
</#if>
</#foreach>
<#if !found>
<#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der ist aber nicht definiert!">
</#if>
<#switch step.type>
<#case "LOAD">
<#case "UNLOAD">
<#case "DOSQL">
<#case "DOQUERY">
<#case "MSG">
<#break/>
<#default>
<#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' hat unbekannten Typ '"+step.type+"'">
</#switch>
</#foreach>
<#foreach step_prop in etl_step_properties>
<#assign found=false/>
<#foreach step in etl_steps>
<#if step.uniquename == step_prop.etl_step>
<#assign found=true/>
</#if>
</#foreach>
<#if !found>
<#stop "DEFINITIONSFEHLER: Eine Etl_step_property '"+step_prop.prop_name+ "' verweist auf etl_step '"+step_prop.etl_step+"' der ist aber nicht definiert!">
</#if>
</#foreach>
</#macro>
<#function etl_step_lookup_systeminfo job_uniquename>
<#assign val=-1/>
<#foreach job in etl_jobs>
<#if job.uniquename == job_uniquename>
<#assign val=job.systeminfo_id/>
</#if>
</#foreach>
<#return val/>
</#function>
<#function etl_properties_contains step propname>
<#assign val=false/>
<#foreach prop in etl_step_properties>
<#if prop.etl_step == step && prop.prop_name == propname>
<#assign val=true/>
</#if>
</#foreach>
<#return val/>
</#function>
<#macro check_etl_properties step_uniquename propname defaultvalue>
<#if !etl_properties_contains(step_uniquename,propname)>
<#assign new_etl_step_properties = new_etl_step_properties + [{
"etl_step": step_uniquename,
"prop_name": propname,
"prop_value": defaultvalue
}] />
</#if>
</#macro>
<#macro ETL_STEPS_FUELLEN>
<@etl_steps_def_check/>
<#assign new_etl_step_properties = etl_step_properties/>
<#foreach step in etl_steps>
<#if etl_step_lookup_systeminfo(step.etl_job) == -1>
<#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der liefert aber keine systeminfo_id">
</#if>
<@check_etl_properties step_uniquename=step.uniquename propname="systemInfoId" defaultvalue=etl_step_lookup_systeminfo(step.etl_job)/>
<@check_etl_properties step_uniquename=step.uniquename propname="active" defaultvalue="true"/>
<@check_etl_properties step_uniquename=step.uniquename propname="custom" defaultvalue="true"/>
<#switch step.type>
<#case "UNLOAD">
<@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarSource" defaultvalue="eduetl"/>
<#break>
<#case "LOAD">
<@check_etl_properties step_uniquename=step.uniquename propname="database" defaultvalue="eduetl"/>
<@check_etl_properties step_uniquename=step.uniquename propname="delimiter" defaultvalue="^"/>
<@check_etl_properties step_uniquename=step.uniquename propname="header" defaultvalue="false"/>
<@check_etl_properties step_uniquename=step.uniquename propname="refresh" defaultvalue="true"/>
<@check_etl_properties step_uniquename=step.uniquename propname="usePostgresCopyApi" defaultvalue="true"/>
<#break>
<#case "DOQUERY">
<@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>
<@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>
<@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>
<#break>
<#case "DOSQL">
<@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>
<@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>
<@check_etl_properties step_uniquename=step.uniquename propname="limitDebugStmts" defaultvalue="-1"/>
<#break>
<#default>
</#switch>
</#foreach>
<#assign etl_step_properties = new_etl_step_properties/>
--Bezeichnungen aktualisieren
<#foreach etl_job in etl_jobs>
update etl_job set caption='${etl_job.name}' where uniquename='${etl_job.uniquename}';
</#foreach>
<#foreach etl_step in etl_steps>
update etl_step set caption='${etl_step.name}' where uniquename='${etl_step.uniquename}' and tid in
(select step_id from etl_step_relation where job_id in (select tid from etl_job where uniquename='${etl_step.etl_job}'));
</#foreach>
create temp table tmp_etl_step(
tid INTEGER,
uniquename VARCHAR(255),
caption VARCHAR(255),
systeminfo_id INTEGER not null,
step_type INTEGER,
step_type_uniquename VARCHAR(255),
sortnr SMALLINT not null,
force_continue SMALLINT,
etl_job_id INTEGER,
parent_step_id INTEGER,
parent_step_uniquename varchar(255),
parent_job_uniquename varchar(255),
logfile varchar(255),
custom_step smallint,
already_exists smallint
);
create temp table tmp_etl_job(
tid INTEGER,
uniquename VARCHAR(255),
caption VARCHAR(255),
systeminfo_id INTEGER not null,
logfile varchar(255),
already_exists smallint,
custom_job smallint
);
create temp table tmp_etl_job_param(
tid SERIAL not null,
etl_job_id INTEGER,
uniquename VARCHAR(255) not null,
name VARCHAR(255),
param_default VARCHAR(255)
);
create temp table tmp_etl_step_property(
tid SERIAL not null,
etl_step_id INTEGER not null,
prop_name VARCHAR(255),
prop_value text
);
create temp table tmp_etl_step_relation(
tid SERIAL not null,
step_id INTEGER not null,
parent_step_id INTEGER,
job_id INTEGER not null,
force_continue SMALLINT default 1,
step_active SMALLINT default 1,
sortnr SMALLINT default 1,
custom_step SMALLINT default 1
);
update etl_job set custom_job=2
where custom_job=0
and uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload');
update etl_step set custom_step=2
where custom_step=0
and tid in (select R.step_id from etl_step_relation R, etl_job J
where R.job_id=J.tid
and J.uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload'));
update etl_step_relation set custom_step=2
where custom_step=0
and job_id in (select J.tid
from etl_job J
where J.uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload'));
<#foreach etl_job in etl_jobs>
truncate table tmp_etl_job;
truncate table tmp_etl_step;
truncate table tmp_etl_job_param;
truncate table tmp_etl_step_property;
truncate table tmp_etl_step_relation;
insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists,custom_job)
values ('${etl_job.uniquename}',
'${etl_job.name}',
${etl_job.systeminfo_id},
'${etl_job.logfile}',0,2);
update tmp_etl_job set tid=J.tid,
already_exists=1,
caption=J.caption,
logfile=J.logfile,
custom_job=J.custom_job
from etl_job J
where J.uniquename=tmp_etl_job.uniquename
and J.systeminfo_id=tmp_etl_job.systeminfo_id;
insert into etl_job(uniquename,caption,systeminfo_id,custom_job)
select uniquename,caption,systeminfo_id,custom_job
from tmp_etl_job
where already_exists=0;
update tmp_etl_job set tid=(select J.tid
from etl_job J
where J.uniquename=tmp_etl_job.uniquename
and J.systeminfo_id=tmp_etl_job.systeminfo_id)
where already_exists=0;
<#foreach etl_job_param in etl_job_params>
<#if etl_job_param.etl_job == etl_job.uniquename>
insert into tmp_etl_job_param(
etl_job_id,
uniquename,
name,
param_default)
select J.tid,
'${etl_job_param.param_name}',
'${etl_job_param.name}',
'${etl_job_param.param_default}'
from tmp_etl_job J;
</#if>
</#foreach>
<#assign sortnr=0/>
<#foreach etl_step in etl_steps>
<#if etl_step.etl_job == etl_job.uniquename>
<#assign sortnr=sortnr+1/>
insert into tmp_etl_step(
uniquename,
caption,
systeminfo_id,
step_type_uniquename,
sortnr,
force_continue,
etl_job_id,
parent_step_uniquename,
parent_job_uniquename,
logfile,
custom_step,
already_exists
)
select '${etl_step.uniquename}',
'${etl_step.name}',
${etl_job.systeminfo_id},
'${etl_step.type}' as step_type_uniquename,
${sortnr}*10 as sortnr,
0 as force_continue,
J.tid as etl_job_id,
<#if etl_step.parent?exists && etl_step.parent != "">
'${etl_step.parent}',
<#else>
'' as parent_step_uniquename,
</#if>
J.uniquename,
'${etl_job.logfile}' as logfile,
2,
0
from etl_job J
where J.uniquename='${etl_job.uniquename}'
and J.systeminfo_id=${etl_job.systeminfo_id};
</#if>
</#foreach>
delete from etl_job_param
where etl_job_id in (
select distinct etl_job_id
from tmp_etl_job_param
);
insert into etl_job_param
(
etl_job_id,
uniquename,
name,
param_default
)
select
etl_job_id,
uniquename,
name,
param_default
from tmp_etl_job_param;
update tmp_etl_step set step_type=(select T.tid from etl_step_type T
where T.uniquename=tmp_etl_step.step_type_uniquename);
update tmp_etl_step set tid=S.tid,
already_exists=1,
caption=S.caption,
logfile=S.logfile,
custom_step=S.custom_step
from etl_step S
where S.uniquename=tmp_etl_step.uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id;
insert into etl_step(
uniquename,
caption,
systeminfo_id,
step_type_id,
logfile,
custom_step)
select
uniquename,
caption,
systeminfo_id,
step_type,
logfile,
custom_step
from tmp_etl_step
where already_exists=0;
update tmp_etl_step set tid=(select S.tid
from etl_step S
where S.uniquename=tmp_etl_step.uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id)
where already_exists=0;
update tmp_etl_step set parent_step_id=(select S.tid
from etl_step S
where S.uniquename=tmp_etl_step.parent_step_uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id);
delete from etl_step_property
where etl_step_id in (select T.tid from tmp_etl_step T);
delete from etl_step_relation
where job_id in (select J.tid from tmp_etl_job J)
and custom_step=2;
<#foreach etl_step_property in etl_step_properties>
insert into tmp_etl_step_property(
etl_step_id,
prop_name)
select
T.tid as etl_step_id,
'${etl_step_property.prop_name}'
from tmp_etl_step T
where uniquename='${etl_step_property.etl_step}';
</#foreach>
insert into etl_step_property(
etl_step_id,
prop_name)
select
T.etl_step_id,
T.prop_name
from tmp_etl_step_property T;
insert into tmp_etl_step_relation(
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
)
select
tid,
parent_step_id,
etl_job_id,
0 as force_continue,
1 as step_active,
sortnr,
2
from tmp_etl_step;
insert into etl_step_relation(
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
)
select
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
from tmp_etl_step_relation;
</#foreach>
drop table tmp_etl_step;
drop table tmp_etl_job;
drop table tmp_etl_job_param;
drop table tmp_etl_step_property;
drop table tmp_etl_step_relation;
<#foreach testfall in testfaelle>
select 'testfall ${testfall.testcase}: ERFOLG'
from xdummy
where ${testfall.assertion}=(${testfall.sql});
select 'testfall ${testfall.testcase}: FEHLER bei ${testfall.testcase}'
from xdummy
where ${testfall.assertion}!=(${testfall.sql});
</#foreach>
<#foreach etl_step_property in etl_step_properties>
update etl_step_property
set prop_value='${etl_step_property.prop_value}'
where prop_name='${etl_step_property.prop_name}'
and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step}');
</#foreach>
</#macro>
<@ETL_STEPS_FUELLEN/>

60
webapps_migration/migrate_superx_webapp.sh
Unescape View File

@ -1,6 +1,6 @@ @@ -1,6 +1,6 @@
#!/usr/bin/env bash
# migrate_superx_webapp.sh
# Version 1.5
# Version 1.7
#
# Migriert eine bestehende SuperX-Installation in eine neue Webapp-Struktur.
# Standardziel: /home/superx/webapps/superx
@ -9,7 +9,7 @@ @@ -9,7 +9,7 @@
set -euo pipefail
SCRIPT_NAME="$(basename "$0")"
SCRIPT_VERSION="1.6"
SCRIPT_VERSION="1.7"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_FILE="$SCRIPT_DIR/migrate_superx.conf"
@ -27,6 +27,7 @@ group_exists() { getent group "$1" >/dev/null 2>&1; } @@ -27,6 +27,7 @@ group_exists() { getent group "$1" >/dev/null 2>&1; }
user_in_group() { id -nG "$1" 2>/dev/null | tr ' ' '\n' | grep -qx "$2"; }
current_user_matches_target() { [ "$(current_user)" = "$WEBAPP_USER" ] && user_in_group "$(current_user)" "$WEBAPP_GROUP"; }
canon_path() { if command -v realpath >/dev/null 2>&1; then realpath -m "$1"; else printf '%s\n' "$1"; fi; }
path_exists() { [ -e "$1" ] || [ -L "$1" ]; }
run() {
if is_true "${DRY_RUN:-false}"; then
@ -52,29 +53,30 @@ USAGE @@ -52,29 +53,30 @@ USAGE
load_config() {
[ -f "$CONFIG_FILE" ] || { usage; fail "Config-Datei nicht gefunden: $CONFIG_FILE"; }
# shellcheck disable=SC1090
if [ "$SQL_ENV" == "" ]
then
SQL_ENV=`which SQL_ENV`
fi
if [ "$TARGET_WEBAPP" == "" ]
then
TARGET_WEBAPP=$WEBAPP
fi
echo "Lade $CONFIG_FILE"
# shellcheck disable=SC1090
. "$CONFIG_FILE"
#echo $SQL_ENV
: "${SQL_ENV:?SQL_ENV ist nicht gesetzt}"
MIGRATION_MODE="${MIGRATION_MODE:-db_only}"
DB_TRANSFER_MODE="${DB_TRANSFER_MODE:-move}"
if [ -z "${SQL_ENV:-}" ]; then
SQL_ENV="$(command -v SQL_ENV 2>/dev/null || true)"
fi
if [ -z "${TARGET_WEBAPP:-}" ]; then
if [ "$MIGRATION_MODE" = "db_only" ]; then
TARGET_WEBAPP="auto"
else
TARGET_WEBAPP="/home/superx/webapps/superx"
fi
fi
: "${SQL_ENV:?SQL_ENV ist nicht gesetzt und konnte nicht im PATH gefunden werden}"
: "${TARGET_WEBAPP:?TARGET_WEBAPP ist nicht gesetzt}"
: "${WEBAPP_USER:?WEBAPP_USER ist nicht gesetzt}"
: "${WEBAPP_GROUP:?WEBAPP_GROUP ist nicht gesetzt}"
MIGRATION_MODE="${MIGRATION_MODE:-full}"
DB_TRANSFER_MODE="${DB_TRANSFER_MODE:-copy}"
STOP_TOMCAT="${STOP_TOMCAT:-false}"
START_TOMCAT="${START_TOMCAT:-false}"
TOMCAT_SERVICE="${TOMCAT_SERVICE:-tomcat10}"
@ -320,8 +322,18 @@ validate_target() { @@ -320,8 +322,18 @@ validate_target() {
[ "$SOURCE_DB_REAL" = "$TARGET_DB_REAL" ] && fail "SOURCE_DB und TARGET_DB sind identisch."
case "$TARGET_DB_REAL" in "$SOURCE_DB_REAL"/*) fail "TARGET_DB liegt innerhalb von SOURCE_DB.";; esac
case "$SOURCE_DB_REAL" in "$TARGET_DB_REAL"/*) fail "SOURCE_DB liegt innerhalb von TARGET_DB.";; esac
[ "$DB_TRANSFER_MODE" = "move" ] && [ -e "$TARGET_DB" ] && fail "DB_TRANSFER_MODE=move, aber TARGET_DB existiert bereits: $TARGET_DB"
[ "$DB_TRANSFER_MODE" = "copy" ] && [ -e "$TARGET_DB" ] && ! is_true "$ALLOW_EXISTING_TARGET_DB" && fail "TARGET_DB existiert bereits: $TARGET_DB"
if path_exists "$TARGET_DB"; then
if [ "$DB_TRANSFER_MODE" = "move" ]; then
! is_true "$ALLOW_EXISTING_TARGET_DB" && fail "TARGET_DB existiert bereits und ALLOW_EXISTING_TARGET_DB=false: $TARGET_DB"
[ -d "$TARGET_DB" ] || [ -L "$TARGET_DB" ] || fail "TARGET_DB existiert, ist aber weder Verzeichnis noch Symlink: $TARGET_DB"
if command -v mountpoint >/dev/null 2>&1 && mountpoint -q "$TARGET_DB"; then
fail "TARGET_DB ist ein Mountpoint und wird nicht automatisch ersetzt: $TARGET_DB"
fi
warn "TARGET_DB existiert bereits und wird bei DB_TRANSFER_MODE=move vollstaendig ersetzt: $TARGET_DB"
elif ! is_true "$ALLOW_EXISTING_TARGET_DB"; then
fail "TARGET_DB existiert bereits: $TARGET_DB"
fi
fi
if is_true "$CREATE_TOMCAT_SYMLINK" && { [ -e "$TOMCAT_SYMLINK_PATH" ] || [ -L "$TOMCAT_SYMLINK_PATH" ]; }; then
if [ -L "$TOMCAT_SYMLINK_PATH" ] && is_true "$REPLACE_EXISTING_SYMLINK"; then :; else
fail "Tomcat-Symlink/Ziel existiert bereits: $TOMCAT_SYMLINK_PATH"
@ -363,7 +375,11 @@ transfer_db() { @@ -363,7 +375,11 @@ transfer_db() {
else
log "Verschiebe DB-Verzeichnis nach WEB-INF/conf/edustore/db..."
run "mkdir -p '$(dirname "$TARGET_DB")'"
run "mv '$SOURCE_DB' '$TARGET_DB'"
if path_exists "$TARGET_DB"; then
log "Entferne bestehendes TARGET_DB vor dem Verschieben..."
run "rm -rf -- '$TARGET_DB'"
fi
run "mv -- '$SOURCE_DB' '$TARGET_DB'"
fi
}

Write Preview
Loading…
Cancel
Save