diff --git a/build.xml b/build.xml index 55b8482..39d50af 100644 --- a/build.xml +++ b/build.xml @@ -76,6 +76,8 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath + + @@ -99,6 +101,29 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath + + + + + + + + + + + + + + + + + + + + diff --git a/src-modules/module/etl/bin/etl_execute_unload_job.x b/src-modules/module/etl/bin/etl_execute_unload_job.x new file mode 100755 index 0000000..c979c89 --- /dev/null +++ b/src-modules/module/etl/bin/etl_execute_unload_job.x @@ -0,0 +1,47 @@ +#!/bin/bash +#Änderungen +#--------------------------------------------------------------------- +# Shell-Kommando "etl_execute_unload_job.x" +# +#--------------------------------------------------------------------- + +if [ "$1" = "" ] + then echo "Aufruf: etl_execute_unload_job.x job_uniquename FILTER " + echo " " + echo "Aktion: etl_execute_unload_job.x entlaedt Tabellen." + echo " Mit Hilfe von etl_execute_upload_job.x koennen diese Daten importiert werden." + echo " " + exit 0 +fi + +#------------------------------------------------------------------- +#-- Metadaten entladen +#------------------------------------------------------------------- + +JOB_UNIQUENAME=$1 + +FILTER=$2 + +if [ "$3" == "" ] + then outfile=$JOB_UNIQUENAME.xml +else + +outfile=$3 + +fi + + + +SX_CLIENT=jdbc +export SX_CLIENT +CP=".:$JDBC_CLASSPATH" + +CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -outfile:$outfile -params:$FILTER" + +echo $CMD + +$CMD + + + + diff --git a/src-modules/module/etl/bin/etl_execute_upload_job.x b/src-modules/module/etl/bin/etl_execute_upload_job.x new file mode 100755 index 0000000..c012db8 --- /dev/null +++ b/src-modules/module/etl/bin/etl_execute_upload_job.x @@ -0,0 +1,47 @@ +#!/bin/bash +#Änderungen +#--------------------------------------------------------------------- +# Shell-Kommando "etl_execute_unload_job.x" +# +#--------------------------------------------------------------------- + +if [ "$1" = "" ] + then echo "Aufruf: etl_execute_upload_job.x job_uniquename FILTER " + echo " " + echo "Aktion: etl_execute_upload_job.x laedt Tabellen." + echo " " + echo " " + exit 0 +fi + +#------------------------------------------------------------------- +#-- Metadaten entladen +#------------------------------------------------------------------- + +JOB_UNIQUENAME=$1 + +FILTER=$2 + +if [ "$3" == "" ] + then infile=$JOB_UNIQUENAME.xml +else + +infile=$3 + +fi + + + +SX_CLIENT=jdbc +export SX_CLIENT +CP=".:$JDBC_CLASSPATH" + +CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -params:PATH_TO_INPUTFILE=$infile" + +echo $CMD + +$CMD + + + + diff --git a/src-modules/module/etl/conf/etl.xml b/src-modules/module/etl/conf/etl.xml index cc28396..e753a1d 100644 --- a/src-modules/module/etl/conf/etl.xml +++ b/src-modules/module/etl/conf/etl.xml @@ -1,6 +1,6 @@ - @@ -161,6 +161,174 @@ parent="Laderoutinen">Abfragen zur Administration + + +Entlädt und lädt die User- und Gruppen-Rechtetabellen zur Übertragung in eine andere Säule + + + +Userinfo-ID. Wenn 0, dann werden alle User geladen +0 + + + +User-Daten +Gruppen-Daten + + + + +tid=$TID or $TID=0 +benutzer + + + + userid=$TID or $TID=0 +1,2 + + + + userinfo_id=$TID or $TID=0 +1,2 + + + + userinfo_id=$TID or $TID=0 +1,2 + + + + userinfo_id=$TID or $TID=0 +1 + + + + userinfo_id=$TID or $TID=0 +1,2 + + + + userinfo_id=$TID or $TID=0 +1,2 + + + + + + tid in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +name + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + groupinfo_id in (select groupinfo_id from user_group_bez B + where B.userinfo_id=$TID) + or $TID=0 +1,2 + + + + + + + + + + + + + + +Entlädt und lädt die Sichten zur Übertragung in eine andere Säule + + + +Sicht-TID. Wenn 0, dann werden alle Sichten geladen +0 + + + +Sichten + + + + +tid=$TID or $TID=0 +name_intern + + + + + + + + + + + + +Entlädt und lädt die Beschriftungen zur Übertragung in eine andere Säule + + + +Caption-TID. Wenn 0, dann werden alle Beschriftungen geladen +0 + + + +Beschriftungen + + + + +tid=$TID or $TID=0 +id,table_name,field_name,record_no,locale + + + + + + + + + + @@ -169,6 +337,9 @@ parent="Laderoutinen">Abfragen zur Administration + + + @@ -199,6 +370,10 @@ version integer + + + + diff --git a/src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql b/src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql new file mode 100644 index 0000000..52cc7a2 --- /dev/null +++ b/src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql @@ -0,0 +1,62 @@ +begin work; + +delete from sichten where name_intern in (select name_intern from tmp_sichten); + +insert into sichten(parent, + systeminfoid, + art, + type, + name_intern, + name, + beschreibung, + sortnr, + quelle, + alt_hier_id, + treecfgtable, + treecfgid, + label, + user_rechte, + rechtequelle, + sesamkey, + standbutton, + attribut1, + attribut2, + attribut3, + attribut4, + xmlmaxentries, + gueltig_seit, + gueltig_bis, + cacheapplet, + cachexml, + aktiv) select parent, + systeminfoid, + art, + type, + name_intern, + name, + beschreibung, + sortnr, + quelle, + alt_hier_id, + treecfgtable, + treecfgid, + label, + user_rechte, + rechtequelle, + sesamkey, + standbutton, + attribut1, + attribut2, + attribut3, + attribut4, + xmlmaxentries, + gueltig_seit, + gueltig_bis, + cacheapplet, + cachexml, + aktiv from tmp_sichten; + +drop table tmp_sichten; + +commit; + diff --git a/src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql b/src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql new file mode 100644 index 0000000..53e898c --- /dev/null +++ b/src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql @@ -0,0 +1,52 @@ +begin work; + + +delete from userinfo where benutzer in (select benutzer from tmp_userinfo); + +delete from user_institution where userid in (select userid from tmp_user_institution); +delete from user_masken_bez where userinfo_id in (select userinfo_id from tmp_user_masken_bez); +delete from user_sachgeb_bez where userinfo_id in (select userinfo_id from tmp_user_sachgeb_bez); +delete from user_pw where userinfo_id in (select userinfo_id from tmp_user_pw); +delete from user_sichtarten where userinfo_id in (select userinfo_id from tmp_user_sichtarten); +delete from user_sichten where userinfo_id in (select userinfo_id from tmp_user_sichten); +delete from groupinfo where tid in (select tid from tmp_groupinfo); + +delete from group_field_pref where groupinfo_id in (select groupinfo_id from tmp_group_field_pref); +delete from group_masken_bez where groupinfo_id in (select groupinfo_id from tmp_group_masken_bez); +delete from group_sachgeb_bez where groupinfo_id in (select groupinfo_id from tmp_group_sachgeb_bez); +delete from group_sichtarten where groupinfo_id in (select groupinfo_id from tmp_group_sichtarten); +delete from group_sichten where groupinfo_id in (select groupinfo_id from tmp_group_sichten); +delete from user_group_bez where groupinfo_id in (select groupinfo_id from tmp_user_group_bez); + + +insert into userinfo select * from tmp_userinfo ; +insert into user_institution select * from tmp_user_institution ; +insert into user_masken_bez select * from tmp_user_masken_bez ; +insert into user_sachgeb_bez select * from tmp_user_sachgeb_bez ; +insert into user_pw select * from tmp_user_pw ; +insert into user_sichtarten select * from tmp_user_sichtarten ; +insert into user_sichten select * from tmp_user_sichten ; +insert into groupinfo select * from tmp_groupinfo ; +insert into group_field_pref select * from tmp_group_field_pref ; +insert into group_masken_bez select * from tmp_group_masken_bez ; +insert into group_sachgeb_bez select * from tmp_group_sachgeb_bez ; +insert into group_sichtarten select * from tmp_group_sichtarten; +insert into group_sichten select * from tmp_group_sichten ; +insert into user_group_bez select * from tmp_user_group_bez ; + +drop table tmp_userinfo; +drop table tmp_user_institution; +drop table tmp_user_masken_bez; +drop table tmp_user_sachgeb_bez; +drop table tmp_user_pw; +drop table tmp_user_sichtarten; +drop table tmp_user_sichten; +drop table tmp_groupinfo; +drop table tmp_group_field_pref; +drop table tmp_group_masken_bez; +drop table tmp_group_sachgeb_bez; +drop table tmp_group_sichtarten; +drop table tmp_group_sichten; +drop table tmp_user_group_bez; + +commit; diff --git a/src-modules/module/etl/schluesseltabellen/etl_transfer.xsl b/src-modules/module/etl/schluesseltabellen/etl_transfer.xsl new file mode 100644 index 0000000..96f6619 --- /dev/null +++ b/src-modules/module/etl/schluesseltabellen/etl_transfer.xsl @@ -0,0 +1,1021 @@ + + + + + + + + + + + +-- XML-generiertes SQL-Script fuer psql +--von etl_transfer.xsl + + + + + --ETL-Transfer : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +drop table if exists tmp_; + + + + + +create temp table tmp_etl_step( +tid INTEGER, +uniquename VARCHAR(255) , +caption VARCHAR(255) , +systeminfo_id INTEGER not null, +step_type INTEGER, +step_type_uniquename VARCHAR(255), +sortnr SMALLINT not null, +force_continue SMALLINT, +etl_job_id INTEGER , +parent_step_id INTEGER , +parent_step_uniquename varchar(255), +parent_job_uniquename varchar(255), +logfile varchar(255), +custom_step smallint, +already_exists smallint +) +; + +create temp table tmp_etl_job( + tid INTEGER, +uniquename VARCHAR(255) , +caption VARCHAR(255) , +systeminfo_id INTEGER not null, +logfile varchar(255), +already_exists smallint, +custom_job smallint +) +; + +create temp table tmp_etl_job_param( + tid SERIAL not null, +etl_job_id INTEGER , +uniquename VARCHAR(255) not null, +name VARCHAR(255) , +param_default VARCHAR(255) +) +; + + +create temp table tmp_etl_step_property( + tid SERIAL not null, +etl_step_id INTEGER not null, +prop_name VARCHAR(255) , +prop_value text + +) +; + +create temp table tmp_etl_step_relation( + tid SERIAL not null, +step_id INTEGER not null, +parent_step_id INTEGER , +job_id INTEGER not null, +force_continue SMALLINT default 1 , +step_active SMALLINT default 1, +sortnr SMALLINT default 1, +custom_step SMALLINT default 1 + +) +; + + + + + + + + + + + + +truncate table tmp_etl_job; +truncate table tmp_etl_step; +truncate table tmp_etl_job_param; +truncate table tmp_etl_step_property; +truncate table tmp_etl_step_relation; + +--tids der jobs dürfen sich nicht ändern, daher +-- +-- 1. vorh. Jobs updaten +-- 2. neue Jobs einfügen +-- 3. alte Jobs löschen + +insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists,custom_job) +values ('', +'', +, +'',0,2); + + + +update tmp_etl_job set tid=J.tid, +already_exists=1, +caption=J.caption, +logfile=J.logfile, +custom_job=J.custom_job +from etl_job J where J.uniquename=tmp_etl_job.uniquename +and J.systeminfo_id=tmp_etl_job.systeminfo_id +; + +--neue jobs: +insert into etl_job(uniquename,caption,systeminfo_id,custom_job) +select uniquename,caption,systeminfo_id,custom_job +from tmp_etl_job +where already_exists=0; +--tid von neuen Jobs ermitteln: +update tmp_etl_job set tid=(select J.tid +from etl_job J where J.uniquename=tmp_etl_job.uniquename +and J.systeminfo_id=tmp_etl_job.systeminfo_id) +where already_exists=0 +; + +--TODO +--obsolete Jobs: sollen bei Deinstallation des Moduls entfernt werden + + + + + + + + + + + +insert into tmp_etl_job_param( +etl_job_id , +uniquename, +name , +param_default) +select J.tid, +'', +'', +'' +from tmp_etl_job J +where J.uniquename='' +; + + + + + +insert into tmp_etl_job_param( +etl_job_id , +uniquename, +name , +param_default) +select J.tid, +'FORMAT', +'Format', +'XML' +from tmp_etl_job J +where J.uniquename='' +; + + + + + +insert into tmp_etl_job_param( +etl_job_id , +uniquename, +name , +param_default) +select J.tid, +'PATH_TO_OUTPUTFILE', +'Ausgabedatei', +'$SUPERX_DIR/db/.xml' +from tmp_etl_job J +where J.uniquename='' +; + + + + + + + +insert into tmp_etl_job_param( +etl_job_id , +uniquename, +name , +param_default) +select J.tid, +'PATH_TO_INPUTFILE', +'Eingabedatei', +'$SUPERX_DIR/db/.xml' +from tmp_etl_job J +where J.uniquename='' +; + + + + + + + +--erst job-params einfügen: + +delete from etl_job_param +where etl_job_id in ( + SELECT distinct + etl_job_id +FROM tmp_etl_job_param ) + ; + +insert into etl_job_param +( + etl_job_id, + uniquename, + name, + param_default + ) + SELECT + etl_job_id, + uniquename, + name, + param_default +FROM tmp_etl_job_param + ; + + + + + + + + + + + + + +--Gruppe + + + + + + + + + + + + + + + +--Unload + + + + + + + + + + + + + + + + + + + + +--Upload + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +--nun steps einfügen: + + +update tmp_etl_step set step_type=(select T.tid from etl_step_type T +where T.uniquename=tmp_etl_step.step_type_uniquename); + +--select * from tmp_etl_step +--where step_type is null;--_uniquename from tmp_etl_step; + + +--vorhandene Steps erkennen: + +update tmp_etl_step set tid=S.tid, +already_exists=1, +caption=S.caption, +logfile=S.logfile, +custom_step=S.custom_step +from etl_step S where S.uniquename=tmp_etl_step.uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id +; + +--neue Steps einfügen: +insert into etl_step( + uniquename, + caption, + systeminfo_id, + step_type_id, + logfile, + custom_step) +select + uniquename, + caption, + systeminfo_id, + step_type, + logfile, + custom_step +FROM tmp_etl_step +where already_exists=0 +; + +--tid von neuen steps ermitteln: +update tmp_etl_step set tid=(select S.tid +from etl_step S where S.uniquename=tmp_etl_step.uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id) +where already_exists=0 +; + +--parent ermitteln: +update tmp_etl_step set parent_step_id=(select S.tid +from etl_step S where S.uniquename=tmp_etl_step.parent_step_uniquename +and S.systeminfo_id=tmp_etl_step.systeminfo_id) +; + + + + + + +delete from etl_step_relation +where job_id in (select J.tid +FROM tmp_etl_job J) +and custom_step=2 + ; +--jetzt step-relation: +insert into tmp_etl_step_relation( + step_id, + parent_step_id, + job_id, + force_continue, + step_active, + sortnr, + custom_step +) +select + tid, + parent_step_id, + etl_job_id, + 0 as force_continue, + 1 as step_active, + sortnr, + 2 +FROM tmp_etl_step + ; + +insert into etl_step_relation( + step_id, + parent_step_id, + job_id, + force_continue, + step_active, + sortnr, + custom_step +) +select + step_id, + parent_step_id, + job_id, + force_continue, + step_active, + sortnr, + custom_step +FROM tmp_etl_step_relation + ; + + + + + + +delete from etl_step_property +where etl_step_id in (select T.tid +FROM tmp_etl_step T ) + ; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +--einfügen in echte Tabelle: +insert into etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.etl_step_id, + T.prop_name, + prop_value +FROM tmp_etl_step_property T +; + + + + + + + + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'msg', + ' + +' +as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + + + + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'PATH_TO_INPUTFILE', + ' + +' +as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + + + + + + + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'select_stmt', + 'select * + + from + + where + + + order by + +' +as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + + + + + + + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'select_stmt', + 'drop table if exists tmp_ +' +as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'select_stmt', + 'select * + + into tmp_ + from + where 1=0' +as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'target_table', + '' as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'format', + 'XML' as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'search_path', + '/etlAction/unload [@name="unload_"]/rs/row' as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + +insert into tmp_etl_step_property( + etl_step_id, + prop_name, + prop_value) +select + T.tid as etl_step_id, + 'path_to_inputfile', + '$PATH_TO_INPUTFILE' as prop_value + +FROM tmp_etl_step T +where uniquename ='' +; + + + + + + + + + + + +, + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +insert into tmp_etl_step( +uniquename , +caption , +systeminfo_id , +step_type_uniquename, +sortnr, +force_continue, +etl_job_id , +parent_step_uniquename, +parent_job_uniquename, +logfile, +custom_step, +already_exists +) +select '', +'', +, +'' as step_type_uniquename, + + 10 as sortnr, +0 as force_continue, +J.tid as etl_job_id, +'' as parent_step_uniquename, +J.uniquename, +'' as logfile, +2, +0 +from etl_job J +where J.uniquename='' +and J.systeminfo_id=; + + + + + + + + +drop table tmp_etl_step; +drop table tmp_etl_job; +drop table tmp_etl_job_param; +drop table tmp_etl_step_property; +drop table tmp_etl_step_relation; + + + + diff --git a/src/de/superx/elt/EtlActionJob.java b/src/de/superx/elt/EtlActionJob.java index d80b091..f19ecda 100644 --- a/src/de/superx/elt/EtlActionJob.java +++ b/src/de/superx/elt/EtlActionJob.java @@ -219,6 +219,13 @@ public class EtlActionJob extends EtlAction { } else { + if(jobReturnCode==1) + { + System.err.println(super.getActionLog()); + throw new Exception("Fehler bei Jobausführung"); + } + else + System.out.println("Jobausführung Erfolg"); try { de.superx.elt.EtlUtils.saveFileContentsWithEncoding(outfile, sw.toString(), null); } catch (Exception e) { diff --git a/superx/WEB-INF/lib/superx-etl.jar b/superx/WEB-INF/lib/superx-etl.jar index c9d26a9..5868d66 100644 Binary files a/superx/WEB-INF/lib/superx-etl.jar and b/superx/WEB-INF/lib/superx-etl.jar differ