SuperX
/
etl
6
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity
Browse Source

Entladen und Hochladen von anderen Tabellengruppen #5§ 

§
master
Daniel Quathamer 2 weeks ago
parent
0fc8423fd2
commit
b32beb082d
9 changed files with 1437 additions and 1 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 25
      build.xml
  2. 47
      src-modules/module/etl/bin/etl_execute_unload_job.x
  3. 47
      src-modules/module/etl/bin/etl_execute_upload_job.x
  4. 177
      src-modules/module/etl/conf/etl.xml
  5. 62
      src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql
  6. 52
      src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql
  7. 1021
      src-modules/module/etl/schluesseltabellen/etl_transfer.xsl
  8. 7
      src/de/superx/elt/EtlActionJob.java
  9. BIN
      superx/WEB-INF/lib/superx-etl.jar

25
build.xml
Unescape View File

@ -76,6 +76,8 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<antcall target="module_copy_source" /> <antcall target="module_copy_source" />
<antcall target="call_module_scripts_create_ant" /> <antcall target="call_module_scripts_create_ant" />
<antcall target="module_copy_webapp_files" /> <antcall target="module_copy_webapp_files" />
<antcall target="generate_etl_jobs" />
</target> </target>
<target name="module_copy_source" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum"> <target name="module_copy_source" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum">
<echo message="Start module_copy_sources für Modul ${MODULE} in Pfad ${MODULE_PATH}" /> <echo message="Start module_copy_sources für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
@ -99,6 +101,29 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<chmod dir="${MODULE_PATH}/rohdaten" perm="ugo+x" includes="**/*.x" /> <chmod dir="${MODULE_PATH}/rohdaten" perm="ugo+x" includes="**/*.x" />
</target> </target>
<target name="generate_etl_jobs" description="generiert die ETL Scripte für Sonderladeroutinen" >
<echo message="Start etl-job users_and_groups für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_users_and_groups.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="users_and_groups" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_users_and_groups.sql" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_sichten.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="sichten" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_sichten.sql" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_sx_captions.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="sx_captions" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_sx_captions.sql" />
</target>
<target name="module_copy_webapp_files" description="Kopiere webapp-Dateien in den WEBAPP-Verzeichnisbaum"> <target name="module_copy_webapp_files" description="Kopiere webapp-Dateien in den WEBAPP-Verzeichnisbaum">
<echo message="Start module_copy_webapp_files für Modul ${MODULE} in Pfad ${MODULE_PATH}" /> <echo message="Start module_copy_webapp_files für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
<copy todir="${WEBAPP}" overwrite="true"> <copy todir="${WEBAPP}" overwrite="true">

47
src-modules/module/etl/bin/etl_execute_unload_job.x
Unescape View File

@ -0,0 +1,47 @@
#!/bin/bash
#Änderungen
#---------------------------------------------------------------------
# Shell-Kommando "etl_execute_unload_job.x"
#
#---------------------------------------------------------------------
if [ "$1" = "" ]
then echo "Aufruf: etl_execute_unload_job.x job_uniquename FILTER <Ausgabedatei (optional)>"
echo " "
echo "Aktion: etl_execute_unload_job.x entlaedt Tabellen."
echo " Mit Hilfe von etl_execute_upload_job.x koennen diese Daten importiert werden."
echo " "
exit 0
fi
#-------------------------------------------------------------------
#-- Metadaten entladen
#-------------------------------------------------------------------
JOB_UNIQUENAME=$1
FILTER=$2
if [ "$3" == "" ]
then outfile=$JOB_UNIQUENAME.xml
else
outfile=$3
fi
SX_CLIENT=jdbc
export SX_CLIENT
CP=".:$JDBC_CLASSPATH"
CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -outfile:$outfile -params:$FILTER"
echo $CMD
$CMD

47
src-modules/module/etl/bin/etl_execute_upload_job.x
Unescape View File

@ -0,0 +1,47 @@
#!/bin/bash
#Änderungen
#---------------------------------------------------------------------
# Shell-Kommando "etl_execute_unload_job.x"
#
#---------------------------------------------------------------------
if [ "$1" = "" ]
then echo "Aufruf: etl_execute_upload_job.x job_uniquename FILTER <Eingabedatei (optional)>"
echo " "
echo "Aktion: etl_execute_upload_job.x laedt Tabellen."
echo " "
echo " "
exit 0
fi
#-------------------------------------------------------------------
#-- Metadaten entladen
#-------------------------------------------------------------------
JOB_UNIQUENAME=$1
FILTER=$2
if [ "$3" == "" ]
then infile=$JOB_UNIQUENAME.xml
else
infile=$3
fi
SX_CLIENT=jdbc
export SX_CLIENT
CP=".:$JDBC_CLASSPATH"
CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -params:PATH_TO_INPUTFILE=$infile"
echo $CMD
$CMD

177
src-modules/module/etl/conf/etl.xml
Unescape View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--<!DOCTYPE module SYSTEM "../../../conf/superx-module.dtd"> --> <!--<!DOCTYPE module SYSTEM "../../../conf/superx-module.dtd"> -->
<module name="etl" version="0.4" sachgebiet_id="270" <module name="etl" version="0.5b" sachgebiet_id="270"
sachgebiet="Laderoutinen" systeminfo_id="270" sachgebiet="Laderoutinen" systeminfo_id="270"
system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen"> system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen">
<database name="superx" system="superx"> <database name="superx" system="superx">
@ -161,6 +161,174 @@ parent="Laderoutinen">Abfragen zur Administration</thema>
<nativeaction sql="update systeminfo set datum=today() where tid in (270)" scriptfile="" database=""/> <nativeaction sql="update systeminfo set datum=today() where tid in (270)" scriptfile="" database=""/>
</action> </action>
</etl-step> </etl-step>
<etl-step id="users_and_groups" name="Konfiguration User/Gruppen" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die User- und Gruppen-Rechtetabellen zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Userinfo-ID. Wenn 0, dann werden alle User geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="user_stammdaten">User-Daten</etl_transfer_group>
<etl_transfer_group uniquename="group_stammdaten">Gruppen-Daten</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="userinfo" transfer_group="user_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>benutzer</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_institution" transfer_group="user_stammdaten">
<whereclause> userid=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_masken_bez" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sachgeb_bez" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_pw" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sichtarten" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sichten" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<!--gruppen:-->
<etl_transfer_table name="groupinfo" transfer_group="group_stammdaten">
<whereclause> tid in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>name</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_field_pref" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_masken_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sachgeb_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sichtarten" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sichten" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_group_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_users_groups" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
<etl-step id="sichten" name="Sichten" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die Sichten zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Sicht-TID. Wenn 0, dann werden alle Sichten geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="sichten_stammdaten">Sichten</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="sichten" transfer_group="sichten_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>name_intern</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_sichten" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_sichten_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
<etl-step id="sx_captions" name="Beschriftungen" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die Beschriftungen zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Caption-TID. Wenn 0, dann werden alle Beschriftungen geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="sx_captions_stammdaten">Beschriftungen</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="sx_captions" transfer_group="sx_captions_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>id,table_name,field_name,record_no,locale</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_sx_captions" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_sx_captions_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
</etl> </etl>
<install> <install>
@ -169,6 +337,9 @@ parent="Laderoutinen">Abfragen zur Administration</thema>
<loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable> <loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_users_groups.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sichten.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sx_captions.sql" database=""/>
<!--<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sx_jobs_fuellen.sql" database=""/>--> <!--<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sx_jobs_fuellen.sql" database=""/>-->
@ -199,6 +370,10 @@ version integer
<action error="stop"> <action error="stop">
<loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable> <loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_users_groups.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sichten.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sx_captions.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/>
</action> </action>
</upgrade-step> </upgrade-step>

62
src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql
Unescape View File

@ -0,0 +1,62 @@
begin work;
delete from sichten where name_intern in (select name_intern from tmp_sichten);
insert into sichten(parent,
systeminfoid,
art,
type,
name_intern,
name,
beschreibung,
sortnr,
quelle,
alt_hier_id,
treecfgtable,
treecfgid,
label,
user_rechte,
rechtequelle,
sesamkey,
standbutton,
attribut1,
attribut2,
attribut3,
attribut4,
xmlmaxentries,
gueltig_seit,
gueltig_bis,
cacheapplet,
cachexml,
aktiv) select parent,
systeminfoid,
art,
type,
name_intern,
name,
beschreibung,
sortnr,
quelle,
alt_hier_id,
treecfgtable,
treecfgid,
label,
user_rechte,
rechtequelle,
sesamkey,
standbutton,
attribut1,
attribut2,
attribut3,
attribut4,
xmlmaxentries,
gueltig_seit,
gueltig_bis,
cacheapplet,
cachexml,
aktiv from tmp_sichten;
drop table tmp_sichten;
commit;

52
src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql
Unescape View File

@ -0,0 +1,52 @@
begin work;
delete from userinfo where benutzer in (select benutzer from tmp_userinfo);
delete from user_institution where userid in (select userid from tmp_user_institution);
delete from user_masken_bez where userinfo_id in (select userinfo_id from tmp_user_masken_bez);
delete from user_sachgeb_bez where userinfo_id in (select userinfo_id from tmp_user_sachgeb_bez);
delete from user_pw where userinfo_id in (select userinfo_id from tmp_user_pw);
delete from user_sichtarten where userinfo_id in (select userinfo_id from tmp_user_sichtarten);
delete from user_sichten where userinfo_id in (select userinfo_id from tmp_user_sichten);
delete from groupinfo where tid in (select tid from tmp_groupinfo);
delete from group_field_pref where groupinfo_id in (select groupinfo_id from tmp_group_field_pref);
delete from group_masken_bez where groupinfo_id in (select groupinfo_id from tmp_group_masken_bez);
delete from group_sachgeb_bez where groupinfo_id in (select groupinfo_id from tmp_group_sachgeb_bez);
delete from group_sichtarten where groupinfo_id in (select groupinfo_id from tmp_group_sichtarten);
delete from group_sichten where groupinfo_id in (select groupinfo_id from tmp_group_sichten);
delete from user_group_bez where groupinfo_id in (select groupinfo_id from tmp_user_group_bez);
insert into userinfo select * from tmp_userinfo ;
insert into user_institution select * from tmp_user_institution ;
insert into user_masken_bez select * from tmp_user_masken_bez ;
insert into user_sachgeb_bez select * from tmp_user_sachgeb_bez ;
insert into user_pw select * from tmp_user_pw ;
insert into user_sichtarten select * from tmp_user_sichtarten ;
insert into user_sichten select * from tmp_user_sichten ;
insert into groupinfo select * from tmp_groupinfo ;
insert into group_field_pref select * from tmp_group_field_pref ;
insert into group_masken_bez select * from tmp_group_masken_bez ;
insert into group_sachgeb_bez select * from tmp_group_sachgeb_bez ;
insert into group_sichtarten select * from tmp_group_sichtarten;
insert into group_sichten select * from tmp_group_sichten ;
insert into user_group_bez select * from tmp_user_group_bez ;
drop table tmp_userinfo;
drop table tmp_user_institution;
drop table tmp_user_masken_bez;
drop table tmp_user_sachgeb_bez;
drop table tmp_user_pw;
drop table tmp_user_sichtarten;
drop table tmp_user_sichten;
drop table tmp_groupinfo;
drop table tmp_group_field_pref;
drop table tmp_group_masken_bez;
drop table tmp_group_sachgeb_bez;
drop table tmp_group_sichtarten;
drop table tmp_group_sichten;
drop table tmp_user_group_bez;
commit;

1021
src-modules/module/etl/schluesseltabellen/etl_transfer.xsl
View File

File diff suppressed because it is too large Load Diff

7
src/de/superx/elt/EtlActionJob.java
Unescape View File

@ -219,6 +219,13 @@ public class EtlActionJob extends EtlAction {
} }
else else
{ {
if(jobReturnCode==1)
{
System.err.println(super.getActionLog());
throw new Exception("Fehler bei Jobausführung");
}
else
System.out.println("Jobausführung Erfolg");
try { try {
de.superx.elt.EtlUtils.saveFileContentsWithEncoding(outfile, sw.toString(), null); de.superx.elt.EtlUtils.saveFileContentsWithEncoding(outfile, sw.toString(), null);
} catch (Exception e) { } catch (Exception e) {

BIN
superx/WEB-INF/lib/superx-etl.jar
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save