Compare commits

..

No commits in common. 'master' and 'etl_0.4_Release' have entirely different histories.

  1. 25
      build.xml
  2. 3
      src-modules/module/etl/bin/SQL_ENV_etl.sam
  3. 47
      src-modules/module/etl/bin/etl_execute_unload_job.x
  4. 47
      src-modules/module/etl/bin/etl_execute_upload_job.x
  5. 179
      src-modules/module/etl/conf/etl.xml
  6. 62
      src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql
  7. 52
      src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql
  8. 1021
      src-modules/module/etl/schluesseltabellen/etl_transfer.xsl
  9. 42
      src-modules/module/etl/upgrade/etl_upgrade_man.sql
  10. 7
      src/de/superx/elt/EtlActionJob.java
  11. 2
      src/de/superx/elt/TableUploader.java
  12. BIN
      superx/WEB-INF/lib/superx-etl.jar

25
build.xml

@ -76,8 +76,6 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath @@ -76,8 +76,6 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<antcall target="module_copy_source" />
<antcall target="call_module_scripts_create_ant" />
<antcall target="module_copy_webapp_files" />
<antcall target="generate_etl_jobs" />
</target>
<target name="module_copy_source" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum">
<echo message="Start module_copy_sources für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
@ -101,29 +99,6 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath @@ -101,29 +99,6 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<chmod dir="${MODULE_PATH}/rohdaten" perm="ugo+x" includes="**/*.x" />
</target>
<target name="generate_etl_jobs" description="generiert die ETL Scripte für Sonderladeroutinen" >
<echo message="Start etl-job users_and_groups für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_users_and_groups.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="users_and_groups" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_users_and_groups.sql" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_sichten.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="sichten" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_sichten.sql" />
<xslt in="${MODULE_PATH}/conf/${MODULE}.xml" style="${MODULE_PATH}/schluesseltabellen/etl_transfer.xsl"
out="${MODULE_PATH}/schluesseltabellen/etl_step_sx_captions.sql" force="true">
<outputproperty name="method" value="text" />
<param name="job" expression="sx_captions" />
</xslt>
<echo message="Datei erzeugt: ${MODULE_PATH}/schluesseltabellen/etl_step_sx_captions.sql" />
</target>
<target name="module_copy_webapp_files" description="Kopiere webapp-Dateien in den WEBAPP-Verzeichnisbaum">
<echo message="Start module_copy_webapp_files für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
<copy todir="${WEBAPP}" overwrite="true">

3
src-modules/module/etl/bin/SQL_ENV_etl.sam

@ -13,7 +13,4 @@ ETL_LOGMAIL=$LOGMAIL; export ETL_LOGMAIL @@ -13,7 +13,4 @@ ETL_LOGMAIL=$LOGMAIL; export ETL_LOGMAIL
PATH=$ETL_PFAD/bin:$PATH
export PATH
JAVA_OPTS="$JAVA_OPTS -DSUPERX_DIR=$SUPERX_DIR"
export JAVA_OPTS

47
src-modules/module/etl/bin/etl_execute_unload_job.x

@ -1,47 +0,0 @@ @@ -1,47 +0,0 @@
#!/bin/bash
#Änderungen
#---------------------------------------------------------------------
# Shell-Kommando "etl_execute_unload_job.x"
#
#---------------------------------------------------------------------
if [ "$1" = "" ]
then echo "Aufruf: etl_execute_unload_job.x job_uniquename FILTER <Ausgabedatei (optional)>"
echo " "
echo "Aktion: etl_execute_unload_job.x entlaedt Tabellen."
echo " Mit Hilfe von etl_execute_upload_job.x koennen diese Daten importiert werden."
echo " "
exit 0
fi
#-------------------------------------------------------------------
#-- Metadaten entladen
#-------------------------------------------------------------------
JOB_UNIQUENAME=$1
FILTER=$2
if [ "$3" == "" ]
then outfile=$JOB_UNIQUENAME.xml
else
outfile=$3
fi
SX_CLIENT=jdbc
export SX_CLIENT
CP=".:$JDBC_CLASSPATH"
CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -outfile:$outfile -params:$FILTER"
echo $CMD
$CMD

47
src-modules/module/etl/bin/etl_execute_upload_job.x

@ -1,47 +0,0 @@ @@ -1,47 +0,0 @@
#!/bin/bash
#Änderungen
#---------------------------------------------------------------------
# Shell-Kommando "etl_execute_unload_job.x"
#
#---------------------------------------------------------------------
if [ "$1" = "" ]
then echo "Aufruf: etl_execute_upload_job.x job_uniquename FILTER <Eingabedatei (optional)>"
echo " "
echo "Aktion: etl_execute_upload_job.x laedt Tabellen."
echo " "
echo " "
exit 0
fi
#-------------------------------------------------------------------
#-- Metadaten entladen
#-------------------------------------------------------------------
JOB_UNIQUENAME=$1
FILTER=$2
if [ "$3" == "" ]
then infile=$JOB_UNIQUENAME.xml
else
infile=$3
fi
SX_CLIENT=jdbc
export SX_CLIENT
CP=".:$JDBC_CLASSPATH"
CMD="java $JAVA_OPTS -cp $CP de.superx.elt.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:$JOB_UNIQUENAME -params:PATH_TO_INPUTFILE=$infile"
echo $CMD
$CMD

179
src-modules/module/etl/conf/etl.xml

@ -1,6 +1,6 @@ @@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--<!DOCTYPE module SYSTEM "../../../conf/superx-module.dtd"> -->
<module name="etl" version="0.5b" sachgebiet_id="270"
<module name="etl" version="0.4b" sachgebiet_id="270"
sachgebiet="Laderoutinen" systeminfo_id="270"
system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen">
<database name="superx" system="superx">
@ -161,174 +161,6 @@ parent="Laderoutinen">Abfragen zur Administration</thema> @@ -161,174 +161,6 @@ parent="Laderoutinen">Abfragen zur Administration</thema>
<nativeaction sql="update systeminfo set datum=today() where tid in (270)" scriptfile="" database=""/>
</action>
</etl-step>
<etl-step id="users_and_groups" name="Konfiguration User/Gruppen" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die User- und Gruppen-Rechtetabellen zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Userinfo-ID. Wenn 0, dann werden alle User geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="user_stammdaten">User-Daten</etl_transfer_group>
<etl_transfer_group uniquename="group_stammdaten">Gruppen-Daten</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="userinfo" transfer_group="user_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>benutzer</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_institution" transfer_group="user_stammdaten">
<whereclause> userid=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_masken_bez" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sachgeb_bez" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_pw" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sichtarten" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_sichten" transfer_group="user_stammdaten">
<whereclause> userinfo_id=$TID or $TID=0</whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<!--gruppen:-->
<etl_transfer_table name="groupinfo" transfer_group="group_stammdaten">
<whereclause> tid in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>name</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_field_pref" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_masken_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sachgeb_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sichtarten" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="group_sichten" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
<etl_transfer_table name="user_group_bez" transfer_group="group_stammdaten">
<whereclause> groupinfo_id in (select groupinfo_id from user_group_bez B
where B.userinfo_id=$TID)
or $TID=0 </whereclause>
<orderclause>1,2</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_users_groups" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
<etl-step id="sichten" name="Sichten" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die Sichten zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Sicht-TID. Wenn 0, dann werden alle Sichten geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="sichten_stammdaten">Sichten</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="sichten" transfer_group="sichten_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>name_intern</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_sichten" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_sichten_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
<etl-step id="sx_captions" name="Beschriftungen" type="manual" generateExecutable="etl_transfer" visible="true" active="true">
<description>Entlädt und lädt die Beschriftungen zur Übertragung in eine andere Säule </description>
<etl_transfer>
<etl_transfer_params>
<etl_transfer_param name="TID">
<description>Caption-TID. Wenn 0, dann werden alle Beschriftungen geladen</description>
<default>0</default>
</etl_transfer_param>
</etl_transfer_params>
<etl_transfer_groups>
<etl_transfer_group uniquename="sx_captions_stammdaten">Beschriftungen</etl_transfer_group>
</etl_transfer_groups>
<etl_transfer_tables>
<etl_transfer_table name="sx_captions" transfer_group="sx_captions_stammdaten">
<whereclause>tid=$TID or $TID=0</whereclause>
<orderclause>id,table_name,field_name,record_no,locale</orderclause>
</etl_transfer_table>
</etl_transfer_tables>
<etl_nativeactions>
<etl_nativeaction name="Trans-Schritt_sx_captions" filename="$SUPERX_DIR/db/module/etl/schluesseltabellen/etl_step_sx_captions_trans.sql" />
</etl_nativeactions>
</etl_transfer>
</etl-step>
</etl>
<install>
@ -337,9 +169,6 @@ parent="Laderoutinen">Abfragen zur Administration</thema> @@ -337,9 +169,6 @@ parent="Laderoutinen">Abfragen zur Administration</thema>
<loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_users_groups.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sichten.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sx_captions.sql" database=""/>
<!--<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sx_jobs_fuellen.sql" database=""/>-->
@ -364,16 +193,10 @@ version integer @@ -364,16 +193,10 @@ version integer
<action error="stop">
<loadtable refresh="true" delimiter="^" header="false" tabname="tmp_templates"><file path="$ETL_PFAD/schluesseltabellen/fm_templates.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/fm_templates_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_man.sql" database=""/>
</action>
<action error="stop">
<loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_users_groups.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sichten.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_sx_captions.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/>
</action>
</upgrade-step>

62
src-modules/module/etl/schluesseltabellen/etl_step_sichten_trans.sql

@ -1,62 +0,0 @@ @@ -1,62 +0,0 @@
begin work;
delete from sichten where name_intern in (select name_intern from tmp_sichten);
insert into sichten(parent,
systeminfoid,
art,
type,
name_intern,
name,
beschreibung,
sortnr,
quelle,
alt_hier_id,
treecfgtable,
treecfgid,
label,
user_rechte,
rechtequelle,
sesamkey,
standbutton,
attribut1,
attribut2,
attribut3,
attribut4,
xmlmaxentries,
gueltig_seit,
gueltig_bis,
cacheapplet,
cachexml,
aktiv) select parent,
systeminfoid,
art,
type,
name_intern,
name,
beschreibung,
sortnr,
quelle,
alt_hier_id,
treecfgtable,
treecfgid,
label,
user_rechte,
rechtequelle,
sesamkey,
standbutton,
attribut1,
attribut2,
attribut3,
attribut4,
xmlmaxentries,
gueltig_seit,
gueltig_bis,
cacheapplet,
cachexml,
aktiv from tmp_sichten;
drop table tmp_sichten;
commit;

52
src-modules/module/etl/schluesseltabellen/etl_step_users_and_groups_trans.sql

@ -1,52 +0,0 @@ @@ -1,52 +0,0 @@
begin work;
delete from userinfo where benutzer in (select benutzer from tmp_userinfo);
delete from user_institution where userid in (select userid from tmp_user_institution);
delete from user_masken_bez where userinfo_id in (select userinfo_id from tmp_user_masken_bez);
delete from user_sachgeb_bez where userinfo_id in (select userinfo_id from tmp_user_sachgeb_bez);
delete from user_pw where userinfo_id in (select userinfo_id from tmp_user_pw);
delete from user_sichtarten where userinfo_id in (select userinfo_id from tmp_user_sichtarten);
delete from user_sichten where userinfo_id in (select userinfo_id from tmp_user_sichten);
delete from groupinfo where tid in (select tid from tmp_groupinfo);
delete from group_field_pref where groupinfo_id in (select groupinfo_id from tmp_group_field_pref);
delete from group_masken_bez where groupinfo_id in (select groupinfo_id from tmp_group_masken_bez);
delete from group_sachgeb_bez where groupinfo_id in (select groupinfo_id from tmp_group_sachgeb_bez);
delete from group_sichtarten where groupinfo_id in (select groupinfo_id from tmp_group_sichtarten);
delete from group_sichten where groupinfo_id in (select groupinfo_id from tmp_group_sichten);
delete from user_group_bez where groupinfo_id in (select groupinfo_id from tmp_user_group_bez);
insert into userinfo select * from tmp_userinfo ;
insert into user_institution select * from tmp_user_institution ;
insert into user_masken_bez select * from tmp_user_masken_bez ;
insert into user_sachgeb_bez select * from tmp_user_sachgeb_bez ;
insert into user_pw select * from tmp_user_pw ;
insert into user_sichtarten select * from tmp_user_sichtarten ;
insert into user_sichten select * from tmp_user_sichten ;
insert into groupinfo select * from tmp_groupinfo ;
insert into group_field_pref select * from tmp_group_field_pref ;
insert into group_masken_bez select * from tmp_group_masken_bez ;
insert into group_sachgeb_bez select * from tmp_group_sachgeb_bez ;
insert into group_sichtarten select * from tmp_group_sichtarten;
insert into group_sichten select * from tmp_group_sichten ;
insert into user_group_bez select * from tmp_user_group_bez ;
drop table tmp_userinfo;
drop table tmp_user_institution;
drop table tmp_user_masken_bez;
drop table tmp_user_sachgeb_bez;
drop table tmp_user_pw;
drop table tmp_user_sichtarten;
drop table tmp_user_sichten;
drop table tmp_groupinfo;
drop table tmp_group_field_pref;
drop table tmp_group_masken_bez;
drop table tmp_group_sachgeb_bez;
drop table tmp_group_sichtarten;
drop table tmp_group_sichten;
drop table tmp_user_group_bez;
commit;

1021
src-modules/module/etl/schluesseltabellen/etl_transfer.xsl

File diff suppressed because it is too large Load Diff

42
src-modules/module/etl/upgrade/etl_upgrade_man.sql

@ -1,42 +0,0 @@ @@ -1,42 +0,0 @@
--obsolete jobs:
delete from etl_step
where tid in (select R.step_id
from etl_step_relation R, etl_job J
where R.job_id =J.tid
and J.uniquename in ('sx_select_mask',
'sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload')
and J.systeminfo_id=9);
delete from etl_step_property
where etl_step_id in (select R.step_id
from etl_step_relation R, etl_job J
where R.job_id =J.tid
and J.uniquename in ('sx_select_mask',
'sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload')
and J.systeminfo_id=9);
delete from etl_step_relation
where job_id in (select J.tid from etl_job J
where J.uniquename in ('sx_select_mask',
'sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload')
and J.systeminfo_id=9);
delete from etl_job
where uniquename in ('sx_select_mask',
'sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload')
and systeminfo_id=9;

7
src/de/superx/elt/EtlActionJob.java

@ -219,13 +219,6 @@ public class EtlActionJob extends EtlAction { @@ -219,13 +219,6 @@ public class EtlActionJob extends EtlAction {
}
else
{
if(jobReturnCode==1)
{
System.err.println(super.getActionLog());
throw new Exception("Fehler bei Jobausführung");
}
else
System.out.println("Jobausführung Erfolg");
try {
de.superx.elt.EtlUtils.saveFileContentsWithEncoding(outfile, sw.toString(), null);
} catch (Exception e) {

2
src/de/superx/elt/TableUploader.java

@ -79,7 +79,7 @@ public class TableUploader { @@ -79,7 +79,7 @@ public class TableUploader {
private boolean continueAfterError;
private boolean removeTrailingDelim=true;
private boolean isPostgres;
private boolean useBatch=false;
private boolean useBatch=true;
private static int maxCols=1000;
private String[] insert_cols = new String[maxCols];
private int[] insert_types = new int[maxCols];

BIN
superx/WEB-INF/lib/superx-etl.jar

Binary file not shown.
Loading…
Cancel
Save