Browse Source

ETL kompatibel zu kern 5.0 #1

etl_0.4_Release
Daniel Quathamer 1 year ago
parent
commit
98678b272d
  1. 12
      .classpath
  2. 17
      .project
  3. 15
      build.xml
  4. 1
      src-modules/module/etl/bin/etl_sx_select_mask.x
  5. 263
      src-modules/module/etl/conf/etl.xml
  6. 1130
      src-modules/module/etl/conf/etl_step_fuellen.xsl
  7. 298
      src-modules/module/etl/conf/etl_step_unload_fuellen.xsl
  8. 2
      src-modules/module/etl/conf/includes.txt
  9. 53
      src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql
  10. 3
      src-modules/module/etl/schluesseltabellen/fm_templates.unl
  11. 220
      src/de/superx/etl/util/GetOptsBackport.java
  12. 2
      src/de/superx/sxrest/MaskXml.java
  13. 16
      superx/WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam
  14. BIN
      superx/WEB-INF/lib/superx-etl.jar
  15. 12
      superx/edit/etl/etl_manager.jsp
  16. 1
      superx/edit/etl/systeminfo_list.inc

12
.classpath

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="src" path="src"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib/freemarker-2.3.25.jar"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib/postgresql-42.2.19.jar"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib/superx5.0.jar"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib/saxon-he-10.5.jar"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib_ext/servlet-api.jar"/>
<classpathentry kind="lib" path="/kern/superx/WEB-INF/lib/jersey-core-1.11.jar"/>
<classpathentry kind="output" path="superx/WEB-INF/classes"/>
</classpath>

17
.project

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>etl</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

15
build.xml

@ -13,7 +13,7 @@ ant -DWEBAPP=$WEBAPP distEtl
ant -DWEBAPP=$WEBAPP cleanBuildPath ant -DWEBAPP=$WEBAPP cleanBuildPath
--> -->
<dirname file="${ant.file.superx}" property="superxBuildBaseDir" /> <dirname file="${ant.file.superx}" property="superxBuildBaseDir" />
<property name="distDir" location="${superxBuildBaseDir}/../dist" /> <property name="distDir" location="${superxBuildBaseDir}/superx/WEB-INF/lib" />
<property environment="env" /> <property environment="env" />
<!-- set global properties for this build --> <!-- set global properties for this build -->
@ -75,6 +75,7 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<target name="all" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum"> <target name="all" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum">
<antcall target="module_copy_source" /> <antcall target="module_copy_source" />
<antcall target="call_module_scripts_create_ant" /> <antcall target="call_module_scripts_create_ant" />
<antcall target="module_copy_webapp_files" />
</target> </target>
<target name="module_copy_source" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum"> <target name="module_copy_source" description="Gruppe: Kopiere Modul Sourcen in den WEB-INF-Verzeichnisbaum">
<echo message="Start module_copy_sources für Modul ${MODULE} in Pfad ${MODULE_PATH}" /> <echo message="Start module_copy_sources für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
@ -98,6 +99,12 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<chmod dir="${MODULE_PATH}/rohdaten" perm="ugo+x" includes="**/*.x" /> <chmod dir="${MODULE_PATH}/rohdaten" perm="ugo+x" includes="**/*.x" />
</target> </target>
<target name="module_copy_webapp_files" description="Kopiere webapp-Dateien in den WEBAPP-Verzeichnisbaum">
<echo message="Start module_copy_webapp_files für Modul ${MODULE} in Pfad ${MODULE_PATH}" />
<copy todir="${WEBAPP}" overwrite="true">
<fileset dir="${BASE_DIR}/superx" />
</copy>
</target>
<target name="dist" description="Create Download package"> <target name="dist" description="Create Download package">
<subant target="erzeuge_paket"> <subant target="erzeuge_paket">
<!--<property name="basedir" value="${MODULE_PATH}/../../conf/" />--> <!--<property name="basedir" value="${MODULE_PATH}/../../conf/" />-->
@ -113,11 +120,11 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath
<fileset dir="${MODULE_PATH}/../../conf" includes="build.xml" /> <fileset dir="${MODULE_PATH}/../../conf" includes="build.xml" />
</subant> </subant>
</target> </target>
<target name="distEtl" depends="compileSuperx" description="Erzeugt die superx-etl${VERSION}.jar, d.h. das Servlet. "> <target name="distEtl" depends="compileSuperx" description="Erzeugt die superx-etl.jar. ">
<delete> <delete>
<fileset dir="${LIB_WEBAPP_DIR}" includes="superx-etl*.jar" /> <fileset dir="${distDir}" includes="superx-etl*.jar" />
</delete> </delete>
<jar destfile="${LIB_WEBAPP_DIR}/WEB-INF/lib/superx-etl${etl_version}.jar" basedir="${BUILD_PATH}" includes="de/superx/etl/** de/superx/sxrest/**" excludes="freemarker/**/* com/**/* javax/**/* **/*.java"> <jar destfile="${distDir}/superx-etl.jar" basedir="${BUILD_PATH}" includes="de/superx/etl/** de/superx/sxrest/**" excludes="freemarker/**/* com/**/* javax/**/* **/*.java">
<manifest> <manifest>
<!-- Who is building this jar? --> <!-- Who is building this jar? -->
<attribute name="Built-By" value="superx"/> <attribute name="Built-By" value="superx"/>

1
src-modules/module/etl/bin/etl_sx_select_mask.x

@ -38,6 +38,7 @@ CP=".:$JDBC_CLASSPATH"
CMD="java $JAVA_OPTS -cp $CP de.superx.etl.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:sx_select_mask -outfile:$outfile -params:TID=$TID" CMD="java $JAVA_OPTS -cp $CP de.superx.etl.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:sx_select_mask -outfile:$outfile -params:TID=$TID"
echo $CMD
$CMD $CMD

263
src-modules/module/etl/conf/etl.xml

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--<!DOCTYPE module SYSTEM "../../../conf/superx-module.dtd"> --> <!--<!DOCTYPE module SYSTEM "../../../conf/superx-module.dtd"> -->
<module name="etl" version="0.2" sachgebiet_id="270" <module name="etl" version="0.3" sachgebiet_id="270"
sachgebiet="Laderoutinen" systeminfo_id="270" sachgebiet="Laderoutinen" systeminfo_id="270"
system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen"> system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen">
<database name="superx" system="superx"> <database name="superx" system="superx">
@ -9,195 +9,29 @@ system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen">
rightname="CS_BIA_STANDARDREPORTS_ADMIN[ETL]" /> rightname="CS_BIA_STANDARDREPORTS_ADMIN[ETL]" />
</sachgebiete> </sachgebiete>
<table name="etl_step" thema="Administration" typ="Schlüsseltabelle" <table name="etl_job_metadata" thema="Administration" releaseUnload="demo"
releaseUnload="empty"> typ="Schlüsseltabelle">
<description>Schlüsseltabelle für Ladeschritte</description> <description>Schlüsseltabelle für Metadaten Ladejobs</description>
<columns> <columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/> <column name="tid" type="SERIAL" size="" default=""
<column name="uniquename" type="VARCHAR" size="255" default="" notnull="true" description="Unique Name" /> notnull="true" description="tid" isKey="true" />
<column name="caption" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" /> <column name="uniquename" type="VARCHAR" size="255" default=""
<column name="systeminfo_id" type="INTEGER" size="" default="" notnull="true" description="Komponente" /> notnull="true" description="Unique Name" />
<column name="step_type_id" type="INTEGER" size="2" default="" notnull="true" description="Typ des Schrittes" ><comment>1=Gruppierung, 2=Loadtable,3=nativeaction,4=loadmask,5=select</comment></column> <column name="caption" type="VARCHAR" size="255" default=""
<column name="logfile" type="VARCHAR" size="255" default="" notnull="false" description="Logdatei" /> notnull="" description="Bezeichnung" />
<column name="custom_step" type="SMALLINT" size="255" default="1" notnull="false" description="Ladeschritt custom"><comment>1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software</comment></column> <column name="systeminfo_id" type="INTEGER" size="" default=""
notnull="true" description="Komponente" />
</columns> </columns>
<primaryKeys><rs> <primaryKeys>
<rs>
<row> <row>
<fld name='table_cat'>superx</fld> <fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld> <fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_step</fld> <fld name='table_name'>etl_job_metadata</fld>
<fld name='column_name'>tid</fld> <fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld> <fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_step_pk</fld> <fld name='pk_name'>etl_job_metadata_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
<index name="ix_etl_step1" type="unique">
<index-column name="uniquename" />
<index-column name="systeminfo_id" />
</index>
</indexes>
</table>
<table name="etl_step_type" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Arten von Ladeschritten, z.B. loadtable, nativeaction,unload</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="uniquename" type="VARCHAR" size="255" default="" notnull="true" description="Unique Name" />
<column name="caption" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" />
<column name="handler" type="VARCHAR" size="255" default="" notnull="false" description="Handler-Anwendung" />
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_step_type</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_step_type_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
</indexes>
</table>
<table name="etl_step_type_param" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Parameter für Arten von Ladeschritten</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="etl_step_type_id" type="INTEGER" size="" default="" notnull="true" description="Arten des Ladeschritts" />
<column name="uniquename" type="VARCHAR" size="255" default="" notnull="true" description="Unique Name" />
<column name="name" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" />
<column name="param_default" type="VARCHAR" size="255" default="" notnull="false" description="Defaultwert" />
<!-- in Zukunft ggf. typisiert: <column name="param_type" type="VARCHAR" size="255" default="" notnull="false" description="Defaultwert" />
-->
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_step_type_param</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_step_type_param_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
</indexes>
</table>
<table name="etl_step_property" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Eigenschaften von Ladeschritten</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="etl_step_id" type="INTEGER" size="" default="" notnull="true" description="Arten des Ladeschritts" />
<column name="prop_name" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" />
<column name="prop_value" type="TEXT" size="255" default="" notnull="false" description="Wert" />
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_step_property</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_step_property_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
</indexes>
</table>
<table name="etl_step_relation" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Beziehungen zwischen Ladeschritten, und Ausführungssteuerung</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="step_id" type="INTEGER" size="2" default="" notnull="true" description="Ladeschritt" ></column>
<column name="parent_step_id" type="INTEGER" size="2" default="" notnull="false" description="Übergeordneter Ladeschritt" ></column>
<column name="job_id" type="INTEGER" size="2" default="" notnull="true" description="Ladejob" ></column>
<column name="force_continue" type="SMALLINT" size="2" default="1" notnull="false" description="Ladejob bei Fehler weiterführen" ></column>
<column name="step_active" type="SMALLINT" size="2" default="1" notnull="false" description="Ladeschritt aktiv" ></column>
<column name="sortnr" type="INTEGER" size="2" default="1" notnull="false" description="Ladeschritt Sortiernr." ></column>
<column name="custom_step" type="SMALLINT" size="2" default="1" notnull="false" description="Ladeschritt custom" ><comment>1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software</comment></column>
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_step_relation</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_step_relation_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
</indexes>
</table>
<table name="etl_job" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Ladejobs</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="uniquename" type="VARCHAR" size="255" default="" notnull="true" description="Schlüssel" />
<column name="caption" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" />
<column name="systeminfo_id" type="INTEGER" size="" default="" notnull="true" description="Komponente" />
<column name="logfile" type="VARCHAR" size="255" default="" notnull="false" description="Logdatei" />
<column name="custom_job" type="SMALLINT" size="255" default="1" notnull="false" description="Job custom" ><comment>1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software</comment></column>
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_job</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_job_pk</fld>
</row>
</rs>
</primaryKeys>
<indexes>
<index name="ix_etl_job1" type="unique">
<index-column name="uniquename" />
</index>
</indexes>
</table>
<table name="etl_job_param" thema="Administration" typ="Schlüsseltabelle"
releaseUnload="empty">
<description>Schlüsseltabelle für Parameter für Ladejobs</description>
<columns>
<column name="tid" type="SERIAL" size="" default="" notnull="true" description="tid" isKey="true"/>
<column name="etl_job_id" type="INTEGER" size="" default="" notnull="true" description="Ladejob" />
<column name="uniquename" type="VARCHAR" size="255" default="" notnull="true" description="Unique Name" />
<column name="name" type="VARCHAR" size="255" default="" notnull="" description="Bezeichnung" />
<column name="param_default" type="VARCHAR" size="255" default="" notnull="false" description="Defaultwert" />
<!-- in Zukunft ggf. typisiert: <column name="param_type" type="VARCHAR" size="255" default="" notnull="false" description="Defaultwert" />
-->
</columns>
<primaryKeys><rs>
<row>
<fld name='table_cat'>superx</fld>
<fld name='table_schem'>superx</fld>
<fld name='table_name'>etl_job_param</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>etl_job_param_pk</fld>
</row> </row>
</rs> </rs>
</primaryKeys> </primaryKeys>
@ -261,7 +95,62 @@ parent="Laderoutinen">Abfragen zur Administration</thema>
</data-integrity> </data-integrity>
</database> </database>
<install_upgrade_job>
<install-upgrade-step name="pre_upgrade" type="container" id="etl_preupgrade_containernode">
<action error="stop"/>
</install-upgrade-step>
<install-upgrade-step name="upgrade_tables" type="" id="etl_upgrade_tables_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_tables_pg.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="upgrade_fields" type="" id="etl_upgrade_fields_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_fields_pg.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="upgrade_views" type="" id="etl_upgrade_views_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_views_pg.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="install_functions" type="container" id="etl_install_functions_containernode">
<action error="stop"/>
</install-upgrade-step>
<install-upgrade-step name="upgrade_indices" type="" id="etl_upgrade_indices_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_indexes_pg.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="fill_tables" type="" id="etl_fill_tables_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/conf/sx_tables_fuellen.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="fill_fields" type="" id="etl_fill_fields_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/conf/sx_fields_fuellen.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="fill_unload_parameter" type="" id="etl_fill_unload_parameter_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/conf/unload_params_fuellen.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="update_module_metadata" type="" id="etl_update_module_metadata_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/conf/module_metadata.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="update_topic_tree" type="" id="etl_update_topictree_actionstep">
<action error="stop">
<nativeaction sql="" scriptfile="$ETL_PFAD/upgrade/etl_upgrade_themenbaum.sql" database=""/>
</action>
</install-upgrade-step>
<install-upgrade-step name="install_masks" type="container" id="etl_install_masks_containernode">
<action error="stop"/>
</install-upgrade-step>
</install_upgrade_job>
<!-- ********************* Liste der ETL-Prozesse ************************** --> <!-- ********************* Liste der ETL-Prozesse ************************** -->
@ -303,13 +192,12 @@ version integer
</action> </action>
<action error="stop"> <action error="stop">
<loadtable refresh="true" delimiter="^" header="false" tabname="tmp_templates"><file path="$ETL_PFAD/schluesseltabellen/fm_templates.unl"/></loadtable> <loadtable refresh="true" delimiter="^" header="false" tabname="tmp_templates"><file path="$ETL_PFAD/schluesseltabellen/fm_templates.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/fm_templates_fuellen.sql" database=""/>
</action> </action>
<action> <action error="stop">
<loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable> <loadtable refresh="true" delimiter="^" header="false" tabname="etl_step_type"><file path="$ETL_PFAD/schluesseltabellen/etl_step_type.unl"/></loadtable>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/etl_step_fuellen.sql" database=""/>
<nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/> <nativeaction sql="" scriptfile="$ETL_PFAD/schluesseltabellen/sachgebiete_fuellen.sql" database=""/>
</action> </action>
</upgrade-step> </upgrade-step>
</upgrade> </upgrade>
@ -366,7 +254,7 @@ mode="update_insert_delete_copy">
<field-selection complete="false" /> <field-selection complete="false" />
<customfield name="tid" nullFieldValue="" /> <customfield name="tid" nullFieldValue="" />
<customfield name="job_id" type="hidden" overrideValue="tid" /> <customfield name="job_id" type="hidden" overrideValue="tid" />
<customfield name="step_id" nullFieldValue="" visibleSize="30" /> <customfield name="step_id" readOnly="true" nullFieldValue="" visibleSize="30" />
<customfield name="parent_step_id" nullFieldValue="" visibleSize="30" /> <customfield name="parent_step_id" nullFieldValue="" visibleSize="30" />
<customfield name="force_continue" nullFieldValue="" pattern="" visibleSize="10" /> <customfield name="force_continue" nullFieldValue="" pattern="" visibleSize="10" />
<customfield name="step_active" nullFieldValue="" pattern="" visibleSize="20" /> <customfield name="step_active" nullFieldValue="" pattern="" visibleSize="20" />
@ -430,4 +318,3 @@ mode="full">
</dbforms> </dbforms>
</module> </module>

1130
src-modules/module/etl/conf/etl_step_fuellen.xsl

File diff suppressed because it is too large Load Diff

298
src-modules/module/etl/conf/etl_step_unload_fuellen.xsl

@ -0,0 +1,298 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xsl:stylesheet version="2.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:import href="etl_step_fuellen.xsl" />
<xsl:param name="sx_client"/>
<xsl:param name="dbsystem"/>
<xsl:param name="jobtype"/>
<xsl:output method="text" name="text"/>
<xsl:variable name="unknown_id" select="999999" />
<xsl:variable name="unknown" select="Unbekannt" />
<xsl:variable name="singleQuote">&#39;</xsl:variable>
<xsl:template match="/">
<xsl:text><![CDATA[
--Freemarker Template
--Erzeugt von $ETL_PFAD/conf/etl_step_unload_fuellen.xsl
<#include "ETL_MAKROS"/>
<sqlvars>
<sqlvar name="etl_step_exists">
select sp_table_exists('etl_step') from xdummy;
</sqlvar>
<sqlvar name="sourcesystem">
select param_val from unload_params where param_id='SOURCESYSTEM' and systeminfo_id=]]></xsl:text><xsl:value-of select="/superx-unload/module/@systeminfo_id"/><xsl:text><![CDATA[;
</sqlvar>
<sqlvar name="database">
select param_val from unload_params where param_id='DATABASE' and systeminfo_id=]]></xsl:text><xsl:value-of select="/superx-unload/module/@systeminfo_id"/><xsl:text><![CDATA[;
</sqlvar>
]]></xsl:text>
<xsl:for-each select="/superx-unload/parameters/param">
<xsl:text><![CDATA[<sqlvar name="]]></xsl:text><xsl:value-of select="@name"/><xsl:text><![CDATA[">
select replace(param_val,'''','''''') from unload_params where systeminfo_id=]]></xsl:text><xsl:value-of select="/superx-unload/module/@systeminfo_id"/><xsl:text><![CDATA[
and param_id=']]></xsl:text><xsl:value-of select="@name"/><xsl:text><![CDATA[';
</sqlvar>
]]></xsl:text>
</xsl:for-each>
<xsl:text><![CDATA[
</sqlvars>
]]></xsl:text>
<xsl:variable name="job_uniquename" select="concat(/superx-unload/module/@id,'_unload')" />
<xsl:variable name="module_name" select="/superx-unload/module/@id" />
<xsl:call-template name="initJob">
<xsl:with-param name="modulkuerzel" select="/superx-unload/module/@id"/>
<xsl:with-param name="uniquename" select="$job_uniquename" />
<xsl:with-param name="caption" select="concat(/superx-unload/module,' Unload')" />
<xsl:with-param name="systeminfo_id" select="/superx-unload/module/@systeminfo_id"/>
<xsl:with-param name="logfile" select="concat('L_',/superx-unload/module/@id,'_Unload.log')" />
</xsl:call-template>
<xsl:for-each select="/superx-unload/sourcesystems/sourcesystem">
<xsl:variable name="sourcesystem" select="@name" />
<xsl:text>--Unload für sourcesystem=</xsl:text><xsl:value-of select="$sourcesystem"/><xsl:text><![CDATA[
<#if sourcesystem=="]]></xsl:text><xsl:value-of select="$sourcesystem"/><xsl:text><![CDATA[">
--First the steps:
]]></xsl:text>
<xsl:text><![CDATA[
<#assign etl_steps = [
]]></xsl:text>
<xsl:for-each select="/superx-unload/unload-job">
<xsl:variable name="nr" select="position()" />
<xsl:if test="extraction-sql/@sourcesystem=$sourcesystem">
<xsl:variable name="step_type">
<xsl:choose>
<xsl:when test="@transfer-unload-file=''">
<xsl:text>DOQUERY</xsl:text>
</xsl:when>
<xsl:otherwise>
<xsl:text>UNLOAD</xsl:text>
</xsl:otherwise>
</xsl:choose>
</xsl:variable>
<xsl:variable name="logfile">
<xsl:value-of select="concat('L_',$module_name,'_unload.log')"/>
</xsl:variable>
<xsl:call-template name="insertStep">
<xsl:with-param name="uniquename" select="concat($module_name,'_unload_',@id)"/>
<xsl:with-param name="parent_uniquename" select="''"/>
<xsl:with-param name="job_uniquename" select="$job_uniquename"/>
<xsl:with-param name="caption" select="concat('Unload ',$module_name,' Ladeschritt ',@id)"/>
<xsl:with-param name="nr" select="$nr"/>
<xsl:with-param name="step_type" select="$step_type"/>
<xsl:with-param name="force_continue" select="'0'"/>
<xsl:with-param name="logfile" select="$logfile"/>
<xsl:with-param name="systeminfo_id" select="/superx-unload/module/@systeminfo_id"/>
<xsl:with-param name="isFinal" select="'0'"/>
</xsl:call-template>
</xsl:if>
</xsl:for-each>
<xsl:call-template name="insertStep">
<xsl:with-param name="uniquename" select="concat($module_name,'_unload_finish')"/>
<xsl:with-param name="caption" select="concat('Ende Unload ',$module_name)"/>
<xsl:with-param name="parent_uniquename" select="''"/>
<xsl:with-param name="job_uniquename" select="$job_uniquename"/>
<xsl:with-param name="nr" select="1000"/>
<xsl:with-param name="step_type" select="'MSG'"/>
<xsl:with-param name="force_continue" select="'0'"/>
<xsl:with-param name="logfile" select="''"/>
<xsl:with-param name="systeminfo_id" select="/superx-unload/module/@systeminfo_id"/>
<xsl:with-param name="isFinal" select="'1'"/>
</xsl:call-template>
<xsl:text><![CDATA[
] />
]]></xsl:text>
<xsl:text><![CDATA[
</#if>
]]></xsl:text>
</xsl:for-each>
<!--Ende etl_step Schleife-->
<xsl:for-each select="/superx-unload/sourcesystems/sourcesystem">
<xsl:call-template name="insertStepProperties">
<xsl:with-param name="sourcesystem" select="@name" />
<xsl:with-param name="database" select="'POSTGRES'" />
<xsl:with-param name="module_name" select="$module_name"/>
</xsl:call-template>
<xsl:call-template name="insertStepProperty">
<xsl:with-param name="uniquename" select="concat($module_name,'_unload_finish')"/>
<xsl:with-param name="caption" select="concat('Ende Unload ',/superx-unload/module/@name)"/>
<xsl:with-param name="nr" select="1"/>
<xsl:with-param name="step_type" select="'MSG'"/>
<xsl:with-param name="isFinal" select="'1'"/>
</xsl:call-template>
<xsl:text><![CDATA[
] />
]]></xsl:text>
<xsl:text><![CDATA[
</#if>
]]></xsl:text>
</xsl:for-each>
<xsl:text><![CDATA[
<#if etl_step_exists==1>
<@ETL_STEPS_FUELLEN />
</#if>
]]></xsl:text>
</xsl:template>
<xsl:template name="insertStepProperties">
<xsl:param name="sourcesystem" />
<xsl:param name="database" />
<xsl:param name="module_name" />
<xsl:variable name="unloadDefaultVersion"><xsl:value-of select="/superx-unload/parameters/param[@name='VERSION']/@defaultValue"/></xsl:variable>
<xsl:text>--Unload properties für sourcesystem=</xsl:text><xsl:value-of select="$sourcesystem"/>
<xsl:text>,database=</xsl:text><xsl:value-of select="$database"/>
<xsl:text>,unloadDefaultVersion=</xsl:text><xsl:value-of select="$unloadDefaultVersion"/>
<xsl:text><![CDATA[
<#if sourcesystem=="]]></xsl:text><xsl:value-of select="$sourcesystem"/>
<xsl:text>" &amp;&amp; database=="</xsl:text><xsl:value-of select="$database"/>
<xsl:text><![CDATA[">
--then the properties:
]]></xsl:text>
<xsl:text><![CDATA[
<#assign etl_step_properties = [
]]></xsl:text>
<xsl:for-each select="/superx-unload/unload-job">
<xsl:variable name="step_uniquename" select="concat($module_name,'_unload_',@id)" />
<xsl:variable name="nr" select="position()" />
<!-- first the params-->
<xsl:for-each select="/superx-unload/parameters/param">
<xsl:choose>
<xsl:when test="@name='JDBC_PARAM' and $sourcesystem='hisinone'">
<xsl:text><![CDATA[
{"etl_step":"]]></xsl:text><xsl:value-of select="$step_uniquename"/>
<xsl:text>", "prop_name":"</xsl:text><xsl:value-of select="@name"/>
<xsl:text>", "prop_value":"set search_path to hisinone"</xsl:text>
<xsl:text><![CDATA[},
]]>
</xsl:text>
</xsl:when>
<xsl:otherwise>
<xsl:text><![CDATA[
{"etl_step":"]]></xsl:text><xsl:value-of select="$step_uniquename"/>
<xsl:text>", "prop_name":"</xsl:text><xsl:value-of select="@name"/>
<xsl:text>", "prop_value":</xsl:text><xsl:value-of select="@name" />
<xsl:text><![CDATA[},
]]>
</xsl:text>
</xsl:otherwise>
</xsl:choose>
</xsl:for-each>
<!--now the extraction-sqls-->
<xsl:for-each select="extraction-sql">
<xsl:if test="@sourcesystem=$sourcesystem and (@database=$database or @database='') and (contains(@version,$unloadDefaultVersion) or @version='')">
<xsl:choose>
<xsl:when test="../@transfer-unload-file=''">
<xsl:call-template name="insertStepProperty">
<xsl:with-param name="uniquename" select="$step_uniquename"/>
<xsl:with-param name="caption" select="'SQL Script'"/>
<xsl:with-param name="nr" select="$nr + 10"/>
<xsl:with-param name="step_type" select="'DOQUERY'"/>
<xsl:with-param name="isFinal" select="'0'"/>
<xsl:with-param name="sql" select=".">
<!--klappt nicht: <xsl:call-template name="parseSqlForUnloadParams">
<xsl:with-param name="sql" select="." />
</xsl:call-template>-->
</xsl:with-param>
<xsl:with-param name="dbconnection" select="$sourcesystem"/>
<xsl:with-param name="sqlVarDbconnection" select="'eduetl'"/>
</xsl:call-template>
</xsl:when>
<xsl:otherwise>
<xsl:variable name="outPath"><xsl:text>$</xsl:text><xsl:call-template name="ucase">
<xsl:with-param name="s1" select="$module_name" /></xsl:call-template><xsl:text>_LOAD_PFAD/</xsl:text>
<xsl:value-of select="../@transfer-unload-file"/></xsl:variable>
<xsl:call-template name="insertStepProperty">
<xsl:with-param name="uniquename" select="$step_uniquename"/>
<xsl:with-param name="caption" select="'SQL Unload'"/>
<xsl:with-param name="nr" select="$nr + 10"/>
<xsl:with-param name="step_type" select="'UNLOAD'"/>
<xsl:with-param name="isFinal" select="'0'"/>
<xsl:with-param name="sql" select=".">
<!--klappt nicht: <xsl:call-template name="parseSqlForUnloadParams">
<xsl:with-param name="sql" select="." />
</xsl:call-template>-->
</xsl:with-param>
<xsl:with-param name="outfile" select="$outPath"/>
<xsl:with-param name="format" select="'csv'"/>
<xsl:with-param name="dbconnection" select="$sourcesystem"/>
<xsl:with-param name="sqlVarDbconnection" select="'eduetl'"/>
</xsl:call-template>
</xsl:otherwise>
</xsl:choose>
</xsl:if>
</xsl:for-each>
</xsl:for-each>
</xsl:template>
<xsl:template name="parseSqlForUnloadParams">
<xsl:param name="sql" />
<xsl:for-each select="/superx-unload/parameters/param">
<xsl:variable name="param_id" select="@name" />
<xsl:call-template name="search_replace">
<xsl:with-param name="string" select="$sql"/>
<xsl:with-param name="search" select="'a'"/>
<xsl:with-param name="replace" select="'A'"/>
</xsl:call-template>
</xsl:for-each>
<!--<xsl:value-of select="$sql"/>
<xsl:with-param name="search" select="concat('$',@name)"/>
<xsl:with-param name="replace" select="concat('&quot;+',@name,'+&quot;')"/>
-->
</xsl:template>
<xsl:template name="ucase">
<xsl:param name="s1"/>
<xsl:value-of select="translate($s1,'abcdefghijklmnopqrstuvwxyz ','ABCDEFGHIJKLMNOPQRSTUVWXYZ_')" />
</xsl:template>
</xsl:stylesheet>

2
src-modules/module/etl/conf/includes.txt

@ -1,4 +1,4 @@
doku/etl_modul/etl.html doku/etl_modul/etl.html
WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam
WEB-INF/lib/superx-etl0.2.jar WEB-INF/lib/superx-etl.jar

53
src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql

@ -19,7 +19,6 @@
{"etl_job":"sx_insert_mask","param_name":"TID", "name":"Maskennr.", "param_default":""}, {"etl_job":"sx_insert_mask","param_name":"TID", "name":"Maskennr.", "param_default":""},
{"etl_job":"sx_insert_mask","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/masken/$TID.xml"}, {"etl_job":"sx_insert_mask","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/masken/$TID.xml"},
{"etl_job":"sx_insert_mask","param_name":"FORMAT", "name":"Format", "param_default":"XML"}, {"etl_job":"sx_insert_mask","param_name":"FORMAT", "name":"Format", "param_default":"XML"},
{"etl_job":"sx_insert_mask","param_name":"SUPERX_DIR", "name":"Superx-Pfad zu WEB-INF/conf/edustore", "param_default":"$WEBAPP/WEB-INF/conf/edustore"},
{"etl_job":"kern_konstanten_update","param_name":"SUPERX_DIR", "name":"Superx-Pfad zu WEB-INF/conf/edustore", "param_default":"$WEBAPP/WEB-INF/conf/edustore"}, {"etl_job":"kern_konstanten_update","param_name":"SUPERX_DIR", "name":"Superx-Pfad zu WEB-INF/conf/edustore", "param_default":"$WEBAPP/WEB-INF/conf/edustore"},
{"etl_job":"kern_konstanten_update","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/install/schluesseltabellen/kern_feste_konstanten_fuellen.sql"} {"etl_job":"kern_konstanten_update","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/install/schluesseltabellen/kern_feste_konstanten_fuellen.sql"}
@ -36,8 +35,6 @@
{"etl_job":"sx_select_mask", "uniquename":"unload_maske_system_bez", "name":"maske_system_bez entladen", "type":"UNLOAD", "parent":"unload_masken_stammdaten"}, {"etl_job":"sx_select_mask", "uniquename":"unload_maske_system_bez", "name":"maske_system_bez entladen", "type":"UNLOAD", "parent":"unload_masken_stammdaten"},
{"etl_job":"sx_select_mask", "uniquename":"unload_themenbaum", "name":"themenbaum entladen", "type":"UNLOAD"}, {"etl_job":"sx_select_mask", "uniquename":"unload_themenbaum", "name":"themenbaum entladen", "type":"UNLOAD"},
{"etl_job":"sx_select_mask", "uniquename":"unload_sx_mask_style", "name":"sx_mask_style entladen", "type":"UNLOAD"}, {"etl_job":"sx_select_mask", "uniquename":"unload_sx_mask_style", "name":"sx_mask_style entladen", "type":"UNLOAD"},
{"etl_job":"sx_select_mask", "uniquename":"unload_sx_stylesheets", "name":"sx_stylesheets entladen", "type":"UNLOAD"},
{"etl_job":"sx_select_mask", "uniquename":"unload_stylesheet_field", "name":"stylesheet_field entladen", "type":"UNLOAD"},
{"etl_job":"sx_insert_mask", "uniquename":"delete_maskeninfo", "name":"Maskeninfo löschen", "type":"DOQUERY" }, {"etl_job":"sx_insert_mask", "uniquename":"delete_maskeninfo", "name":"Maskeninfo löschen", "type":"DOQUERY" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_maskeninfo", "name":"Maskeninfo hochladen", "type":"LOAD" }, {"etl_job":"sx_insert_mask", "uniquename":"upload_maskeninfo", "name":"Maskeninfo hochladen", "type":"LOAD" },
@ -56,18 +53,6 @@
{"etl_job":"sx_insert_mask", "uniquename":"delete_maske_system_bez", "name":"maske_system_bez löschen", "type":"DOQUERY" }, {"etl_job":"sx_insert_mask", "uniquename":"delete_maske_system_bez", "name":"maske_system_bez löschen", "type":"DOQUERY" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_maske_system_bez", "name":"maske_system_bez hochladen", "type":"LOAD" }, {"etl_job":"sx_insert_mask", "uniquename":"upload_maske_system_bez", "name":"maske_system_bez hochladen", "type":"LOAD" },
{"etl_job":"sx_insert_mask", "uniquename":"create_tmp_etl_themenbaum", "name":"Tabelle tmp_etl_themenbaum erzeugen", "type":"DOSQL" },
{"etl_job":"sx_insert_mask", "uniquename":"create_tmp_etl_stylesheets", "name":"Tabelle tmp_etl_stylesheets erzeugen", "type":"DOSQL" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_themenbaum", "name":"tmp_etl_themenbaum hochladen", "type":"LOAD" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_stylesheets", "name":"tmp_etl_stylesheets hochladen", "type":"LOAD" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_mask_style", "name":"tmp_etl_mask_style hochladen", "type":"LOAD" },
{"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_stylesheet_field", "name":"tmp_etl_stylesheet_field hochladen", "type":"LOAD" },
{"etl_job":"sx_insert_mask", "uniquename":"themenbaum_fuellen", "name":"themenbaum_fuellen", "type":"DOSQL" },
{"etl_job":"sx_insert_mask", "uniquename":"sx_stylesheets_fuellen", "name":"sx_stylesheets_fuellen", "type":"DOSQL" },
{"etl_job":"kern_konstanten_update", "uniquename":"kern_konstanten_update", "name":"Kern Konstanten aktualisieren", "type":"DOSQL" } {"etl_job":"kern_konstanten_update", "uniquename":"kern_konstanten_update", "name":"Kern Konstanten aktualisieren", "type":"DOSQL" }
@ -80,10 +65,8 @@
{"etl_step":"unload_masken_felder_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,felderinfo_id from masken_felder_bez where maskeninfo_id=$TID order by 1,2" }, {"etl_step":"unload_masken_felder_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,felderinfo_id from masken_felder_bez where maskeninfo_id=$TID order by 1,2" },
{"etl_step":"unload_sachgeb_maske_bez","prop_name":"select_stmt", "prop_value":"select sachgebiete_id,maskeninfo_id from sachgeb_maske_bez where maskeninfo_id=$TID order by 1,2" }, {"etl_step":"unload_sachgeb_maske_bez","prop_name":"select_stmt", "prop_value":"select sachgebiete_id,maskeninfo_id from sachgeb_maske_bez where maskeninfo_id=$TID order by 1,2" },
{"etl_step":"unload_maske_system_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,systeminfo_id from maske_system_bez where maskeninfo_id=$TID order by 1,2" }, {"etl_step":"unload_maske_system_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,systeminfo_id from maske_system_bez where maskeninfo_id=$TID order by 1,2" },
{"etl_step":"unload_themenbaum","prop_name":"select_stmt", "prop_value":"select T.tid,name,maskeninfo_id,parent,(select name from themenbaum where tid=T.parent) as parent_name,gueltig_seit,gueltig_bis,erlaeuterung,sort,css_class from themenbaum T where maskeninfo_id=$TID order by 2,1" }, {"etl_step":"unload_themenbaum","prop_name":"select_stmt", "prop_value":"select name,maskeninfo_id,parent,(select name from themenbaum where tid=T.parent) as parent_name,gueltig_seit,gueltig_bis,erlaeuterung,sort,css_class from themenbaum T where maskeninfo_id=$TID order by 2,1" },
{"etl_step":"unload_sx_mask_style","prop_name":"select_stmt", "prop_value":"select S.tid,S.maskeninfo_id,S.stylesheet_id,S.ord,(select filename from sx_stylesheets where tid=S.stylesheet_id) as stylesheet_filename from sx_mask_style S where maskeninfo_id=$TID order by 1,2,3" }, {"etl_step":"unload_sx_mask_style","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,stylesheet_id,ord,(select filename from sx_stylesheets where tid=S.stylesheet_id) as stylesheet_filename from sx_mask_style S where maskeninfo_id=$TID order by 1,2,3" },
{"etl_step":"unload_sx_stylesheets","prop_name":"select_stmt", "prop_value":"select S.tid,S.filename,S.caption,S.description,S.relation,S.useragent,S.contenttype from sx_stylesheets S, sx_mask_style M where S.tid=M.stylesheet_id and M.maskeninfo_id=$TID order by 1,2,3" },
{"etl_step":"unload_stylesheet_field","prop_name":"select_stmt", "prop_value":"select F.tid,F.stylesheet_id,S.filename as stylesheet_filename,F.tablename,F.fieldname FROM stylesheet_field F, sx_stylesheets S, sx_mask_style M where F.stylesheet_id=S.tid and S.tid=M.stylesheet_id and M.maskeninfo_id=$TID order by 1,2,3,4,5" },
{"etl_step":"delete_maskeninfo","prop_name":"select_stmt", "prop_value":"delete from maskeninfo where tid=$TID" }, {"etl_step":"delete_maskeninfo","prop_name":"select_stmt", "prop_value":"delete from maskeninfo where tid=$TID" },
{"etl_step":"upload_maskeninfo","prop_name":"target_table", "prop_value":"maskeninfo" }, {"etl_step":"upload_maskeninfo","prop_name":"target_table", "prop_value":"maskeninfo" },
@ -115,32 +98,6 @@
{"etl_step":"upload_maske_system_bez","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_maske_system_bez\"]/rs/row" }, {"etl_step":"upload_maske_system_bez","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_maske_system_bez\"]/rs/row" },
{"etl_step":"upload_maske_system_bez","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, {"etl_step":"upload_maske_system_bez","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" },
{"etl_step":"create_tmp_etl_themenbaum","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/create_tmp_etl_themenbaum.sql" },
{"etl_step":"create_tmp_etl_stylesheets","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/create_tmp_etl_stylesheets.sql" },
{"etl_step":"upload_tmp_etl_themenbaum","prop_name":"target_table", "prop_value":"tmp_etl_themenbaum" },
{"etl_step":"upload_tmp_etl_themenbaum","prop_name":"format", "prop_value":"xml" },
{"etl_step":"upload_tmp_etl_themenbaum","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_themenbaum\"]/rs/row" },
{"etl_step":"upload_tmp_etl_themenbaum","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" },
{"etl_step":"upload_tmp_etl_stylesheets","prop_name":"target_table", "prop_value":"tmp_etl_stylesheets" },
{"etl_step":"upload_tmp_etl_stylesheets","prop_name":"format", "prop_value":"xml" },
{"etl_step":"upload_tmp_etl_stylesheets","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_sx_stylesheets\"]/rs/row" },
{"etl_step":"upload_tmp_etl_stylesheets","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" },
{"etl_step":"upload_tmp_etl_mask_style","prop_name":"target_table", "prop_value":"tmp_etl_mask_style" },
{"etl_step":"upload_tmp_etl_mask_style","prop_name":"format", "prop_value":"xml" },
{"etl_step":"upload_tmp_etl_mask_style","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_sx_mask_style\"]/rs/row" },
{"etl_step":"upload_tmp_etl_mask_style","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" },
{"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"target_table", "prop_value":"tmp_etl_stylesheet_field" },
{"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"format", "prop_value":"xml" },
{"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_stylesheet_field\"]/rs/row" },
{"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" },
{"etl_step":"themenbaum_fuellen","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/themenbaum_fuellen.sql" },
{"etl_step":"sx_stylesheets_fuellen","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/sx_stylesheets_fuellen.sql" },
{"etl_step":"kern_konstanten_update","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" } {"etl_step":"kern_konstanten_update","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }
@ -149,9 +106,9 @@
<#assign testfaelle = [ <#assign testfaelle = [
{"testcase":"test_sx_select_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_select_mask'" }, {"testcase":"test_sx_select_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_select_mask'" },
{"testcase":"test_sx_insert_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_insert_mask'" }, {"testcase":"test_sx_insert_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_insert_mask'" },
{"testcase":"test_sx_insert_mask_params","assertion":4, "sql":"select count(*) from etl_job_param P, etl_job J where J.tid=P.etl_job_id and J.uniquename='sx_insert_mask'" }, {"testcase":"test_sx_insert_mask_params","assertion":3, "sql":"select count(*) from etl_job_param P, etl_job J where J.tid=P.etl_job_id and J.uniquename='sx_insert_mask'" },
{"testcase":"test_sx_insert_mask_steps","assertion":18, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_insert_mask'" }, {"testcase":"test_sx_insert_mask_steps","assertion":10, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_insert_mask'" },
{"testcase":"test_sx_select_mask_steps","assertion":10, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_select_mask'" } {"testcase":"test_sx_select_mask_steps","assertion":8, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_select_mask'" }
] /> ] />

3
src-modules/module/etl/schluesseltabellen/fm_templates.unl

@ -236,9 +236,6 @@ FROM tmp_etl_job_param \
update tmp_etl_step set step_type=(select T.tid from etl_step_type T\ update tmp_etl_step set step_type=(select T.tid from etl_step_type T\
where T.uniquename=tmp_etl_step.step_type_uniquename);\ where T.uniquename=tmp_etl_step.step_type_uniquename);\
\ \
select * from tmp_etl_step\
where step_type is null;--_uniquename from tmp_etl_step;\
\
\ \
--vorhandene Steps erkennen:\ --vorhandene Steps erkennen:\
<#if SQLdialect='Postgres'>\ <#if SQLdialect='Postgres'>\

220
src/de/superx/etl/util/GetOptsBackport.java

@ -0,0 +1,220 @@
package de.superx.etl.util;
/**
* Class to analyze arguments passed to Java class
* Makes ETL compatible to BI 2021.06 and higher
*/
public class GetOptsBackport{
public enum Options{
opt_logger("-logger"),
opt_dbprops("-dbproperties"),
opt_table("-table"),
opt_outFormat("-outFormat"),
opt_outFile("-outfile"),
opt_outfileAttribute("-outfileattribute"),
opt_cancel("-cancel"),
opt_sql("-sqlfile"),
opt_sqlAttribute("-sqlattribute"),
opt_delim("-delim"),
opt_header("-header"),
opt_params("-params"),
opt_param("-param"),
opt_in("-in"),
opt_inFormat("-informat"),
opt_out("-out"),
opt_xsl("-xsl"),
opt_xslPfad("-xslpfad"),
opt_fopxconf("-fopxconf"),
opt_user("-user"),
opt_tid("-tid"),
opt_mandID("-mandantenid"),
opt_locale("-locale"),
opt_simpPars("-simpleParser"),
opt_webinf("-web_inf_pfad"),
opt_module("-module_pfad"),
opt_jobUniqueName("-job_uniquename"),
opt_uploadpath("-path_to_uploadfile"),
opt_method("-method"),
opt_xml("-xml"),
opt_xmlconfig("-xmlconfig"),
opt_db("-database"),
opt_version("-version"),
opt_unl("-unl"),
opt_unlParams("-unload_params"),
opt_jrxml("-jrxml"),
opt_jasper("-jasper"),
opt_jrPrint("-jrprint"),
opt_ignorePagination("-ignore_pagination"),
opt_idescr("-i_descr"),
opt_odescr("-o_descr"),
opt_element("-element"),
opt_function("-function"),
opt_mode("-mode"),
opt_inserts("-inserts"),
opt_encoding("-encoding"),
opt_hsnr("-hsnr"),
opt_noDelete("-nodelete"),
opt_pause("-pause"),
opt_jahr("-jahr"),
opt_datentyp("-datentyp"),
opt_runs("-runs"),
opt_checkval("-checkval"),
opt_url("-url"),
opt_setname("-setname"),
opt_kernTabellenFreischalten("-kern_tabellen_freischalten"),
opt_noguiVar0("-nogui"), opt_noguiVar1("-no-gui"), opt_noguiVar2("--no-gui"),
opt_job("-job");
private String value;
private Options(String value) {
this.value = value;
}
public String stringValue() {
return value;
}
}
private static String[] arguments = null;
private GetOptsBackport(){} //Do not instatiate
/**
* If you pass the arguments you want to analyse to this static helper class
* using setOpts, you can later use the short form of the methods e.g.
* isPresent(String option) without having to pass the arguments again.
*
* @param args
*/
public static void setOpts(String args[]) {
arguments = args;
}
/**
* checks if the arguments passed before by setOpts contain the given option
*
* @param Options
* option
* @return true if arguments contain the option, i.e. one String which
* starts with the option-String
*/
public static boolean isPresent(Options option){
if(arguments == null){
throw new IllegalStateException("must either call setOpts() before or call the long version of this method");
}
return isPresent(arguments, option);
}
/**
* checks if the arguments contain the given option
*
* @param args
* @param Options
* option
* @return true if arguments contain the option, i.e. one String which
* starts with the option-String
*/
public static boolean isPresent(String args[], Options option){
boolean result = false;
for(int i = 0; i < args.length; i++){
if (args[i] != null && args[i].toLowerCase().startsWith(option.stringValue())){
result = true;
}
}
return result;
}
/**
* Gets a named option from the arguments passed before with setOps. E.g.
* getOpt("--log") would return the "--log:true" in the arguments
*
* @param Options
* option
* @return String option and value
*/
public static String getOpt(Options option){
if (arguments == null){
throw new IllegalStateException("must either call setOpts() before or call the long version of this method");
}
return getOpt(arguments, option);
}
/**
* Gets a named option from the arguments. E.g. getOpt("--log") would return
* the "--log:true" in the arguments
*
* @param Options
* option
* @return String option and value
*/
public static String getOpt(String args[], Options option){
String result = null;
for(int i=0; i < args.length; i++){
if(args[i].toLowerCase().startsWith(option.stringValue())){
result = args[i];
}
}
return result;
}
/**
* Gets the value of an option from the arguments passed before with setOpts
* if for example, you ask getValue("--log:") and the arguments passed
* before with setOpts contain a String "--log:true") "true" is returned
*
* @param Options
* option
* @return String value of the option
*/
public static String getValue(Options option){
if (arguments == null){
throw new IllegalStateException("must either call setOpts() before or call the long version of this method");
}
return getValue(arguments, option);
}
/**
* Gets the value of an option, if for example, you ask getValue("--log:")
* and the arguments passed before with setOpts contain a String
* "--log:true") "true" is returned
*
* @param Options
* option
* @return String value of the option
*/
public static String getValue(String args[], Options option) {
String value = null, temp = getOpt(args, option);
int pos = option.stringValue().length();
if(temp.charAt(pos) == ':') {
pos++;
}
value = temp.substring(pos);
return value;
}
/**
* Prüft, ob alle notwendigen Optionen angegeben sind.
*
* @param Options[]
* opts, ein Array mit Optionen die überprüft werden sollen
* @return null - alles OK, ansonsten String mit den fehlenden Optionen
*/
public static String isAllRequiredOptionsPresent(Options[] opts){
String missingOpts = null;
for (Options o : opts){
if(!isPresent(o)){
if(missingOpts == null){
missingOpts = o.stringValue()+", ";
}else{
missingOpts += o.stringValue()+", ";
}
}
}
return missingOpts;
}
}

2
src/de/superx/sxrest/MaskXml.java

@ -55,7 +55,7 @@ public class MaskXml {
@GET @GET
@Path("/get/{param}") @Path("/get/{param}")
@Produces(MediaType.APPLICATION_XML) @Produces(MediaType.TEXT_PLAIN)
public String printXml(@PathParam("param") String tid,@Context HttpServletRequest request) { public String printXml(@PathParam("param") String tid,@Context HttpServletRequest request) {

16
superx/WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam

@ -0,0 +1,16 @@
#Beispielumgebung für das ETL-Modul:
#SUPERX_MODULE=$SUPERX_DIR/db/module
#SUPERX_ROHDATEN=rohdaten
ETL_PFAD=$SUPERX_MODULE/etl; export ETL_PFAD
ETL_ERRORDAT=$ETL_PFAD/etl_update.err; export ETL_ERRORDAT
#Hier stehen die Rohdaten (im Unterverzeichnis unl)
ETL_LOAD_PFAD=$ETL_PFAD/$SUPERX_ROHDATEN; export ETL_LOAD_PFAD
ETL_ERRORMAIL=$ERRORMAIL; export ETL_ERRORMAIL
ETL_LOGMAIL=$LOGMAIL; export ETL_LOGMAIL
PATH=$ETL_PFAD/bin:$PATH
export PATH

BIN
superx/WEB-INF/lib/superx-etl.jar

Binary file not shown.

12
superx/edit/etl/etl_manager.jsp

@ -51,13 +51,17 @@ document.getElementById("MaskSubHeader").innerHTML="<form method=\"post\" name=\
function getMaskSrc(tid) function getMaskSrc(tid)
{ {
//document.getElementById("MaskHeader").innerHTML="Masken-Quellcode laden";
//document.getElementById("MaskSubHeader").innerHTML="Maske TID="+tid;
var xhttp = new XMLHttpRequest(); var xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() { xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) { if (this.readyState == 4 && this.status == 200) {
var xmlDoc=this.responseXML; document.getElementById("Inhalt").innerHTML = this.responseText;
var xmlTxt=new XMLSerializer().serializeToString(xmlDoc); //alert(this.responseText);
document.getElementById("Inhalt").value=xmlTxt; }
else
{
document.getElementById("Inhalt").innerHTML="Serveranfrage Antwort... "+this.status;
} }
}; };
xhttp.open("GET", "../../sxrest/maskxml/get/"+tid, true); xhttp.open("GET", "../../sxrest/maskxml/get/"+tid, true);

1
superx/edit/etl/systeminfo_list.inc

@ -42,4 +42,3 @@ componentVersion="Unbekannt";
<db:footer> <db:footer>
</ul> </ul>
</db:footer></db:dbform> </db:footer></db:dbform>

Loading…
Cancel
Save