diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..cdbd67d --- /dev/null +++ b/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..1112c94 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + etl + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/build.xml b/build.xml index 9e6f510..0b2e8b6 100644 --- a/build.xml +++ b/build.xml @@ -13,7 +13,7 @@ ant -DWEBAPP=$WEBAPP distEtl ant -DWEBAPP=$WEBAPP cleanBuildPath --> - + @@ -75,6 +75,7 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath + @@ -98,6 +99,12 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath + + + + + + @@ -113,11 +120,11 @@ ant -DWEBAPP=$WEBAPP cleanBuildPath - + - + - + diff --git a/src-modules/module/etl/bin/etl_sx_select_mask.x b/src-modules/module/etl/bin/etl_sx_select_mask.x index c4d9c10..0712929 100755 --- a/src-modules/module/etl/bin/etl_sx_select_mask.x +++ b/src-modules/module/etl/bin/etl_sx_select_mask.x @@ -38,6 +38,7 @@ CP=".:$JDBC_CLASSPATH" CMD="java $JAVA_OPTS -cp $CP de.superx.etl.bin.EtlJobExecutor -dbproperties:$DB_PROPERTIES -job:sx_select_mask -outfile:$outfile -params:TID=$TID" +echo $CMD $CMD diff --git a/src-modules/module/etl/conf/etl.xml b/src-modules/module/etl/conf/etl.xml index 61448c1..2d85cd1 100644 --- a/src-modules/module/etl/conf/etl.xml +++ b/src-modules/module/etl/conf/etl.xml @@ -1,6 +1,6 @@ - @@ -9,200 +9,34 @@ system="Laderoutinen" thema="Laderoutinen" thema_parent="Abfragen"> rightname="CS_BIA_STANDARDREPORTS_ADMIN[ETL]" /> - - Schlüsseltabelle für Ladeschritte +
+ Schlüsseltabelle für Metadaten Ladejobs - - - - - 1=Gruppierung, 2=Loadtable,3=nativeaction,4=loadmask,5=select - - 1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software + + + + - - -superx -superx -etl_step -tid -1 -etl_step_pk - - - - - - - - - -
- - - - - Schlüsseltabelle für Arten von Ladeschritten, z.B. loadtable, nativeaction,unload - - - - - - - - - -superx -superx -etl_step_type -tid -1 -etl_step_type_pk - - - - - -
- - - Schlüsseltabelle für Parameter für Arten von Ladeschritten - - - - - - - - - - - -superx -superx -etl_step_type_param -tid -1 -etl_step_type_param_pk - - - - - -
- - - Schlüsseltabelle für Eigenschaften von Ladeschritten - - - - - - - - - -superx -superx -etl_step_property -tid -1 -etl_step_property_pk - - - - - -
- - - Schlüsseltabelle für Beziehungen zwischen Ladeschritten, und Ausführungssteuerung - - - - - - - - - 1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software - - - -superx -superx -etl_step_relation -tid -1 -etl_step_relation_pk - - - - - -
- - - Schlüsseltabelle für Ladejobs - - - - - - - 1=ja, d.h. von der Hochschule angepaßt, 0=nein, d.h. Auslieferung der Software - - - -superx -superx -etl_job -tid -1 -etl_job_pk - - - - - - - - -
- - Schlüsseltabelle für Parameter für Ladejobs - - - - - - - - - - - -superx -superx -etl_job_param -tid -1 -etl_job_param_pk - - - - - + + + + superx + superx + etl_job_metadata + tid + 1 + etl_job_metadata_pk + + + + +
@@ -261,7 +95,62 @@ parent="Laderoutinen">Abfragen zur Administration - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -303,13 +192,12 @@ version integer + - - + - @@ -366,7 +254,7 @@ mode="update_insert_delete_copy"> - + @@ -430,4 +318,3 @@ mode="full"> - diff --git a/src-modules/module/etl/conf/etl_step_fuellen.xsl b/src-modules/module/etl/conf/etl_step_fuellen.xsl index 0cb4ecb..7377736 100644 --- a/src-modules/module/etl/conf/etl_step_fuellen.xsl +++ b/src-modules/module/etl/conf/etl_step_fuellen.xsl @@ -11,6 +11,8 @@ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> ' +" +'' + + + + @@ -116,6 +124,467 @@ select sp_table_exists('etl_step') from xdummy; + + ]]> + + + + + + + + + + + + + + + ]]> + + + ]]> + + + + + + + + + + + + + ]]> + + + + + + + + + + + + + + + + ]]> + + + + + + + + + + + + + + + + ]]> + + + + + + +<@ETL_STEPS_FUELLEN /> + + + +]]> + + + + + + + + + + + + + + + +", "uniquename":" +", "name":" +", "type":" + +", "parent":" + + + +, + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", "prop_name":" +msg", "prop_value":" + + + + + + + +", "prop_name":" +select_stmt", "prop_value":" + + + +", "prop_name":" +dbconnection", "prop_value":" + + + + + + +", "prop_name":" +dbconnection", "prop_value":" + + + +", "prop_name":" +sqlVarDbconnection", "prop_value":" + + + +", "prop_name":"PATH_TO_INPUTFILE", "prop_value":" + + + + + +", "prop_name":"target_table", "prop_value":" + + + + +", "prop_name":"path_to_inputfile", "prop_value":" + + + +", "prop_name":"truncateTargetTable", "prop_value":" + + + +", "prop_name":"format", "prop_value":" + + + + + + + + +", "prop_name":" +dbconnection", "prop_value":" + + + +", "prop_name":" +sqlVarDbconnection", "prop_value":" + + + +", "prop_name":"select_stmt", "prop_value":" + + + + +", "prop_name":"outfile", "prop_value":" + + + +", "prop_name":"format", "prop_value":" + + + + + + + + + +, + + + + + + + + + + + + + + + + + +<#assign etl_job_params = [ + {"etl_job":"]]> + ","param_name":" + + + + + ","param_name":" + + + + +]]> + + + + + + +Load + + +Transformation + + +Aggregation + + +Test + + +Standdatum + + + + + + + + + + + + + + + DOSQL + + + + DOQUERY + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -133,65 +602,22 @@ select sp_table_exists('etl_step') from xdummy; + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + - - + @@ -208,78 +634,50 @@ select sp_table_exists('etl_step') from xdummy; -<#assign etl_step_properties = [ - - -]]> - - - - - - - - - - - - - - - - - - - - - - - - + ]]> + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + +]]> + + + + + + + - - - + + - + - + + + + + + + + + + + + + + + - @@ -295,18 +693,19 @@ select sp_table_exists('etl_step') from xdummy; ] /> ]]> - - - - + + + + + + + + + + - - - - - - - @@ -352,7 +751,7 @@ select sp_table_exists('etl_step') from xdummy; - + @@ -399,6 +798,12 @@ select sp_table_exists('etl_step') from xdummy; +]]> + + + + - + @@ -455,340 +860,35 @@ select sp_table_exists('etl_step') from xdummy; - - - - - - - - - - - - - - - - - - - - - - - -]]> - - - - - - -<@ETL_STEPS_FUELLEN /> - - - -]]> - - - - - - - - - - - - - - - -", "uniquename":" -", "name":" -", "type":" - -", "parent":" - - - -, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -", "prop_name":" -msg", "prop_value":" - - - - - - - -", "prop_name":" -select_stmt", "prop_value":" - - - - - -", "prop_name":"PATH_TO_INPUTFILE", "prop_value":" - - - - - -", "prop_name":"target_table", "prop_value":" - - - - -", "prop_name":"path_to_inputfile", "prop_value":" - - - -", "prop_name":"truncateTargetTable", "prop_value":" - - - -", "prop_name":"format", "prop_value":" - - + + + + + + - - - - -, - - + - + - - - - - - - - - - - - -<#assign etl_job_params = [ - {"etl_job":"]]> - ","param_name":" - - - - - ","param_name":" - - + + + + + + - - -<#assign etl_steps = [ -]]> - + - - - - -Load - - -Transformation - - -Aggregation - - -Test - - -Standdatum - - - +]]> - - - - - - - - - DOSQL - - - - DOQUERY - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -886,28 +986,71 @@ select sp_table_exists('etl_step') from xdummy; + + + - + - + + + + + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + + @@ -1324,4 +1467,145 @@ select sp_table_exists('etl_step') from xdummy; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src-modules/module/etl/conf/etl_step_unload_fuellen.xsl b/src-modules/module/etl/conf/etl_step_unload_fuellen.xsl new file mode 100644 index 0000000..42b1e36 --- /dev/null +++ b/src-modules/module/etl/conf/etl_step_unload_fuellen.xsl @@ -0,0 +1,298 @@ + + + + + + + + + + + + +' + + + + + + +select sp_table_exists('etl_step') from xdummy; + + +select param_val from unload_params where param_id='SOURCESYSTEM' and systeminfo_id=]]> + +select param_val from unload_params where param_id='DATABASE' and systeminfo_id=]]> +]]> + + +select replace(param_val,'''','''''') from unload_params where systeminfo_id=]]> +]]> + + +]]> + + + + + + + + + + + + + + + + +--Unload für sourcesystem= + +--First the steps: +]]> + + + + + + + + + + + +DOQUERY + + +UNLOAD + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +]]> + + +]]> + + + + + + + + + + + + + + + + + + + + + + + +]]> + + +]]> + + + + + + + + +<@ETL_STEPS_FUELLEN /> + + + +]]> + + + + + + + + + +--Unload properties für sourcesystem= +,database= +,unloadDefaultVersion= + + +" && database==" + + +--then the properties: +]]> + + + + + + + + + + + +", "prop_name":" +", "prop_value":"set search_path to hisinone" + + + + + +", "prop_name":" +", "prop_value": + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +$ +_LOAD_PFAD/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src-modules/module/etl/conf/includes.txt b/src-modules/module/etl/conf/includes.txt index 6abd85a..e3887f0 100644 --- a/src-modules/module/etl/conf/includes.txt +++ b/src-modules/module/etl/conf/includes.txt @@ -1,4 +1,4 @@ doku/etl_modul/etl.html WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam -WEB-INF/lib/superx-etl0.2.jar +WEB-INF/lib/superx-etl.jar diff --git a/src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql b/src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql index 50b8f5a..ce8e9f6 100644 --- a/src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql +++ b/src-modules/module/etl/schluesseltabellen/etl_step_fuellen.sql @@ -19,7 +19,6 @@ {"etl_job":"sx_insert_mask","param_name":"TID", "name":"Maskennr.", "param_default":""}, {"etl_job":"sx_insert_mask","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/masken/$TID.xml"}, {"etl_job":"sx_insert_mask","param_name":"FORMAT", "name":"Format", "param_default":"XML"}, - {"etl_job":"sx_insert_mask","param_name":"SUPERX_DIR", "name":"Superx-Pfad zu WEB-INF/conf/edustore", "param_default":"$WEBAPP/WEB-INF/conf/edustore"}, {"etl_job":"kern_konstanten_update","param_name":"SUPERX_DIR", "name":"Superx-Pfad zu WEB-INF/conf/edustore", "param_default":"$WEBAPP/WEB-INF/conf/edustore"}, {"etl_job":"kern_konstanten_update","param_name":"PATH_TO_INPUTFILE", "name":"Eingabepfad", "param_default":"$SUPERX_DIR/db/install/schluesseltabellen/kern_feste_konstanten_fuellen.sql"} @@ -36,8 +35,6 @@ {"etl_job":"sx_select_mask", "uniquename":"unload_maske_system_bez", "name":"maske_system_bez entladen", "type":"UNLOAD", "parent":"unload_masken_stammdaten"}, {"etl_job":"sx_select_mask", "uniquename":"unload_themenbaum", "name":"themenbaum entladen", "type":"UNLOAD"}, {"etl_job":"sx_select_mask", "uniquename":"unload_sx_mask_style", "name":"sx_mask_style entladen", "type":"UNLOAD"}, - {"etl_job":"sx_select_mask", "uniquename":"unload_sx_stylesheets", "name":"sx_stylesheets entladen", "type":"UNLOAD"}, - {"etl_job":"sx_select_mask", "uniquename":"unload_stylesheet_field", "name":"stylesheet_field entladen", "type":"UNLOAD"}, {"etl_job":"sx_insert_mask", "uniquename":"delete_maskeninfo", "name":"Maskeninfo löschen", "type":"DOQUERY" }, {"etl_job":"sx_insert_mask", "uniquename":"upload_maskeninfo", "name":"Maskeninfo hochladen", "type":"LOAD" }, @@ -55,18 +52,6 @@ {"etl_job":"sx_insert_mask", "uniquename":"delete_maske_system_bez", "name":"maske_system_bez löschen", "type":"DOQUERY" }, {"etl_job":"sx_insert_mask", "uniquename":"upload_maske_system_bez", "name":"maske_system_bez hochladen", "type":"LOAD" }, - - {"etl_job":"sx_insert_mask", "uniquename":"create_tmp_etl_themenbaum", "name":"Tabelle tmp_etl_themenbaum erzeugen", "type":"DOSQL" }, - {"etl_job":"sx_insert_mask", "uniquename":"create_tmp_etl_stylesheets", "name":"Tabelle tmp_etl_stylesheets erzeugen", "type":"DOSQL" }, - - {"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_themenbaum", "name":"tmp_etl_themenbaum hochladen", "type":"LOAD" }, - {"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_stylesheets", "name":"tmp_etl_stylesheets hochladen", "type":"LOAD" }, - {"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_mask_style", "name":"tmp_etl_mask_style hochladen", "type":"LOAD" }, - {"etl_job":"sx_insert_mask", "uniquename":"upload_tmp_etl_stylesheet_field", "name":"tmp_etl_stylesheet_field hochladen", "type":"LOAD" }, - - {"etl_job":"sx_insert_mask", "uniquename":"themenbaum_fuellen", "name":"themenbaum_fuellen", "type":"DOSQL" }, - {"etl_job":"sx_insert_mask", "uniquename":"sx_stylesheets_fuellen", "name":"sx_stylesheets_fuellen", "type":"DOSQL" }, - {"etl_job":"kern_konstanten_update", "uniquename":"kern_konstanten_update", "name":"Kern Konstanten aktualisieren", "type":"DOSQL" } @@ -80,10 +65,8 @@ {"etl_step":"unload_masken_felder_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,felderinfo_id from masken_felder_bez where maskeninfo_id=$TID order by 1,2" }, {"etl_step":"unload_sachgeb_maske_bez","prop_name":"select_stmt", "prop_value":"select sachgebiete_id,maskeninfo_id from sachgeb_maske_bez where maskeninfo_id=$TID order by 1,2" }, {"etl_step":"unload_maske_system_bez","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,systeminfo_id from maske_system_bez where maskeninfo_id=$TID order by 1,2" }, - {"etl_step":"unload_themenbaum","prop_name":"select_stmt", "prop_value":"select T.tid,name,maskeninfo_id,parent,(select name from themenbaum where tid=T.parent) as parent_name,gueltig_seit,gueltig_bis,erlaeuterung,sort,css_class from themenbaum T where maskeninfo_id=$TID order by 2,1" }, - {"etl_step":"unload_sx_mask_style","prop_name":"select_stmt", "prop_value":"select S.tid,S.maskeninfo_id,S.stylesheet_id,S.ord,(select filename from sx_stylesheets where tid=S.stylesheet_id) as stylesheet_filename from sx_mask_style S where maskeninfo_id=$TID order by 1,2,3" }, - {"etl_step":"unload_sx_stylesheets","prop_name":"select_stmt", "prop_value":"select S.tid,S.filename,S.caption,S.description,S.relation,S.useragent,S.contenttype from sx_stylesheets S, sx_mask_style M where S.tid=M.stylesheet_id and M.maskeninfo_id=$TID order by 1,2,3" }, - {"etl_step":"unload_stylesheet_field","prop_name":"select_stmt", "prop_value":"select F.tid,F.stylesheet_id,S.filename as stylesheet_filename,F.tablename,F.fieldname FROM stylesheet_field F, sx_stylesheets S, sx_mask_style M where F.stylesheet_id=S.tid and S.tid=M.stylesheet_id and M.maskeninfo_id=$TID order by 1,2,3,4,5" }, + {"etl_step":"unload_themenbaum","prop_name":"select_stmt", "prop_value":"select name,maskeninfo_id,parent,(select name from themenbaum where tid=T.parent) as parent_name,gueltig_seit,gueltig_bis,erlaeuterung,sort,css_class from themenbaum T where maskeninfo_id=$TID order by 2,1" }, + {"etl_step":"unload_sx_mask_style","prop_name":"select_stmt", "prop_value":"select maskeninfo_id,stylesheet_id,ord,(select filename from sx_stylesheets where tid=S.stylesheet_id) as stylesheet_filename from sx_mask_style S where maskeninfo_id=$TID order by 1,2,3" }, {"etl_step":"delete_maskeninfo","prop_name":"select_stmt", "prop_value":"delete from maskeninfo where tid=$TID" }, {"etl_step":"upload_maskeninfo","prop_name":"target_table", "prop_value":"maskeninfo" }, @@ -114,33 +97,7 @@ {"etl_step":"upload_maske_system_bez","prop_name":"format", "prop_value":"xml" }, {"etl_step":"upload_maske_system_bez","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_maske_system_bez\"]/rs/row" }, {"etl_step":"upload_maske_system_bez","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, - - {"etl_step":"create_tmp_etl_themenbaum","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/create_tmp_etl_themenbaum.sql" }, - {"etl_step":"create_tmp_etl_stylesheets","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/create_tmp_etl_stylesheets.sql" }, - {"etl_step":"upload_tmp_etl_themenbaum","prop_name":"target_table", "prop_value":"tmp_etl_themenbaum" }, - {"etl_step":"upload_tmp_etl_themenbaum","prop_name":"format", "prop_value":"xml" }, - {"etl_step":"upload_tmp_etl_themenbaum","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_themenbaum\"]/rs/row" }, - {"etl_step":"upload_tmp_etl_themenbaum","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, - - {"etl_step":"upload_tmp_etl_stylesheets","prop_name":"target_table", "prop_value":"tmp_etl_stylesheets" }, - {"etl_step":"upload_tmp_etl_stylesheets","prop_name":"format", "prop_value":"xml" }, - {"etl_step":"upload_tmp_etl_stylesheets","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_sx_stylesheets\"]/rs/row" }, - {"etl_step":"upload_tmp_etl_stylesheets","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, - - {"etl_step":"upload_tmp_etl_mask_style","prop_name":"target_table", "prop_value":"tmp_etl_mask_style" }, - {"etl_step":"upload_tmp_etl_mask_style","prop_name":"format", "prop_value":"xml" }, - {"etl_step":"upload_tmp_etl_mask_style","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_sx_mask_style\"]/rs/row" }, - {"etl_step":"upload_tmp_etl_mask_style","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, - - {"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"target_table", "prop_value":"tmp_etl_stylesheet_field" }, - {"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"format", "prop_value":"xml" }, - {"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"search_path", "prop_value":"/etlAction/unload [@name=\"unload_stylesheet_field\"]/rs/row" }, - {"etl_step":"upload_tmp_etl_stylesheet_field","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" }, - - {"etl_step":"themenbaum_fuellen","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/themenbaum_fuellen.sql" }, - {"etl_step":"sx_stylesheets_fuellen","prop_name":"PATH_TO_INPUTFILE", "prop_value":"$SUPERX_DIR/db/module/etl/schluesseltabellen/sx_stylesheets_fuellen.sql" }, - {"etl_step":"kern_konstanten_update","prop_name":"path_to_inputfile", "prop_value":"$PATH_TO_INPUTFILE" } @@ -149,9 +106,9 @@ <#assign testfaelle = [ {"testcase":"test_sx_select_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_select_mask'" }, {"testcase":"test_sx_insert_mask","assertion":1, "sql":"select count(*) from etl_job where uniquename='sx_insert_mask'" }, - {"testcase":"test_sx_insert_mask_params","assertion":4, "sql":"select count(*) from etl_job_param P, etl_job J where J.tid=P.etl_job_id and J.uniquename='sx_insert_mask'" }, - {"testcase":"test_sx_insert_mask_steps","assertion":18, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_insert_mask'" }, - {"testcase":"test_sx_select_mask_steps","assertion":10, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_select_mask'" } + {"testcase":"test_sx_insert_mask_params","assertion":3, "sql":"select count(*) from etl_job_param P, etl_job J where J.tid=P.etl_job_id and J.uniquename='sx_insert_mask'" }, + {"testcase":"test_sx_insert_mask_steps","assertion":10, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_insert_mask'" }, + {"testcase":"test_sx_select_mask_steps","assertion":8, "sql":"select count(*) from etl_step S, etl_step_relation R, etl_job J where J.tid=R.job_id and S.tid=R.step_id and J.uniquename='sx_select_mask'" } ] /> @@ -390,8 +347,8 @@ FROM tmp_etl_job_param update tmp_etl_step set step_type=(select T.tid from etl_step_type T where T.uniquename=tmp_etl_step.step_type_uniquename); --- select * from tmp_etl_step --- where step_type is null;--_uniquename from tmp_etl_step; +--select * from tmp_etl_step +--where step_type is null;--_uniquename from tmp_etl_step; --vorhandene Steps erkennen: diff --git a/src-modules/module/etl/schluesseltabellen/fm_templates.unl b/src-modules/module/etl/schluesseltabellen/fm_templates.unl index 0073229..205bf7d 100644 --- a/src-modules/module/etl/schluesseltabellen/fm_templates.unl +++ b/src-modules/module/etl/schluesseltabellen/fm_templates.unl @@ -236,9 +236,6 @@ FROM tmp_etl_job_param \ update tmp_etl_step set step_type=(select T.tid from etl_step_type T\ where T.uniquename=tmp_etl_step.step_type_uniquename);\ \ -select * from tmp_etl_step\ -where step_type is null;--_uniquename from tmp_etl_step;\ -\ \ --vorhandene Steps erkennen:\ <#if SQLdialect='Postgres'>\ diff --git a/src/de/superx/etl/util/GetOptsBackport.java b/src/de/superx/etl/util/GetOptsBackport.java new file mode 100644 index 0000000..dea4d8e --- /dev/null +++ b/src/de/superx/etl/util/GetOptsBackport.java @@ -0,0 +1,220 @@ +package de.superx.etl.util; + +/** + * Class to analyze arguments passed to Java class + * Makes ETL compatible to BI 2021.06 and higher + */ +public class GetOptsBackport{ + + public enum Options{ + opt_logger("-logger"), + opt_dbprops("-dbproperties"), + opt_table("-table"), + opt_outFormat("-outFormat"), + opt_outFile("-outfile"), + opt_outfileAttribute("-outfileattribute"), + opt_cancel("-cancel"), + opt_sql("-sqlfile"), + opt_sqlAttribute("-sqlattribute"), + opt_delim("-delim"), + opt_header("-header"), + opt_params("-params"), + opt_param("-param"), + opt_in("-in"), + opt_inFormat("-informat"), + opt_out("-out"), + opt_xsl("-xsl"), + opt_xslPfad("-xslpfad"), + opt_fopxconf("-fopxconf"), + opt_user("-user"), + opt_tid("-tid"), + opt_mandID("-mandantenid"), + opt_locale("-locale"), + opt_simpPars("-simpleParser"), + opt_webinf("-web_inf_pfad"), + opt_module("-module_pfad"), + opt_jobUniqueName("-job_uniquename"), + opt_uploadpath("-path_to_uploadfile"), + opt_method("-method"), + opt_xml("-xml"), + opt_xmlconfig("-xmlconfig"), + opt_db("-database"), + opt_version("-version"), + opt_unl("-unl"), + opt_unlParams("-unload_params"), + opt_jrxml("-jrxml"), + opt_jasper("-jasper"), + opt_jrPrint("-jrprint"), + opt_ignorePagination("-ignore_pagination"), + opt_idescr("-i_descr"), + opt_odescr("-o_descr"), + opt_element("-element"), + opt_function("-function"), + opt_mode("-mode"), + opt_inserts("-inserts"), + opt_encoding("-encoding"), + opt_hsnr("-hsnr"), + opt_noDelete("-nodelete"), + opt_pause("-pause"), + opt_jahr("-jahr"), + opt_datentyp("-datentyp"), + opt_runs("-runs"), + opt_checkval("-checkval"), + opt_url("-url"), + opt_setname("-setname"), + + opt_kernTabellenFreischalten("-kern_tabellen_freischalten"), + opt_noguiVar0("-nogui"), opt_noguiVar1("-no-gui"), opt_noguiVar2("--no-gui"), + opt_job("-job"); + + private String value; + private Options(String value) { + this.value = value; + } + + public String stringValue() { + return value; + } + } + + private static String[] arguments = null; + private GetOptsBackport(){} //Do not instatiate + + /** + * If you pass the arguments you want to analyse to this static helper class + * using setOpts, you can later use the short form of the methods e.g. + * isPresent(String option) without having to pass the arguments again. + * + * @param args + */ + public static void setOpts(String args[]) { + arguments = args; + } + + /** + * checks if the arguments passed before by setOpts contain the given option + * + * @param Options + * option + * @return true if arguments contain the option, i.e. one String which + * starts with the option-String + */ + public static boolean isPresent(Options option){ + if(arguments == null){ + throw new IllegalStateException("must either call setOpts() before or call the long version of this method"); + } + return isPresent(arguments, option); + } + + /** + * checks if the arguments contain the given option + * + * @param args + * @param Options + * option + * @return true if arguments contain the option, i.e. one String which + * starts with the option-String + */ + public static boolean isPresent(String args[], Options option){ + boolean result = false; + + for(int i = 0; i < args.length; i++){ + if (args[i] != null && args[i].toLowerCase().startsWith(option.stringValue())){ + result = true; + } + } + + return result; + } + + /** + * Gets a named option from the arguments passed before with setOps. E.g. + * getOpt("--log") would return the "--log:true" in the arguments + * + * @param Options + * option + * @return String option and value + */ + public static String getOpt(Options option){ + if (arguments == null){ + throw new IllegalStateException("must either call setOpts() before or call the long version of this method"); + } + return getOpt(arguments, option); + } + + /** + * Gets a named option from the arguments. E.g. getOpt("--log") would return + * the "--log:true" in the arguments + * + * @param Options + * option + * @return String option and value + */ + public static String getOpt(String args[], Options option){ + String result = null; + for(int i=0; i < args.length; i++){ + if(args[i].toLowerCase().startsWith(option.stringValue())){ + result = args[i]; + } + } + return result; + } + + /** + * Gets the value of an option from the arguments passed before with setOpts + * if for example, you ask getValue("--log:") and the arguments passed + * before with setOpts contain a String "--log:true") "true" is returned + * + * @param Options + * option + * @return String value of the option + */ + public static String getValue(Options option){ + if (arguments == null){ + throw new IllegalStateException("must either call setOpts() before or call the long version of this method"); + } + return getValue(arguments, option); + } + + /** + * Gets the value of an option, if for example, you ask getValue("--log:") + * and the arguments passed before with setOpts contain a String + * "--log:true") "true" is returned + * + * @param Options + * option + * @return String value of the option + */ + public static String getValue(String args[], Options option) { + String value = null, temp = getOpt(args, option); + int pos = option.stringValue().length(); + if(temp.charAt(pos) == ':') { + pos++; + } + value = temp.substring(pos); + return value; + } + + /** + * Prüft, ob alle notwendigen Optionen angegeben sind. + * + * @param Options[] + * opts, ein Array mit Optionen die überprüft werden sollen + * @return null - alles OK, ansonsten String mit den fehlenden Optionen + */ + public static String isAllRequiredOptionsPresent(Options[] opts){ + String missingOpts = null; + for (Options o : opts){ + if(!isPresent(o)){ + if(missingOpts == null){ + missingOpts = o.stringValue()+", "; + }else{ + missingOpts += o.stringValue()+", "; + } + } + } + return missingOpts; + } + + +} diff --git a/src/de/superx/sxrest/MaskXml.java b/src/de/superx/sxrest/MaskXml.java index 8ebab72..edfe164 100644 --- a/src/de/superx/sxrest/MaskXml.java +++ b/src/de/superx/sxrest/MaskXml.java @@ -55,7 +55,7 @@ public class MaskXml { @GET @Path("/get/{param}") - @Produces(MediaType.APPLICATION_XML) + @Produces(MediaType.TEXT_PLAIN) public String printXml(@PathParam("param") String tid,@Context HttpServletRequest request) { diff --git a/superx/WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam b/superx/WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam new file mode 100644 index 0000000..19b680e --- /dev/null +++ b/superx/WEB-INF/conf/edustore/db/bin/SQL_ENV_etl.sam @@ -0,0 +1,16 @@ +#Beispielumgebung für das ETL-Modul: + +#SUPERX_MODULE=$SUPERX_DIR/db/module +#SUPERX_ROHDATEN=rohdaten + +ETL_PFAD=$SUPERX_MODULE/etl; export ETL_PFAD +ETL_ERRORDAT=$ETL_PFAD/etl_update.err; export ETL_ERRORDAT +#Hier stehen die Rohdaten (im Unterverzeichnis unl) +ETL_LOAD_PFAD=$ETL_PFAD/$SUPERX_ROHDATEN; export ETL_LOAD_PFAD +ETL_ERRORMAIL=$ERRORMAIL; export ETL_ERRORMAIL +ETL_LOGMAIL=$LOGMAIL; export ETL_LOGMAIL + +PATH=$ETL_PFAD/bin:$PATH +export PATH + + diff --git a/superx/WEB-INF/lib/superx-etl.jar b/superx/WEB-INF/lib/superx-etl.jar new file mode 100644 index 0000000..7b9dee4 Binary files /dev/null and b/superx/WEB-INF/lib/superx-etl.jar differ diff --git a/superx/edit/etl/etl_manager.jsp b/superx/edit/etl/etl_manager.jsp index 7686d22..8d53997 100644 --- a/superx/edit/etl/etl_manager.jsp +++ b/superx/edit/etl/etl_manager.jsp @@ -51,13 +51,17 @@ document.getElementById("MaskSubHeader").innerHTML="
-