diff --git a/src-modules/module/etl/schluesseltabellen/fm_templates.unl b/src-modules/module/etl/schluesseltabellen/fm_templates.unl index 205bf7d..d19a859 100644 --- a/src-modules/module/etl/schluesseltabellen/fm_templates.unl +++ b/src-modules/module/etl/schluesseltabellen/fm_templates.unl @@ -1,4 +1,56 @@ -216^ETL_MAKROS^<#macro ETL_STEPS_FUELLEN>\ +17^ETL_MAKROS^<#macro ETL_STEPS_FUELLEN>\ + <@etl_steps_def_check/>\ + \ + <#-- Hinterlegen von Defaultwerten für step_properties -->\ +<#assign new_etl_step_properties = etl_step_properties>\ +<#foreach step in etl_steps>\ + <#-- gültig für alle Steps -->\ + <#if etl_step_lookup_systeminfo(step.etl_job)==-1><#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der liefert aber keine systeminfo_id"> \ + <@check_etl_properties step_uniquename=step.uniquename propname="systemInfoId" defaultvalue=etl_step_lookup_systeminfo(step.etl_job)/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="active" defaultvalue="true"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="custom" defaultvalue="true"/>\ + \ + <#switch step.type>\ + <#case "UNLOAD">\ + <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarSource" defaultvalue="eduetl"/>\ + <#break>\ + <#case "LOAD">\ + <@check_etl_properties step_uniquename=step.uniquename propname="database" defaultvalue="eduetl"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="delimiter" defaultvalue="\^"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="header" defaultvalue="false"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="refresh" defaultvalue="true"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="usePostgresCopyApi" defaultvalue="true"/>\ + <#break>\ + <#case "DOQUERY">\ + <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>\ + <#break>\ + <#case "DOSQL">\ + <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>\ + <@check_etl_properties step_uniquename=step.uniquename propname="limitDebugStmts" defaultvalue="-1"/>\ + <#break>\ + <#default>\ + \ +\ + \ +<#assign etl_step_properties = new_etl_step_properties>\ +\ + \ +--Bezeichnungen aktualisieren\ +<#foreach etl_job in etl_jobs>\ +update etl_job set caption='${etl_job.name}' where uniquename='${etl_job.uniquename}'; -- ggfs noch systeminfo_id?\ +\ +<#foreach etl_step in etl_steps>\ +update etl_step set caption='${etl_step.name}' where uniquename='${etl_step.uniquename}' and tid in\ +(select step_id from etl_step_relation where job_id in (select tid from etl_job where uniquename='${etl_step.etl_job}'));\ +\ +\ +\ \ \ create temp table tmp_etl_step(\ @@ -63,6 +115,34 @@ custom_step SMALLINT default 1\ ) \ ;\ \ +--jobs with custom_job=0 get deleted in HISinOne-BI\ +--therefore set to 2:\ +update etl_job set custom_job=2\ +where custom_job=0\ +and uniquename in ('sx_select_mask','sx_insert_mask',\ +'kern_konstanten_update',\ +'sichten_unload',\ +'sos_gewichtung_unload');\ +\ +\ +update etl_step set custom_step=2\ +where custom_step=0\ +and tid in (select R.step_id from etl_step_relation R, etl_job J\ +where R.job_id=J.tid\ +and J.uniquename in ('sx_select_mask','sx_insert_mask',\ +'kern_konstanten_update',\ +'sichten_unload',\ +'sos_gewichtung_unload'));\ +\ +update etl_step_relation set custom_step=2\ +where custom_step=0\ +and job_id in (select J.tid\ +from etl_job J\ +where J.uniquename in ('sx_select_mask','sx_insert_mask',\ +'kern_konstanten_update',\ +'sichten_unload',\ +'sos_gewichtung_unload'));\ +\ <#foreach etl_job in etl_jobs>\ \ truncate table tmp_etl_job;\ @@ -81,10 +161,9 @@ insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists, values ('${etl_job.uniquename}',\ '${etl_job.name}',\ ${etl_job.systeminfo_id},\ -'${etl_job.logfile}',0,0);\ +'${etl_job.logfile}',0,2);\ +\ \ -<#if SQLdialect='Postgres'>\ ---Postgres Dialekt:\ \ update tmp_etl_job set tid=J.tid,\ already_exists=1,\ @@ -95,33 +174,6 @@ from etl_job J where J.uniquename=tmp_etl_job.uniquename\ and J.systeminfo_id=tmp_etl_job.systeminfo_id\ ;\ \ -<#else>\ -\ ---Informix Dialekt:\ -update tmp_etl_job set (tid,\ -already_exists,\ -caption,\ -logfile,\ -custom_job) \ - = ((select \ - tid,\ -1 as already_exists,\ -caption,\ -logfile,\ -custom_job\ -from etl_job J where J.uniquename=tmp_etl_job.uniquename\ -and J.systeminfo_id=tmp_etl_job.systeminfo_id))\ -where 0 <(select count(*)\ -from etl_job J where J.uniquename=tmp_etl_job.uniquename\ -and J.systeminfo_id=tmp_etl_job.systeminfo_id)\ -\ -;\ -\ -\ -\ -\ -\ ---TODO Informix\ \ --neue jobs:\ insert into etl_job(uniquename,caption,systeminfo_id,custom_job)\ @@ -194,7 +246,7 @@ J.tid as etl_job_id,\ \ J.uniquename,\ '${etl_job.logfile}' as logfile,\ -0,\ +2,\ 0\ from etl_job J\ where J.uniquename='${etl_job.uniquename}'\ @@ -236,10 +288,12 @@ FROM tmp_etl_job_param \ update tmp_etl_step set step_type=(select T.tid from etl_step_type T\ where T.uniquename=tmp_etl_step.step_type_uniquename);\ \ +--select * from tmp_etl_step\ +--where step_type is null;--_uniquename from tmp_etl_step;\ +\ \ --vorhandene Steps erkennen:\ -<#if SQLdialect='Postgres'>\ ---Postgres Dialekt:\ +\ \ update tmp_etl_step set tid=S.tid,\ already_exists=1,\ @@ -249,31 +303,6 @@ custom_step=S.custom_step\ from etl_step S where S.uniquename=tmp_etl_step.uniquename\ and S.systeminfo_id=tmp_etl_step.systeminfo_id\ ;\ -<#else>\ -\ ---Informix Dialekt:\ -update tmp_etl_step set (tid,\ -already_exists,\ -caption,\ -logfile,\ -custom_step) \ - = ((select \ - tid,\ -1 as already_exists,\ -caption,\ -logfile,\ -custom_step\ -from etl_step S where S.uniquename=tmp_etl_step.uniquename\ -and S.systeminfo_id=tmp_etl_step.systeminfo_id))\ -where 0 <(select count(*)\ -from etl_step S where S.uniquename=tmp_etl_step.uniquename\ -and S.systeminfo_id=tmp_etl_step.systeminfo_id)\ -;\ -\ -\ -\ -\ -\ \ --neue Steps einfügen:\ insert into etl_step(\ @@ -315,7 +344,7 @@ FROM tmp_etl_step T )\ delete from etl_step_relation\ where job_id in (select J.tid\ FROM tmp_etl_job J)\ -and custom_step=0\ +and custom_step=2\ ;\ \ \ @@ -362,7 +391,7 @@ select \ 0 as force_continue,\ 1 as step_active,\ sortnr,\ - 0\ + 2\ FROM tmp_etl_step \ ;\ \ @@ -394,7 +423,6 @@ drop table tmp_etl_job_param;\ drop table tmp_etl_step_property;\ drop table tmp_etl_step_relation;\ \ -<#if testfaelle?exists>\ \ <#foreach testfall in testfaelle>\ select 'testfall ${testfall.testcase}: ERFOLG'\ @@ -408,30 +436,69 @@ where ${testfall.assertion}!=(${testfall.sql})\ \ \ --Ende Testfälle\ -\ -\ -<#if SQLdialect='Informix'> \ ---nun xupdates:\ ---bei informix kann man TEXT Spalten nicht updaten, daher per jdbc updaten:\ ---damit DOSQL nicht den xupdater beim Start dieses Scriptes anwirft,\ ---wird das in eine temp. Datei ausgelagert.\ -! echo "" > "./tmp"$MANDANTID".sql"\ -<#foreach etl_step_property in etl_step_properties>\ -\ -! echo '${etl_step_property.prop_value}' >>"./tmp"$MANDANTID".sql"\ -\ -\ -\ -! echo "" >> "./tmp"$MANDANTID".sql"\ -! DOSQL "./tmp"$MANDANTID".sql"\ \ -<#else>\ ---Postgres:\ <#foreach etl_step_property in etl_step_properties>\ -\ -update etl_step_property set prop_value='${etl_step_property.prop_value?replace("'", "''")}' where prop_name='${etl_step_property.prop_name}' and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step?replace("'", "''")}');\ -\ +update etl_step_property set prop_value='${etl_step_property.prop_value}' where prop_name='${etl_step_property.prop_name}' and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step}');\ \ -\ -\ -^Datenbankunabhängigkeit^^1^ +\ +\ +\ +<#-- Makro zum Prüfen der Definition auf mögliche Fehler -->\ + <#macro etl_steps_def_check>\ + <#foreach step in etl_steps>\ + <#-- checking job --> \ + <#assign found=false/>\ + <#foreach job in etl_jobs>\ + <#if job.uniquename=step.etl_job> <#assign found=true> \ + \ + <#if found=false> <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der ist aber nicht definiert!"> \ + <#-- checking step type-->\ + <#switch step.type>\ + <#case "LOAD">\ + <#case "UNLOAD">\ + <#case "DOSQL">\ + <#case "DOQUERY">\ + <#case "MSG"><#-- MSG wird von HisInOne noch nicht unterstuetzt-->\ + <#break/>\ + <#default>\ + <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' hat unbekannten Typ '"+step.type+"'"> \ + \ + \ + \ + <#-- checking step properties existing step-->\ + <#foreach step_prop in etl_step_properties>\ + \ + <#assign found=false/>\ + <#foreach step in etl_steps>\ + <#if step.uniquename=step_prop.etl_step> <#assign found=true> \ + \ + <#if found=false> <#stop "DEFINITIONSFEHLER: Eine Etl_step_property '"+step_prop.prop_name+ "' verweist auf etl_step '"+step_prop.etl_step+"' der ist aber nicht definiert!"> \ + \ + \ +\ + <#function etl_step_lookup_systeminfo job_uniquename>\ + <#assign val=-1/>\ + <#foreach job in etl_jobs>\ + <#if job.uniquename=job_uniquename> <#assign val=job.systeminfo_id/>\ + \ + <#return val/>\ + \ + \ + <#macro check_etl_properties step_uniquename propname defaultvalue>\ + <#if !etl_properties_contains(step_uniquename,propname)>\ + <#assign new_etl_step_properties = new_etl_step_properties + [{\ + "etl_step": step_uniquename,\ + "prop_name": propname,\ + "prop_value": defaultvalue\ + }] />\ + \ + \ + \ + \ + <#function etl_properties_contains step propname>\ + <#assign val=false/>\ + <#foreach prop in etl_step_properties>\ + <#if prop.etl_step==step&&prop.prop_name==propname> <#assign val=true/> \ + \ + <#return val>\ + ^Makros des ETL-Moduls^^1^