SuperX
/
etl
7
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity
Browse Source

Aktualisierung der Makros zur einfacheren Installation mit Quelldateien vergl im Wiki Hochschuleigene Laderoutinen

etl_0.5
Meikel Bisping 1 week ago
parent
a5f12f59c9
commit
2b7d2be65d
1 changed files with 153 additions and 86 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 239
      src-modules/module/etl/schluesseltabellen/fm_templates.unl

239
src-modules/module/etl/schluesseltabellen/fm_templates.unl
Unescape View File

@ -1,4 +1,56 @@
216^ETL_MAKROS^<#macro ETL_STEPS_FUELLEN>\ 17^ETL_MAKROS^<#macro ETL_STEPS_FUELLEN>\
<@etl_steps_def_check/>\
\
<#-- Hinterlegen von Defaultwerten für step_properties -->\
<#assign new_etl_step_properties = etl_step_properties>\
<#foreach step in etl_steps>\
<#-- gültig für alle Steps -->\
<#if etl_step_lookup_systeminfo(step.etl_job)==-1><#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der liefert aber keine systeminfo_id"> </#if>\
<@check_etl_properties step_uniquename=step.uniquename propname="systemInfoId" defaultvalue=etl_step_lookup_systeminfo(step.etl_job)/>\
<@check_etl_properties step_uniquename=step.uniquename propname="active" defaultvalue="true"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="custom" defaultvalue="true"/>\
\
<#switch step.type>\
<#case "UNLOAD">\
<@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>\
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarSource" defaultvalue="eduetl"/>\
<#break>\
<#case "LOAD">\
<@check_etl_properties step_uniquename=step.uniquename propname="database" defaultvalue="eduetl"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="delimiter" defaultvalue="\^"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="header" defaultvalue="false"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="refresh" defaultvalue="true"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="usePostgresCopyApi" defaultvalue="true"/>\
<#break>\
<#case "DOQUERY">\
<@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>\
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>\
<#break>\
<#case "DOSQL">\
<@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>\
<@check_etl_properties step_uniquename=step.uniquename propname="limitDebugStmts" defaultvalue="-1"/>\
<#break>\
<#default>\
\
</#switch>\
</#foreach>\
<#assign etl_step_properties = new_etl_step_properties>\
\
\
--Bezeichnungen aktualisieren\
<#foreach etl_job in etl_jobs>\
update etl_job set caption='${etl_job.name}' where uniquename='${etl_job.uniquename}'; -- ggfs noch systeminfo_id?\
</#foreach>\
<#foreach etl_step in etl_steps>\
update etl_step set caption='${etl_step.name}' where uniquename='${etl_step.uniquename}' and tid in\
(select step_id from etl_step_relation where job_id in (select tid from etl_job where uniquename='${etl_step.etl_job}'));\
</#foreach>\
\
\
\ \
\ \
create temp table tmp_etl_step(\ create temp table tmp_etl_step(\
@ -63,6 +115,34 @@ custom_step SMALLINT default 1\
) \ ) \
;\ ;\
\ \
--jobs with custom_job=0 get deleted in HISinOne-BI\
--therefore set to 2:\
update etl_job set custom_job=2\
where custom_job=0\
and uniquename in ('sx_select_mask','sx_insert_mask',\
'kern_konstanten_update',\
'sichten_unload',\
'sos_gewichtung_unload');\
\
\
update etl_step set custom_step=2\
where custom_step=0\
and tid in (select R.step_id from etl_step_relation R, etl_job J\
where R.job_id=J.tid\
and J.uniquename in ('sx_select_mask','sx_insert_mask',\
'kern_konstanten_update',\
'sichten_unload',\
'sos_gewichtung_unload'));\
\
update etl_step_relation set custom_step=2\
where custom_step=0\
and job_id in (select J.tid\
from etl_job J\
where J.uniquename in ('sx_select_mask','sx_insert_mask',\
'kern_konstanten_update',\
'sichten_unload',\
'sos_gewichtung_unload'));\
\
<#foreach etl_job in etl_jobs>\ <#foreach etl_job in etl_jobs>\
\ \
truncate table tmp_etl_job;\ truncate table tmp_etl_job;\
@ -81,10 +161,9 @@ insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists,
values ('${etl_job.uniquename}',\ values ('${etl_job.uniquename}',\
'${etl_job.name}',\ '${etl_job.name}',\
${etl_job.systeminfo_id},\ ${etl_job.systeminfo_id},\
'${etl_job.logfile}',0,0);\ '${etl_job.logfile}',0,2);\
\
\ \
<#if SQLdialect='Postgres'>\
--Postgres Dialekt:\
\ \
update tmp_etl_job set tid=J.tid,\ update tmp_etl_job set tid=J.tid,\
already_exists=1,\ already_exists=1,\
@ -95,33 +174,6 @@ from etl_job J where J.uniquename=tmp_etl_job.uniquename\
and J.systeminfo_id=tmp_etl_job.systeminfo_id\ and J.systeminfo_id=tmp_etl_job.systeminfo_id\
;\ ;\
\ \
<#else>\
\
--Informix Dialekt:\
update tmp_etl_job set (tid,\
already_exists,\
caption,\
logfile,\
custom_job) \
= ((select \
tid,\
1 as already_exists,\
caption,\
logfile,\
custom_job\
from etl_job J where J.uniquename=tmp_etl_job.uniquename\
and J.systeminfo_id=tmp_etl_job.systeminfo_id))\
where 0 <(select count(*)\
from etl_job J where J.uniquename=tmp_etl_job.uniquename\
and J.systeminfo_id=tmp_etl_job.systeminfo_id)\
\
;\
\
</#if>\
\
\
\
--TODO Informix\
\ \
--neue jobs:\ --neue jobs:\
insert into etl_job(uniquename,caption,systeminfo_id,custom_job)\ insert into etl_job(uniquename,caption,systeminfo_id,custom_job)\
@ -194,7 +246,7 @@ J.tid as etl_job_id,\
</#if>\ </#if>\
J.uniquename,\ J.uniquename,\
'${etl_job.logfile}' as logfile,\ '${etl_job.logfile}' as logfile,\
0,\ 2,\
0\ 0\
from etl_job J\ from etl_job J\
where J.uniquename='${etl_job.uniquename}'\ where J.uniquename='${etl_job.uniquename}'\
@ -236,10 +288,12 @@ FROM tmp_etl_job_param \
update tmp_etl_step set step_type=(select T.tid from etl_step_type T\ update tmp_etl_step set step_type=(select T.tid from etl_step_type T\
where T.uniquename=tmp_etl_step.step_type_uniquename);\ where T.uniquename=tmp_etl_step.step_type_uniquename);\
\ \
--select * from tmp_etl_step\
--where step_type is null;--_uniquename from tmp_etl_step;\
\
\ \
--vorhandene Steps erkennen:\ --vorhandene Steps erkennen:\
<#if SQLdialect='Postgres'>\ \
--Postgres Dialekt:\
\ \
update tmp_etl_step set tid=S.tid,\ update tmp_etl_step set tid=S.tid,\
already_exists=1,\ already_exists=1,\
@ -249,31 +303,6 @@ custom_step=S.custom_step\
from etl_step S where S.uniquename=tmp_etl_step.uniquename\ from etl_step S where S.uniquename=tmp_etl_step.uniquename\
and S.systeminfo_id=tmp_etl_step.systeminfo_id\ and S.systeminfo_id=tmp_etl_step.systeminfo_id\
;\ ;\
<#else>\
\
--Informix Dialekt:\
update tmp_etl_step set (tid,\
already_exists,\
caption,\
logfile,\
custom_step) \
= ((select \
tid,\
1 as already_exists,\
caption,\
logfile,\
custom_step\
from etl_step S where S.uniquename=tmp_etl_step.uniquename\
and S.systeminfo_id=tmp_etl_step.systeminfo_id))\
where 0 <(select count(*)\
from etl_step S where S.uniquename=tmp_etl_step.uniquename\
and S.systeminfo_id=tmp_etl_step.systeminfo_id)\
;\
\
\
\
</#if>\
\
\ \
--neue Steps einfügen:\ --neue Steps einfügen:\
insert into etl_step(\ insert into etl_step(\
@ -315,7 +344,7 @@ FROM tmp_etl_step T )\
delete from etl_step_relation\ delete from etl_step_relation\
where job_id in (select J.tid\ where job_id in (select J.tid\
FROM tmp_etl_job J)\ FROM tmp_etl_job J)\
and custom_step=0\ and custom_step=2\
;\ ;\
\ \
\ \
@ -362,7 +391,7 @@ select \
0 as force_continue,\ 0 as force_continue,\
1 as step_active,\ 1 as step_active,\
sortnr,\ sortnr,\
0\ 2\
FROM tmp_etl_step \ FROM tmp_etl_step \
;\ ;\
\ \
@ -394,7 +423,6 @@ drop table tmp_etl_job_param;\
drop table tmp_etl_step_property;\ drop table tmp_etl_step_property;\
drop table tmp_etl_step_relation;\ drop table tmp_etl_step_relation;\
\ \
<#if testfaelle?exists>\
\ \
<#foreach testfall in testfaelle>\ <#foreach testfall in testfaelle>\
select 'testfall ${testfall.testcase}: ERFOLG'\ select 'testfall ${testfall.testcase}: ERFOLG'\
@ -408,30 +436,69 @@ where ${testfall.assertion}!=(${testfall.sql})\
\ \
\ \
</#foreach> --Ende Testfälle\ </#foreach> --Ende Testfälle\
</#if>\
\
<#if SQLdialect='Informix'> \
--nun xupdates:\
--bei informix kann man TEXT Spalten nicht updaten, daher per jdbc updaten:\
--damit DOSQL nicht den xupdater beim Start dieses Scriptes anwirft,\
--wird das in eine temp. Datei ausgelagert.\
! echo "<xup""date>" > "./tmp"$MANDANTID".sql"\
<#foreach etl_step_property in etl_step_properties>\
\
! echo '<text table="etl_step_property" field="prop_value" where="prop_name='\\''${etl_step_property.prop_name}'\\'' and etl_step_id=(select S.tid from etl_step S where S.uniquename='\\''${etl_step_property.etl_step}'\\'')">${etl_step_property.prop_value}</text>' >>"./tmp"$MANDANTID".sql"\
\
\
</#foreach>\
! echo "</xup""date>" >> "./tmp"$MANDANTID".sql"\
! DOSQL "./tmp"$MANDANTID".sql"\
\ \
<#else>\
--Postgres:\
<#foreach etl_step_property in etl_step_properties>\ <#foreach etl_step_property in etl_step_properties>\
\ update etl_step_property set prop_value='${etl_step_property.prop_value}' where prop_name='${etl_step_property.prop_name}' and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step}');\
update etl_step_property set prop_value='${etl_step_property.prop_value?replace("'", "''")}' where prop_name='${etl_step_property.prop_name}' and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step?replace("'", "''")}');\
\
</#foreach>\ </#foreach>\
</#if>\ </#macro>\
\ \
</#macro>^Datenbankunabhängigkeit^^1^ \
<#-- Makro zum Prüfen der Definition auf mögliche Fehler -->\
<#macro etl_steps_def_check>\
<#foreach step in etl_steps>\
<#-- checking job --> \
<#assign found=false/>\
<#foreach job in etl_jobs>\
<#if job.uniquename=step.etl_job> <#assign found=true> </#if>\
</#foreach>\
<#if found=false> <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der ist aber nicht definiert!"> </#if>\
<#-- checking step type-->\
<#switch step.type>\
<#case "LOAD">\
<#case "UNLOAD">\
<#case "DOSQL">\
<#case "DOQUERY">\
<#case "MSG"><#-- MSG wird von HisInOne noch nicht unterstuetzt-->\
<#break/>\
<#default>\
<#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' hat unbekannten Typ '"+step.type+"'"> \
</#switch>\
</#foreach>\
\
<#-- checking step properties existing step-->\
<#foreach step_prop in etl_step_properties>\
\
<#assign found=false/>\
<#foreach step in etl_steps>\
<#if step.uniquename=step_prop.etl_step> <#assign found=true> </#if>\
</#foreach>\
<#if found=false> <#stop "DEFINITIONSFEHLER: Eine Etl_step_property '"+step_prop.prop_name+ "' verweist auf etl_step '"+step_prop.etl_step+"' der ist aber nicht definiert!"> </#if>\
</#foreach>\
</#macro>\
\
<#function etl_step_lookup_systeminfo job_uniquename>\
<#assign val=-1/>\
<#foreach job in etl_jobs>\
<#if job.uniquename=job_uniquename> <#assign val=job.systeminfo_id/></#if>\
</#foreach>\
<#return val/>\
</#function>\
\
<#macro check_etl_properties step_uniquename propname defaultvalue>\
<#if !etl_properties_contains(step_uniquename,propname)>\
<#assign new_etl_step_properties = new_etl_step_properties + [{\
"etl_step": step_uniquename,\
"prop_name": propname,\
"prop_value": defaultvalue\
}] />\
</#if>\
</#macro>\
\
\
<#function etl_properties_contains step propname>\
<#assign val=false/>\
<#foreach prop in etl_step_properties>\
<#if prop.etl_step==step&&prop.prop_name==propname> <#assign val=true/> </#if>\
</#foreach>\
<#return val>\
</#function>^Makros des ETL-Moduls^^1^

Write Preview
Loading…
Cancel
Save