BI_Maintenance/sonderladeroutine/kern_sonderladeroutine_demo.sql

--Freemarker Template

-- Demo einer SuperX-Sonderladeroutine.
--
-- Ablauf:
--   1. Einen kleinen Beispieldatensatz aus eduetl in eine UNL-Datei entladen.
--   2. Eine einfache Zieltabelle in eduetl anlegen.
--   3. Die UNL-Datei in diese Tabelle laden.
--
-- Nach dem Lauf kann das Ergebnis geprüft werden mit:
--   select * from tmp_kern_special_demo;
--
-- Beim erneuten Lauf leert der LOAD-Schritt die Tabelle wegen refresh=true.

<#assign etl_jobs = [
{"uniquename":"kern_special_demo_special","name":"Demo Sonderladeroutine","systeminfo_id":9,"logfile":""}
] />

<#assign etl_job_params = [
] />

<#assign etl_steps = [
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_unload","name":"Demo-Daten entladen","type":"UNLOAD","parent":""},
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_create","name":"Demo-Zieltabelle anlegen","type":"DOQUERY","parent":""},
{"etl_job":"kern_special_demo_special","uniquename":"kern_special_demo_load","name":"Demo-Daten laden","type":"LOAD","parent":""}
] />

<#assign etl_step_properties = [

{"etl_step":"kern_special_demo_unload","prop_name":"datasource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_unload","prop_name":"sqlVarSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_unload","prop_name":"sql","prop_value":"select 1 as id, ''Demo'' as name"},
{"etl_step":"kern_special_demo_unload","prop_name":"unlFile","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"},

{"etl_step":"kern_special_demo_create","prop_name":"dataSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_create","prop_name":"sqlVarDataSource","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_create","prop_name":"sql","prop_value":"create table if not exists tmp_kern_special_demo (id integer, name varchar(20))"},

{"etl_step":"kern_special_demo_load","prop_name":"database","prop_value":"eduetl"},
{"etl_step":"kern_special_demo_load","prop_name":"unlFilePath","prop_value":"$KERN_LOAD_PFAD/unl/kern_special_demo.unl"},
{"etl_step":"kern_special_demo_load","prop_name":"tableName","prop_value":"tmp_kern_special_demo"},
{"etl_step":"kern_special_demo_load","prop_name":"delimiter","prop_value":"^"},
{"etl_step":"kern_special_demo_load","prop_name":"header","prop_value":"false"},
{"etl_step":"kern_special_demo_load","prop_name":"refresh","prop_value":"true"},
{"etl_step":"kern_special_demo_load","prop_name":"usePostgresCopyApi","prop_value":"true"}

] />

<#assign testfaelle = [
] />


<#macro etl_steps_def_check>
  <#foreach step in etl_steps>
    <#assign found=false/>
    <#foreach job in etl_jobs>
      <#if job.uniquename == step.etl_job>
        <#assign found=true/>
      </#if>
    </#foreach>
    <#if !found>
      <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der ist aber nicht definiert!">
    </#if>

    <#switch step.type>
      <#case "LOAD">
      <#case "UNLOAD">
      <#case "DOSQL">
      <#case "DOQUERY">
      <#case "MSG">
        <#break/>
      <#default>
        <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' hat unbekannten Typ '"+step.type+"'">
    </#switch>
  </#foreach>

  <#foreach step_prop in etl_step_properties>
    <#assign found=false/>
    <#foreach step in etl_steps>
      <#if step.uniquename == step_prop.etl_step>
        <#assign found=true/>
      </#if>
    </#foreach>
    <#if !found>
      <#stop "DEFINITIONSFEHLER: Eine Etl_step_property '"+step_prop.prop_name+ "' verweist auf etl_step '"+step_prop.etl_step+"' der ist aber nicht definiert!">
    </#if>
  </#foreach>
</#macro>

<#function etl_step_lookup_systeminfo job_uniquename>
  <#assign val=-1/>
  <#foreach job in etl_jobs>
    <#if job.uniquename == job_uniquename>
      <#assign val=job.systeminfo_id/>
    </#if>
  </#foreach>
  <#return val/>
</#function>

<#function etl_properties_contains step propname>
  <#assign val=false/>
  <#foreach prop in etl_step_properties>
    <#if prop.etl_step == step && prop.prop_name == propname>
      <#assign val=true/>
    </#if>
  </#foreach>
  <#return val/>
</#function>

<#macro check_etl_properties step_uniquename propname defaultvalue>
  <#if !etl_properties_contains(step_uniquename,propname)>
    <#assign new_etl_step_properties = new_etl_step_properties + [{
      "etl_step": step_uniquename,
      "prop_name": propname,
      "prop_value": defaultvalue
    }] />
  </#if>
</#macro>

<#macro ETL_STEPS_FUELLEN>
  <@etl_steps_def_check/>

  <#assign new_etl_step_properties = etl_step_properties/>
  <#foreach step in etl_steps>
    <#if etl_step_lookup_systeminfo(step.etl_job) == -1>
      <#stop "DEFINITIONSFEHLER: Etl_step '"+step.uniquename+ "' verweist auf job '"+step.etl_job+"' der liefert aber keine systeminfo_id">
    </#if>

    <@check_etl_properties step_uniquename=step.uniquename propname="systemInfoId" defaultvalue=etl_step_lookup_systeminfo(step.etl_job)/>
    <@check_etl_properties step_uniquename=step.uniquename propname="active" defaultvalue="true"/>
    <@check_etl_properties step_uniquename=step.uniquename propname="custom" defaultvalue="true"/>

    <#switch step.type>
      <#case "UNLOAD">
        <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>
        <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarSource" defaultvalue="eduetl"/>
        <#break>
      <#case "LOAD">
        <@check_etl_properties step_uniquename=step.uniquename propname="database" defaultvalue="eduetl"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="delimiter" defaultvalue="^"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="header" defaultvalue="false"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="refresh" defaultvalue="true"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="usePostgresCopyApi" defaultvalue="true"/>
        <#break>
      <#case "DOQUERY">
        <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="sourceVersions" defaultvalue=""/>
        <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>
        <#break>
      <#case "DOSQL">
        <@check_etl_properties step_uniquename=step.uniquename propname="databaseSystem" defaultvalue="postgres"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="dataSource" defaultvalue="eduetl"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="sqlVarDataSource" defaultvalue="eduetl"/>
        <@check_etl_properties step_uniquename=step.uniquename propname="limitDebugStmts" defaultvalue="-1"/>
        <#break>
      <#default>
    </#switch>
  </#foreach>

  <#assign etl_step_properties = new_etl_step_properties/>

--Bezeichnungen aktualisieren
<#foreach etl_job in etl_jobs>
update etl_job set caption='${etl_job.name}' where uniquename='${etl_job.uniquename}';
</#foreach>
<#foreach etl_step in etl_steps>
update etl_step set caption='${etl_step.name}' where uniquename='${etl_step.uniquename}' and tid in
(select step_id from etl_step_relation where job_id in (select tid from etl_job where uniquename='${etl_step.etl_job}'));
</#foreach>

create temp table tmp_etl_step(
tid INTEGER,
uniquename VARCHAR(255),
caption VARCHAR(255),
systeminfo_id INTEGER not null,
step_type INTEGER,
step_type_uniquename VARCHAR(255),
sortnr SMALLINT not null,
force_continue SMALLINT,
etl_job_id INTEGER,
parent_step_id INTEGER,
parent_step_uniquename varchar(255),
parent_job_uniquename varchar(255),
logfile varchar(255),
custom_step smallint,
already_exists smallint
);

create temp table tmp_etl_job(
tid INTEGER,
uniquename VARCHAR(255),
caption VARCHAR(255),
systeminfo_id INTEGER not null,
logfile varchar(255),
already_exists smallint,
custom_job smallint
);

create temp table tmp_etl_job_param(
tid SERIAL not null,
etl_job_id INTEGER,
uniquename VARCHAR(255) not null,
name VARCHAR(255),
param_default VARCHAR(255)
);

create temp table tmp_etl_step_property(
tid SERIAL not null,
etl_step_id INTEGER not null,
prop_name VARCHAR(255),
prop_value text
);

create temp table tmp_etl_step_relation(
tid SERIAL not null,
step_id INTEGER not null,
parent_step_id INTEGER,
job_id INTEGER not null,
force_continue SMALLINT default 1,
step_active SMALLINT default 1,
sortnr SMALLINT default 1,
custom_step SMALLINT default 1
);

update etl_job set custom_job=2
where custom_job=0
and uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload');

update etl_step set custom_step=2
where custom_step=0
and tid in (select R.step_id from etl_step_relation R, etl_job J
where R.job_id=J.tid
and J.uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload'));

update etl_step_relation set custom_step=2
where custom_step=0
and job_id in (select J.tid
from etl_job J
where J.uniquename in ('sx_select_mask','sx_insert_mask',
'kern_konstanten_update',
'sichten_unload',
'sos_gewichtung_unload'));

<#foreach etl_job in etl_jobs>

truncate table tmp_etl_job;
truncate table tmp_etl_step;
truncate table tmp_etl_job_param;
truncate table tmp_etl_step_property;
truncate table tmp_etl_step_relation;

insert into tmp_etl_job(uniquename,caption,systeminfo_id,logfile,already_exists,custom_job)
values ('${etl_job.uniquename}',
'${etl_job.name}',
${etl_job.systeminfo_id},
'${etl_job.logfile}',0,2);

update tmp_etl_job set tid=J.tid,
already_exists=1,
caption=J.caption,
logfile=J.logfile,
custom_job=J.custom_job
from etl_job J
where J.uniquename=tmp_etl_job.uniquename
and J.systeminfo_id=tmp_etl_job.systeminfo_id;

insert into etl_job(uniquename,caption,systeminfo_id,custom_job)
select uniquename,caption,systeminfo_id,custom_job
from tmp_etl_job
where already_exists=0;

update tmp_etl_job set tid=(select J.tid
from etl_job J
where J.uniquename=tmp_etl_job.uniquename
and J.systeminfo_id=tmp_etl_job.systeminfo_id)
where already_exists=0;

<#foreach etl_job_param in etl_job_params>
  <#if etl_job_param.etl_job == etl_job.uniquename>
insert into tmp_etl_job_param(
etl_job_id,
uniquename,
name,
param_default)
select J.tid,
'${etl_job_param.param_name}',
'${etl_job_param.name}',
'${etl_job_param.param_default}'
from tmp_etl_job J;
  </#if>
</#foreach>

<#assign sortnr=0/>
<#foreach etl_step in etl_steps>
  <#if etl_step.etl_job == etl_job.uniquename>
    <#assign sortnr=sortnr+1/>

insert into tmp_etl_step(
uniquename,
caption,
systeminfo_id,
step_type_uniquename,
sortnr,
force_continue,
etl_job_id,
parent_step_uniquename,
parent_job_uniquename,
logfile,
custom_step,
already_exists
)
select '${etl_step.uniquename}',
'${etl_step.name}',
${etl_job.systeminfo_id},
'${etl_step.type}' as step_type_uniquename,
${sortnr}*10 as sortnr,
0 as force_continue,
J.tid as etl_job_id,
<#if etl_step.parent?exists && etl_step.parent != "">
'${etl_step.parent}',
<#else>
'' as parent_step_uniquename,
</#if>
J.uniquename,
'${etl_job.logfile}' as logfile,
2,
0
from etl_job J
where J.uniquename='${etl_job.uniquename}'
and J.systeminfo_id=${etl_job.systeminfo_id};

  </#if>
</#foreach>

delete from etl_job_param
where etl_job_id in (
select distinct etl_job_id
from tmp_etl_job_param
);

insert into etl_job_param
(
etl_job_id,
uniquename,
name,
param_default
)
select
etl_job_id,
uniquename,
name,
param_default
from tmp_etl_job_param;

update tmp_etl_step set step_type=(select T.tid from etl_step_type T
where T.uniquename=tmp_etl_step.step_type_uniquename);

update tmp_etl_step set tid=S.tid,
already_exists=1,
caption=S.caption,
logfile=S.logfile,
custom_step=S.custom_step
from etl_step S
where S.uniquename=tmp_etl_step.uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id;

insert into etl_step(
uniquename,
caption,
systeminfo_id,
step_type_id,
logfile,
custom_step)
select
uniquename,
caption,
systeminfo_id,
step_type,
logfile,
custom_step
from tmp_etl_step
where already_exists=0;

update tmp_etl_step set tid=(select S.tid
from etl_step S
where S.uniquename=tmp_etl_step.uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id)
where already_exists=0;

update tmp_etl_step set parent_step_id=(select S.tid
from etl_step S
where S.uniquename=tmp_etl_step.parent_step_uniquename
and S.systeminfo_id=tmp_etl_step.systeminfo_id);

delete from etl_step_property
where etl_step_id in (select T.tid from tmp_etl_step T);

delete from etl_step_relation
where job_id in (select J.tid from tmp_etl_job J)
and custom_step=2;

<#foreach etl_step_property in etl_step_properties>
insert into tmp_etl_step_property(
etl_step_id,
prop_name)
select
T.tid as etl_step_id,
'${etl_step_property.prop_name}'
from tmp_etl_step T
where uniquename='${etl_step_property.etl_step}';
</#foreach>

insert into etl_step_property(
etl_step_id,
prop_name)
select
T.etl_step_id,
T.prop_name
from tmp_etl_step_property T;

insert into tmp_etl_step_relation(
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
)
select
tid,
parent_step_id,
etl_job_id,
0 as force_continue,
1 as step_active,
sortnr,
2
from tmp_etl_step;

insert into etl_step_relation(
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
)
select
step_id,
parent_step_id,
job_id,
force_continue,
step_active,
sortnr,
custom_step
from tmp_etl_step_relation;

</#foreach>

drop table tmp_etl_step;
drop table tmp_etl_job;
drop table tmp_etl_job_param;
drop table tmp_etl_step_property;
drop table tmp_etl_step_relation;

<#foreach testfall in testfaelle>
select 'testfall ${testfall.testcase}: ERFOLG'
from xdummy
where ${testfall.assertion}=(${testfall.sql});
select 'testfall ${testfall.testcase}: FEHLER bei ${testfall.testcase}'
from xdummy
where ${testfall.assertion}!=(${testfall.sql});
</#foreach>

<#foreach etl_step_property in etl_step_properties>
update etl_step_property
set prop_value='${etl_step_property.prop_value}'
where prop_name='${etl_step_property.prop_name}'
and etl_step_id=(select S.tid from etl_step S where S.uniquename='${etl_step_property.etl_step}');
</#foreach>

</#macro>

<@ETL_STEPS_FUELLEN/>