SuperX
/
sxc
7
0
Fork 0
Code Issues 6 Pull Requests Projects Releases Wiki Activity
Browse Source

importjob umgebaut fuer Rohdaten - etl metadata injection #53

sxc_0.3_Release
Imo John 7 months ago
parent
51002260a0
commit
3cfff68e83
4 changed files with 1434 additions and 16 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 26
      src-modules/module/sxc/conf/sxc.xml
  2. 101
      src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb
  3. 637
      src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr
  4. 686
      src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr

26
src-modules/module/sxc/conf/sxc.xml
Unescape View File

@ -532,6 +532,32 @@ rightname="CS_BIA_STANDARDREPORTS_ADMIN[SXC]" />
</columns> </columns>
</table> </table>
<table name="sxc_katalog" typ="Schlüsseltabelle" thema="" releaseUnload="demo">
<description>Formeln für Grunddaten und Kennzahlen</description>
<columns>
<column name="tid" type="SERIAL" size ="4" default ="" notnull ="true" description="Tupelidentifier" />
<column name="id" type="CHAR" size="100" default="" notnull="true" description="ID der Kennzahl" />
<column name="name" type="CHAR" size="255" default="" notnull="false" description="Name der Kennzahl" />
</columns>
<primaryKeys>
<rs>
<row>
<fld name='table_cat' />
<fld name='table_schem'>public</fld>
<fld name='table_name'>sxc_katalog</fld>
<fld name='column_name'>tid</fld>
<fld name='key_seq'>1</fld>
<fld name='pk_name'>c_sxc_katalog</fld>
</row>
</rs>
</primaryKeys>
<indexes>
<index name="ix_sxc_katalog" type="unique">
<index-column name="id" />
</index>
</indexes>
</table>
<views> <views>
<view name="sxc_hochschulen_art"> <view name="sxc_hochschulen_art">
<columns> <columns>

101
src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb
Unescape View File

@ -13,7 +13,7 @@
<parameters> <parameters>
<parameter> <parameter>
<name>PATH_TO_UPLOADFILE</name> <name>PATH_TO_UPLOADFILE</name>
<default_value/> <default_value>&#x2f;home&#x2f;superx&#x2f;Downloads&#x2f;stamm all - Auszug UDE-3.csv</default_value>
<description/> <description/>
</parameter> </parameter>
</parameters> </parameters>
@ -323,12 +323,12 @@
<yloc>192</yloc> <yloc>192</yloc>
</entry> </entry>
<entry> <entry>
<name>excel_into_tmp</name> <name>metadata_injection</name>
<description/> <description/>
<type>TRANS</type> <type>TRANS</type>
<specification_method>filename</specification_method> <specification_method>filename</specification_method>
<trans_object_id/> <trans_object_id/>
<filename>&#x24;&#x7b;Internal.Job.Filename.Directory&#x7d;&#x2f;import_bestand_mwk_nrw_hskennz_1.ktr</filename> <filename>&#x24;&#x7b;Internal.Job.Filename.Directory&#x7d;&#x2f;import_bestand_mwk_nrw_hskennz_injection.ktr</filename>
<transname/> <transname/>
<arg_from_previous>N</arg_from_previous> <arg_from_previous>N</arg_from_previous>
<params_from_previous>N</params_from_previous> <params_from_previous>N</params_from_previous>
@ -354,14 +354,14 @@
<parallel>N</parallel> <parallel>N</parallel>
<draw>Y</draw> <draw>Y</draw>
<nr>0</nr> <nr>0</nr>
<xloc>272</xloc> <xloc>688</xloc>
<yloc>320</yloc> <yloc>352</yloc>
</entry> </entry>
<entry> <entry>
<name>create tmp_sxc_zahl_wert</name> <name>create tmp_sxc_zahl_wert</name>
<description/> <description/>
<type>SQL</type> <type>SQL</type>
<sql>drop table if exists tmp_sxc_zahl_wert&#x3b;&#xa;&#xa;CREATE TABLE tmp_sxc_zahl_wert&#xa;&#x28;&#xa; tid serial,&#xa; hs_nr_str varchar&#x28;255&#x29;,&#xa; hs_nr char&#x28;40&#x29;,&#xa; kenn_zahl char&#x28;100&#x29; NOT NULL,&#xa; jahr smallint,&#xa; sem smallint,&#xa; ch110_institut char&#x28;10&#x29;,&#xa; bland integer,&#xa; sxc_bestand_id integer,&#xa; wert_str varchar&#x28;255&#x29;,&#xa; wert decimal&#x28;30,20&#x29;,&#xa; generic_dim1 char&#x28;100&#x29;,&#xa; generic_dim2 char&#x28;100&#x29;&#xa;&#x29;&#x3b;</sql> <sql>drop table if exists tmp_sxc_zahl_wert&#x3b;&#xa;drop table if exists tmp_feld&#x3b;&#xa;drop table if exists tmp_feld_all&#x3b;&#xa;&#xa;&#xa;create table tmp_feld &#x28;Feld varchar&#x28;255&#x29;&#x29;&#x3b;&#xa;create table tmp_feld_all &#x28;Feld varchar&#x28;255&#x29;&#x29;&#x3b;&#xa;&#xa;CREATE TABLE tmp_sxc_zahl_wert&#xa;&#x28;&#xa; tid serial,&#xa; hs_nr_str varchar&#x28;255&#x29;,&#xa; hs_nr char&#x28;40&#x29;,&#xa; kenn_zahl char&#x28;100&#x29; NOT NULL,&#xa; jahr smallint,&#xa; sem smallint,&#xa; ch110_institut char&#x28;10&#x29;,&#xa; bland integer,&#xa; sxc_bestand_id integer,&#xa; wert_str varchar&#x28;255&#x29;,&#xa; wert decimal&#x28;30,20&#x29;,&#xa; generic_dim1 char&#x28;100&#x29;,&#xa; generic_dim2 char&#x28;100&#x29;&#xa;&#x29;&#x3b;</sql>
<useVariableSubstitution>F</useVariableSubstitution> <useVariableSubstitution>F</useVariableSubstitution>
<sqlfromfile>F</sqlfromfile> <sqlfromfile>F</sqlfromfile>
<sqlfilename/> <sqlfilename/>
@ -405,6 +405,57 @@
<xloc>880</xloc> <xloc>880</xloc>
<yloc>272</yloc> <yloc>272</yloc>
</entry> </entry>
<entry>
<name>tmp_feld_write</name>
<description/>
<type>TRANS</type>
<specification_method>filename</specification_method>
<trans_object_id/>
<filename>&#x24;&#x7b;Internal.Job.Filename.Directory&#x7d;&#x2f;tmp_feld_write.ktr</filename>
<transname/>
<arg_from_previous>N</arg_from_previous>
<params_from_previous>N</params_from_previous>
<exec_per_row>N</exec_per_row>
<clear_rows>N</clear_rows>
<clear_files>N</clear_files>
<set_logfile>N</set_logfile>
<logfile/>
<logext/>
<add_date>N</add_date>
<add_time>N</add_time>
<loglevel>Basic</loglevel>
<cluster>N</cluster>
<slave_server_name/>
<set_append_logfile>N</set_append_logfile>
<wait_until_finished>Y</wait_until_finished>
<follow_abort_remote>N</follow_abort_remote>
<create_parent_folder>N</create_parent_folder>
<logging_remote_work>N</logging_remote_work>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>272</xloc>
<yloc>352</yloc>
</entry>
<entry>
<name>delete from tmp_feld</name>
<description/>
<type>SQL</type>
<sql>delete from tmp_feld&#xa;where Feld in &#x28;&#x27;IDLFB&#x27;,&#x27;J&#x27;,&#x27;H&#x27;,&#x27;L&#x27;,&#x27;UHSARTMINGRUPPTEXT&#x27;,&#x27;UHSARTMINTEXT&#x27;,&#x27;ME&#x27;,&#xa; &#x27;UHSTEXTHS&#x27;,&#x27;UHSTEXTNSTO&#x27;,&#x27;ULEFOTEXT&#x27;,&#x27;UFGSCHL&#x27;,&#x27;UFGTEXT&#x27;&#x29;&#x3b;&#xa;</sql>
<useVariableSubstitution>F</useVariableSubstitution>
<sqlfromfile>F</sqlfromfile>
<sqlfilename/>
<sendOneStatement>F</sendOneStatement>
<connection>eduetl</connection>
<parallel>N</parallel>
<draw>Y</draw>
<nr>0</nr>
<xloc>480</xloc>
<yloc>352</yloc>
</entry>
</entries> </entries>
<hops> <hops>
<hop> <hop>
@ -416,15 +467,6 @@
<evaluation>Y</evaluation> <evaluation>Y</evaluation>
<unconditional>Y</unconditional> <unconditional>Y</unconditional>
</hop> </hop>
<hop>
<from>create tmp_sxc_zahl_wert</from>
<to>excel_into_tmp</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop> <hop>
<from>insert into sxc_zahl_wert &#x2b; NRW-Durchschnitt</from> <from>insert into sxc_zahl_wert &#x2b; NRW-Durchschnitt</from>
<to>Success</to> <to>Success</to>
@ -453,7 +495,7 @@
<unconditional>N</unconditional> <unconditional>N</unconditional>
</hop> </hop>
<hop> <hop>
<from>excel_into_tmp</from> <from>metadata_injection</from>
<to>update tmp_sxc_zahl_wert</to> <to>update tmp_sxc_zahl_wert</to>
<from_nr>0</from_nr> <from_nr>0</from_nr>
<to_nr>0</to_nr> <to_nr>0</to_nr>
@ -461,6 +503,33 @@
<evaluation>Y</evaluation> <evaluation>Y</evaluation>
<unconditional>N</unconditional> <unconditional>N</unconditional>
</hop> </hop>
<hop>
<from>create tmp_sxc_zahl_wert</from>
<to>tmp_feld_write</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>tmp_feld_write</from>
<to>delete from tmp_feld</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>delete from tmp_feld</from>
<to>metadata_injection</to>
<from_nr>0</from_nr>
<to_nr>0</to_nr>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops> </hops>
<notepads> <notepads>
</notepads> </notepads>

637
src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr
Unescape View File

@ -0,0 +1,637 @@
<?xml version="1.0" encoding="UTF-8"?>
<transformation>
<info>
<name>import_bestand_mwk_nrw_hskennz_injection</name>
<description/>
<extended_description/>
<trans_version/>
<trans_type>Normal</trans_type>
<trans_status>0</trans_status>
<directory>&#x2f;</directory>
<parameters>
</parameters>
<log>
<trans-log-table>
<connection/>
<schema/>
<table/>
<size_limit_lines/>
<interval/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STATUS</id>
<enabled>Y</enabled>
<name>STATUS</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
<subject/>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
<subject/>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
<subject/>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
<subject/>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
<subject/>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
<subject/>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>STARTDATE</id>
<enabled>Y</enabled>
<name>STARTDATE</name>
</field>
<field>
<id>ENDDATE</id>
<enabled>Y</enabled>
<name>ENDDATE</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>DEPDATE</id>
<enabled>Y</enabled>
<name>DEPDATE</name>
</field>
<field>
<id>REPLAYDATE</id>
<enabled>Y</enabled>
<name>REPLAYDATE</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>Y</enabled>
<name>LOG_FIELD</name>
</field>
<field>
<id>EXECUTING_SERVER</id>
<enabled>N</enabled>
<name>EXECUTING_SERVER</name>
</field>
<field>
<id>EXECUTING_USER</id>
<enabled>N</enabled>
<name>EXECUTING_USER</name>
</field>
<field>
<id>CLIENT</id>
<enabled>N</enabled>
<name>CLIENT</name>
</field>
</trans-log-table>
<perf-log-table>
<connection/>
<schema/>
<table/>
<interval/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>SEQ_NR</id>
<enabled>Y</enabled>
<name>SEQ_NR</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>INPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>INPUT_BUFFER_ROWS</name>
</field>
<field>
<id>OUTPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>OUTPUT_BUFFER_ROWS</name>
</field>
</perf-log-table>
<channel-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>LOGGING_OBJECT_TYPE</id>
<enabled>Y</enabled>
<name>LOGGING_OBJECT_TYPE</name>
</field>
<field>
<id>OBJECT_NAME</id>
<enabled>Y</enabled>
<name>OBJECT_NAME</name>
</field>
<field>
<id>OBJECT_COPY</id>
<enabled>Y</enabled>
<name>OBJECT_COPY</name>
</field>
<field>
<id>REPOSITORY_DIRECTORY</id>
<enabled>Y</enabled>
<name>REPOSITORY_DIRECTORY</name>
</field>
<field>
<id>FILENAME</id>
<enabled>Y</enabled>
<name>FILENAME</name>
</field>
<field>
<id>OBJECT_ID</id>
<enabled>Y</enabled>
<name>OBJECT_ID</name>
</field>
<field>
<id>OBJECT_REVISION</id>
<enabled>Y</enabled>
<name>OBJECT_REVISION</name>
</field>
<field>
<id>PARENT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>PARENT_CHANNEL_ID</name>
</field>
<field>
<id>ROOT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>ROOT_CHANNEL_ID</name>
</field>
</channel-log-table>
<step-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>N</enabled>
<name>LOG_FIELD</name>
</field>
</step-log-table>
<metrics-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>METRICS_DATE</id>
<enabled>Y</enabled>
<name>METRICS_DATE</name>
</field>
<field>
<id>METRICS_CODE</id>
<enabled>Y</enabled>
<name>METRICS_CODE</name>
</field>
<field>
<id>METRICS_DESCRIPTION</id>
<enabled>Y</enabled>
<name>METRICS_DESCRIPTION</name>
</field>
<field>
<id>METRICS_SUBJECT</id>
<enabled>Y</enabled>
<name>METRICS_SUBJECT</name>
</field>
<field>
<id>METRICS_TYPE</id>
<enabled>Y</enabled>
<name>METRICS_TYPE</name>
</field>
<field>
<id>METRICS_VALUE</id>
<enabled>Y</enabled>
<name>METRICS_VALUE</name>
</field>
</metrics-log-table>
</log>
<maxdate>
<connection/>
<table/>
<field/>
<offset>0.0</offset>
<maxdiff>0.0</maxdiff>
</maxdate>
<size_rowset>10000</size_rowset>
<sleep_time_empty>50</sleep_time_empty>
<sleep_time_full>50</sleep_time_full>
<unique_connections>N</unique_connections>
<feedback_shown>Y</feedback_shown>
<feedback_size>50000</feedback_size>
<using_thread_priorities>Y</using_thread_priorities>
<shared_objects_file/>
<capture_step_performance>N</capture_step_performance>
<step_performance_capturing_delay>1000</step_performance_capturing_delay>
<step_performance_capturing_size_limit>100</step_performance_capturing_size_limit>
<dependencies>
</dependencies>
<partitionschemas>
</partitionschemas>
<slaveservers>
</slaveservers>
<clusterschemas>
</clusterschemas>
<created_user>-</created_user>
<created_date>2023&#x2f;03&#x2f;01 20&#x3a;02&#x3a;48.054</created_date>
<modified_user>-</modified_user>
<modified_date>2023&#x2f;03&#x2f;01 20&#x3a;02&#x3a;48.054</modified_date>
<key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA&#x3d;</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Data Grid</from>
<to>ETL Metadata Injection</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>tmp_feld</from>
<to>ETL Metadata Injection</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>tmp_feld_all</from>
<to>ETL Metadata Injection</to>
<enabled>Y</enabled>
</hop>
</order>
<step>
<name>Data Grid</name>
<type>DataGrid</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<name>type_inputFields</name>
<type/>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
</field>
<field>
<name>trans_new_field</name>
<type/>
<format/>
<currency/>
<decimal/>
<group/>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
</field>
</fields>
<data>
<line> <item>String</item><item>wert</item> </line>
</data>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>320</xloc>
<yloc>128</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>ETL Metadata Injection</name>
<type>MetaInject</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<specification_method>filename</specification_method>
<trans_object_id/>
<trans_name/>
<filename>&#x24;&#x7b;Internal.Transformation.Filename.Directory&#x7d;&#x2f;import_bestand_mwk_nrw_hskennz_template.ktr</filename>
<directory_path/>
<source_step/>
<source_output_fields> </source_output_fields> <target_file/>
<no_execution>N</no_execution>
<stream_source_step/>
<stream_target_step/>
<mappings> <mapping> <target_step_name>Row Normaliser</target_step_name>
<target_attribute_key>VALUE</target_attribute_key>
<target_detail>Y</target_detail>
<source_step>tmp_feld</source_step>
<source_field>feld</source_field>
</mapping> <mapping> <target_step_name>Row Normaliser</target_step_name>
<target_attribute_key>NORMALISED</target_attribute_key>
<target_detail>Y</target_detail>
<source_step>Data Grid</source_step>
<source_field>trans_new_field</source_field>
</mapping> <mapping> <target_step_name>Row Normaliser</target_step_name>
<target_attribute_key>NAME</target_attribute_key>
<target_detail>Y</target_detail>
<source_step>tmp_feld</source_step>
<source_field>feld</source_field>
</mapping> <mapping> <target_step_name>Text file input</target_step_name>
<target_attribute_key>FIELD_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
<source_step>Data Grid</source_step>
<source_field>type_inputFields</source_field>
</mapping> <mapping> <target_step_name>Text file input</target_step_name>
<target_attribute_key>FIELD_NAME</target_attribute_key>
<target_detail>Y</target_detail>
<source_step>tmp_feld_all</source_step>
<source_field>feld</source_field>
</mapping> </mappings> <cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>848</xloc>
<yloc>208</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>tmp_feld</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>eduetl</connection>
<sql>SELECT Feld FROM tmp_feld&#x3b;</sql>
<limit>0</limit>
<lookup/>
<execute_each_row>N</execute_each_row>
<variables_active>N</variables_active>
<lazy_conversion_active>N</lazy_conversion_active>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>320</xloc>
<yloc>240</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>tmp_feld_all</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>eduetl</connection>
<sql>SELECT Feld FROM tmp_feld_all&#x3b;</sql>
<limit>0</limit>
<lookup/>
<execute_each_row>N</execute_each_row>
<variables_active>N</variables_active>
<lazy_conversion_active>N</lazy_conversion_active>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>320</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step_error_handling>
</step_error_handling>
<slave-step-copy-partition-distribution>
</slave-step-copy-partition-distribution>
<slave_transformation>N</slave_transformation>
</transformation>

686
src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr
Unescape View File

@ -0,0 +1,686 @@
<?xml version="1.0" encoding="UTF-8"?>
<transformation>
<info>
<name>import_bestand_mwk_nrw_hskennz_template</name>
<description/>
<extended_description/>
<trans_version/>
<trans_type>Normal</trans_type>
<trans_status>0</trans_status>
<directory>&#x2f;</directory>
<parameters>
<parameter>
<name>PATH_TO_UPLOADFILE</name>
<default_value>&#x2f;home&#x2f;superx&#x2f;Downloads&#x2f;stamm all - Auszug UDE-3.csv</default_value>
<description/>
</parameter>
</parameters>
<log>
<trans-log-table>
<connection/>
<schema/>
<table/>
<size_limit_lines/>
<interval/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STATUS</id>
<enabled>Y</enabled>
<name>STATUS</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
<subject/>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
<subject/>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
<subject/>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
<subject/>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
<subject/>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
<subject/>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>STARTDATE</id>
<enabled>Y</enabled>
<name>STARTDATE</name>
</field>
<field>
<id>ENDDATE</id>
<enabled>Y</enabled>
<name>ENDDATE</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>DEPDATE</id>
<enabled>Y</enabled>
<name>DEPDATE</name>
</field>
<field>
<id>REPLAYDATE</id>
<enabled>Y</enabled>
<name>REPLAYDATE</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>Y</enabled>
<name>LOG_FIELD</name>
</field>
<field>
<id>EXECUTING_SERVER</id>
<enabled>N</enabled>
<name>EXECUTING_SERVER</name>
</field>
<field>
<id>EXECUTING_USER</id>
<enabled>N</enabled>
<name>EXECUTING_USER</name>
</field>
<field>
<id>CLIENT</id>
<enabled>N</enabled>
<name>CLIENT</name>
</field>
</trans-log-table>
<perf-log-table>
<connection/>
<schema/>
<table/>
<interval/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>SEQ_NR</id>
<enabled>Y</enabled>
<name>SEQ_NR</name>
</field>
<field>
<id>LOGDATE</id>
<enabled>Y</enabled>
<name>LOGDATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>INPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>INPUT_BUFFER_ROWS</name>
</field>
<field>
<id>OUTPUT_BUFFER_ROWS</id>
<enabled>Y</enabled>
<name>OUTPUT_BUFFER_ROWS</name>
</field>
</perf-log-table>
<channel-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>LOGGING_OBJECT_TYPE</id>
<enabled>Y</enabled>
<name>LOGGING_OBJECT_TYPE</name>
</field>
<field>
<id>OBJECT_NAME</id>
<enabled>Y</enabled>
<name>OBJECT_NAME</name>
</field>
<field>
<id>OBJECT_COPY</id>
<enabled>Y</enabled>
<name>OBJECT_COPY</name>
</field>
<field>
<id>REPOSITORY_DIRECTORY</id>
<enabled>Y</enabled>
<name>REPOSITORY_DIRECTORY</name>
</field>
<field>
<id>FILENAME</id>
<enabled>Y</enabled>
<name>FILENAME</name>
</field>
<field>
<id>OBJECT_ID</id>
<enabled>Y</enabled>
<name>OBJECT_ID</name>
</field>
<field>
<id>OBJECT_REVISION</id>
<enabled>Y</enabled>
<name>OBJECT_REVISION</name>
</field>
<field>
<id>PARENT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>PARENT_CHANNEL_ID</name>
</field>
<field>
<id>ROOT_CHANNEL_ID</id>
<enabled>Y</enabled>
<name>ROOT_CHANNEL_ID</name>
</field>
</channel-log-table>
<step-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>TRANSNAME</id>
<enabled>Y</enabled>
<name>TRANSNAME</name>
</field>
<field>
<id>STEPNAME</id>
<enabled>Y</enabled>
<name>STEPNAME</name>
</field>
<field>
<id>STEP_COPY</id>
<enabled>Y</enabled>
<name>STEP_COPY</name>
</field>
<field>
<id>LINES_READ</id>
<enabled>Y</enabled>
<name>LINES_READ</name>
</field>
<field>
<id>LINES_WRITTEN</id>
<enabled>Y</enabled>
<name>LINES_WRITTEN</name>
</field>
<field>
<id>LINES_UPDATED</id>
<enabled>Y</enabled>
<name>LINES_UPDATED</name>
</field>
<field>
<id>LINES_INPUT</id>
<enabled>Y</enabled>
<name>LINES_INPUT</name>
</field>
<field>
<id>LINES_OUTPUT</id>
<enabled>Y</enabled>
<name>LINES_OUTPUT</name>
</field>
<field>
<id>LINES_REJECTED</id>
<enabled>Y</enabled>
<name>LINES_REJECTED</name>
</field>
<field>
<id>ERRORS</id>
<enabled>Y</enabled>
<name>ERRORS</name>
</field>
<field>
<id>LOG_FIELD</id>
<enabled>N</enabled>
<name>LOG_FIELD</name>
</field>
</step-log-table>
<metrics-log-table>
<connection/>
<schema/>
<table/>
<timeout_days/>
<field>
<id>ID_BATCH</id>
<enabled>Y</enabled>
<name>ID_BATCH</name>
</field>
<field>
<id>CHANNEL_ID</id>
<enabled>Y</enabled>
<name>CHANNEL_ID</name>
</field>
<field>
<id>LOG_DATE</id>
<enabled>Y</enabled>
<name>LOG_DATE</name>
</field>
<field>
<id>METRICS_DATE</id>
<enabled>Y</enabled>
<name>METRICS_DATE</name>
</field>
<field>
<id>METRICS_CODE</id>
<enabled>Y</enabled>
<name>METRICS_CODE</name>
</field>
<field>
<id>METRICS_DESCRIPTION</id>
<enabled>Y</enabled>
<name>METRICS_DESCRIPTION</name>
</field>
<field>
<id>METRICS_SUBJECT</id>
<enabled>Y</enabled>
<name>METRICS_SUBJECT</name>
</field>
<field>
<id>METRICS_TYPE</id>
<enabled>Y</enabled>
<name>METRICS_TYPE</name>
</field>
<field>
<id>METRICS_VALUE</id>
<enabled>Y</enabled>
<name>METRICS_VALUE</name>
</field>
</metrics-log-table>
</log>
<maxdate>
<connection/>
<table/>
<field/>
<offset>0.0</offset>
<maxdiff>0.0</maxdiff>
</maxdate>
<size_rowset>10000</size_rowset>
<sleep_time_empty>50</sleep_time_empty>
<sleep_time_full>50</sleep_time_full>
<unique_connections>N</unique_connections>
<feedback_shown>Y</feedback_shown>
<feedback_size>50000</feedback_size>
<using_thread_priorities>Y</using_thread_priorities>
<shared_objects_file/>
<capture_step_performance>N</capture_step_performance>
<step_performance_capturing_delay>1000</step_performance_capturing_delay>
<step_performance_capturing_size_limit>100</step_performance_capturing_size_limit>
<dependencies>
</dependencies>
<partitionschemas>
</partitionschemas>
<slaveservers>
</slaveservers>
<clusterschemas>
</clusterschemas>
<created_user>-</created_user>
<created_date>2025&#x2f;05&#x2f;09 08&#x3a;34&#x3a;54.065</created_date>
<modified_user>-</modified_user>
<modified_date>2025&#x2f;05&#x2f;09 08&#x3a;34&#x3a;54.065</modified_date>
<key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA&#x3d;</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Jahr to Integer</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Row Normaliser</from>
<to>Jahr to Integer</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Text file input</from>
<to>Row Normaliser</to>
<enabled>Y</enabled>
</hop>
</order>
<step>
<name>Jahr to Integer</name>
<type>SelectValues</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields> <select_unspecified>N</select_unspecified>
<meta> <name>J</name>
<rename>J</rename>
<type>Integer</type>
<length>-2</length>
<precision>-2</precision>
<conversion_mask/>
<date_format_lenient>false</date_format_lenient>
<date_format_locale/>
<date_format_timezone/>
<lenient_string_to_number>false</lenient_string_to_number>
<encoding/>
<decimal_symbol/>
<grouping_symbol/>
<currency_symbol/>
<storage_type/>
</meta> </fields> <cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>912</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Row Normaliser</name>
<type>Normaliser</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<typefield>kenn_zahl</typefield>
<fields> </fields> <cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>384</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>eduetl</connection>
<schema/>
<table>tmp_sxc_zahl_wert</table>
<commit>1000</commit>
<truncate>N</truncate>
<ignore_errors>N</ignore_errors>
<use_batch>Y</use_batch>
<specify_fields>Y</specify_fields>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_daily>N</partitioning_daily>
<partitioning_monthly>Y</partitioning_monthly>
<tablename_in_field>N</tablename_in_field>
<tablename_field/>
<tablename_in_table>Y</tablename_in_table>
<return_keys>N</return_keys>
<return_field/>
<fields>
<field>
<column_name>kenn_zahl</column_name>
<stream_name>kenn_zahl</stream_name>
</field>
<field>
<column_name>wert_str</column_name>
<stream_name>wert</stream_name>
</field>
<field>
<column_name>hs_nr_str</column_name>
<stream_name>UHSTEXTNSTO</stream_name>
</field>
<field>
<column_name>jahr</column_name>
<stream_name>J</stream_name>
</field>
<field>
<column_name>generic_dim1</column_name>
<stream_name>L</stream_name>
</field>
</fields>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>1072</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step>
<name>Text file input</name>
<type>TextFileInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<accept_filenames>N</accept_filenames>
<passing_through_fields>N</passing_through_fields>
<accept_field>filename</accept_field>
<accept_stepname/>
<separator>&#x3b;</separator>
<enclosure>&#x22;</enclosure>
<enclosure_breaks>N</enclosure_breaks>
<escapechar/>
<header>Y</header>
<nr_headerlines>1</nr_headerlines>
<footer>N</footer>
<nr_footerlines>1</nr_footerlines>
<line_wrapped>N</line_wrapped>
<nr_wraps>1</nr_wraps>
<layout_paged>N</layout_paged>
<nr_lines_per_page>80</nr_lines_per_page>
<nr_lines_doc_header>0</nr_lines_doc_header>
<noempty>Y</noempty>
<include>N</include>
<include_field/>
<rownum>N</rownum>
<rownumByFile>N</rownumByFile>
<rownum_field/>
<format>mixed</format>
<encoding>ISO-8859-1</encoding>
<add_to_result_filenames>Y</add_to_result_filenames>
<file>
<name>&#x24;&#x7b;PATH_TO_UPLOADFILE&#x7d;</name>
<filemask/>
<exclude_filemask/>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
<type>CSV</type>
<compression>None</compression>
</file>
<filters>
</filters>
<fields>
</fields>
<limit>0</limit>
<error_ignored>N</error_ignored>
<skip_bad_files>N</skip_bad_files>
<file_error_field/>
<file_error_message_field/>
<error_line_skipped>N</error_line_skipped>
<error_count_field/>
<error_fields_field/>
<error_text_field/>
<bad_line_files_destination_directory/>
<bad_line_files_extension>warning</bad_line_files_extension>
<error_line_files_destination_directory/>
<error_line_files_extension>error</error_line_files_extension>
<line_number_files_destination_directory/>
<line_number_files_extension>line</line_number_files_extension>
<date_format_lenient>Y</date_format_lenient>
<date_format_locale>de_DE</date_format_locale>
<shortFileFieldName>filename</shortFileFieldName>
<pathFieldName/>
<hiddenFieldName/>
<lastModificationTimeFieldName/>
<uriNameFieldName/>
<rootUriNameFieldName/>
<extensionFieldName/>
<sizeFieldName/>
<cluster_schema/>
<remotesteps>
<input>
</input>
<output>
</output>
</remotesteps>
<GUI>
<xloc>176</xloc>
<yloc>320</yloc>
<draw>Y</draw>
</GUI>
</step>
<step_error_handling>
</step_error_handling>
<slave-step-copy-partition-distribution>
</slave-step-copy-partition-distribution>
<slave_transformation>N</slave_transformation>
</transformation>
Write Preview
Loading…
Cancel
Save