From 3cfff68e833ba98f9be5dda1f45e0823f455b9b1 Mon Sep 17 00:00:00 2001 From: Imo John Date: Wed, 3 Dec 2025 11:19:51 +0100 Subject: [PATCH] importjob umgebaut fuer Rohdaten - etl metadata injection #53 --- src-modules/module/sxc/conf/sxc.xml | 26 + .../import_bestand_mwk_nrw_hskennz.kjb | 101 ++- ...port_bestand_mwk_nrw_hskennz_injection.ktr | 637 ++++++++++++++++ ...mport_bestand_mwk_nrw_hskennz_template.ktr | 686 ++++++++++++++++++ 4 files changed, 1434 insertions(+), 16 deletions(-) create mode 100644 src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr create mode 100644 src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr diff --git a/src-modules/module/sxc/conf/sxc.xml b/src-modules/module/sxc/conf/sxc.xml index 1c6a55a..7d7d6e2 100644 --- a/src-modules/module/sxc/conf/sxc.xml +++ b/src-modules/module/sxc/conf/sxc.xml @@ -532,6 +532,32 @@ rightname="CS_BIA_STANDARDREPORTS_ADMIN[SXC]" /> + + Formeln für Grunddaten und Kennzahlen + + + + + + + + + + public + sxc_katalog + tid + 1 + c_sxc_katalog + + + + + + + + +
+ diff --git a/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb index cba1e69..768b1c4 100644 --- a/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb +++ b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz.kjb @@ -13,7 +13,7 @@ PATH_TO_UPLOADFILE - + /home/superx/Downloads/stamm all - Auszug UDE-3.csv @@ -323,12 +323,12 @@ 192 - excel_into_tmp + metadata_injection TRANS filename - ${Internal.Job.Filename.Directory}/import_bestand_mwk_nrw_hskennz_1.ktr + ${Internal.Job.Filename.Directory}/import_bestand_mwk_nrw_hskennz_injection.ktr N N @@ -354,14 +354,14 @@ N Y 0 - 272 - 320 + 688 + 352 create tmp_sxc_zahl_wert SQL - drop table if exists tmp_sxc_zahl_wert; CREATE TABLE tmp_sxc_zahl_wert ( tid serial, hs_nr_str varchar(255), hs_nr char(40), kenn_zahl char(100) NOT NULL, jahr smallint, sem smallint, ch110_institut char(10), bland integer, sxc_bestand_id integer, wert_str varchar(255), wert decimal(30,20), generic_dim1 char(100), generic_dim2 char(100) ); + drop table if exists tmp_sxc_zahl_wert; drop table if exists tmp_feld; drop table if exists tmp_feld_all; create table tmp_feld (Feld varchar(255)); create table tmp_feld_all (Feld varchar(255)); CREATE TABLE tmp_sxc_zahl_wert ( tid serial, hs_nr_str varchar(255), hs_nr char(40), kenn_zahl char(100) NOT NULL, jahr smallint, sem smallint, ch110_institut char(10), bland integer, sxc_bestand_id integer, wert_str varchar(255), wert decimal(30,20), generic_dim1 char(100), generic_dim2 char(100) ); F F @@ -405,6 +405,57 @@ 880 272 + + tmp_feld_write + + TRANS + filename + + ${Internal.Job.Filename.Directory}/tmp_feld_write.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + + Y + + N + Y + 0 + 272 + 352 + + + delete from tmp_feld + + SQL + delete from tmp_feld where Feld in ('IDLFB','J','H','L','UHSARTMINGRUPPTEXT','UHSARTMINTEXT','ME', 'UHSTEXTHS','UHSTEXTNSTO','ULEFOTEXT','UFGSCHL','UFGTEXT'); + F + F + + F + eduetl + N + Y + 0 + 480 + 352 + @@ -416,15 +467,6 @@ Y Y - - create tmp_sxc_zahl_wert - excel_into_tmp - 0 - 0 - Y - Y - N - insert into sxc_zahl_wert + NRW-Durchschnitt Success @@ -453,7 +495,7 @@ N - excel_into_tmp + metadata_injection update tmp_sxc_zahl_wert 0 0 @@ -461,6 +503,33 @@ Y N + + create tmp_sxc_zahl_wert + tmp_feld_write + 0 + 0 + Y + Y + N + + + tmp_feld_write + delete from tmp_feld + 0 + 0 + Y + Y + N + + + delete from tmp_feld + metadata_injection + 0 + 0 + Y + Y + N + diff --git a/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr new file mode 100644 index 0000000..3d7d18e --- /dev/null +++ b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_injection.ktr @@ -0,0 +1,637 @@ + + + + import_bestand_mwk_nrw_hskennz_injection + + + + Normal + 0 + / + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2023/03/01 20:02:48.054 + - + 2023/03/01 20:02:48.054 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Data Grid + ETL Metadata Injection + Y + + + tmp_feld + ETL Metadata Injection + Y + + + tmp_feld_all + ETL Metadata Injection + Y + + + + Data Grid + DataGrid + + Y + + 1 + + none + + + + + type_inputFields + + + + + + -1 + -1 + N + + + trans_new_field + + + + + + -1 + -1 + N + + + + Stringwert + + + + + + + + + + 320 + 128 + Y + + + + + ETL Metadata Injection + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Transformation.Filename.Directory}/import_bestand_mwk_nrw_hskennz_template.ktr + + + + N + + + Row Normaliser + VALUE + Y + tmp_feld + feld + Row Normaliser + NORMALISED + Y + Data Grid + trans_new_field + Row Normaliser + NAME + Y + tmp_feld + feld + Text file input + FIELD_TYPE + Y + Data Grid + type_inputFields + Text file input + FIELD_NAME + Y + tmp_feld_all + feld + + + + + + + + + 848 + 208 + Y + + + + + tmp_feld + TableInput + + N + + 1 + + none + + + eduetl + SELECT Feld FROM tmp_feld; + 0 + + N + N + N + + + + + + + + + 320 + 240 + Y + + + + + tmp_feld_all + TableInput + + N + + 1 + + none + + + eduetl + SELECT Feld FROM tmp_feld_all; + 0 + + N + N + N + + + + + + + + + 320 + 320 + Y + + + + + + + + N + diff --git a/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr new file mode 100644 index 0000000..919ae1a --- /dev/null +++ b/src-modules/module/sxc/etl/import_bestand/mwk_nrw_hochschulkennzahlen/import_bestand_mwk_nrw_hskennz_template.ktr @@ -0,0 +1,686 @@ + + + + import_bestand_mwk_nrw_hskennz_template + + + + Normal + 0 + / + + + PATH_TO_UPLOADFILE + /home/superx/Downloads/stamm all - Auszug UDE-3.csv + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2025/05/09 08:34:54.065 + - + 2025/05/09 08:34:54.065 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Jahr to Integer + Table output + Y + + + Row Normaliser + Jahr to Integer + Y + + + Text file input + Row Normaliser + Y + + + + Jahr to Integer + SelectValues + + Y + + 1 + + none + + + N + J + J + Integer + -2 + -2 + + false + + + false + + + + + + + + + + + + + + 912 + 320 + Y + + + + + Row Normaliser + Normaliser + + Y + + 1 + + none + + + kenn_zahl + + + + + + + + + 384 + 320 + Y + + + + + Table output + TableOutput + + Y + + 1 + + none + + + eduetl + +
tmp_sxc_zahl_wert
+ 1000 + N + N + Y + Y + N + + N + Y + N + + Y + N + + + + kenn_zahl + kenn_zahl + + + wert_str + wert + + + hs_nr_str + UHSTEXTNSTO + + + jahr + J + + + generic_dim1 + L + + + + + + + + + + + 1072 + 320 + Y + + + + + Text file input + TextFileInput + + Y + + 1 + + none + + + N + N + filename + + ; + " + N + +
Y
+ 1 +
N
+ 1 + N + 1 + N + 80 + 0 + Y + N + + N + N + + mixed + ISO-8859-1 + Y + + ${PATH_TO_UPLOADFILE} + + + N + N + CSV + None + + + + + + 0 + N + N + + + N + + + + + warning + + error + + line + Y + de_DE + filename + + + + + + + + + + + + + + + + 176 + 320 + Y + + + + + + + + N +