diff --git a/.classpath b/.classpath
index 53cf095..74839ed 100644
--- a/.classpath
+++ b/.classpath
@@ -297,11 +297,6 @@
-
-
-
-
-
@@ -867,5 +862,6 @@
+
diff --git a/src/de/memtext/util/FileUtils.java b/src/de/memtext/util/FileUtils.java
index 3e598bf..b7132fb 100644
--- a/src/de/memtext/util/FileUtils.java
+++ b/src/de/memtext/util/FileUtils.java
@@ -3,6 +3,7 @@ package de.memtext.util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -10,6 +11,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.RandomAccessFile;
import java.io.StringWriter;
import java.net.URL;
import java.net.URLConnection;
@@ -321,5 +323,52 @@ public class FileUtils {
return fileContents;
}
+ /*
+ Unix tail -n in java
+ Returns n lines from the end of a (potentially big) textfile, useful für big Logfiles
+ */
+ public static String tail(File myFile, int nLines) throws IOException {
+ int numberOfLines = 0;
+ StringBuilder builder = new StringBuilder();
+ try (RandomAccessFile randomAccessFile = new RandomAccessFile(myFile, "r")) {
+ try (ByteArrayOutputStream myByteArrayOutputStream = new ByteArrayOutputStream()) {
+ long fileLength = myFile.length() - 1;
+ /*
+ * go to the end of the file:
+ */
+ randomAccessFile.seek(fileLength);
+
+ for (long myPointer = fileLength; myPointer >= 0; myPointer--) {
+ randomAccessFile.seek(myPointer);
+ byte b = (byte) randomAccessFile.read();
+ if (b == '\n') {
+ numberOfLines++;
+ // identify last line, often terminated with a line separator
+ if (numberOfLines == (nLines + 1))
+ break;
+ }
+ myByteArrayOutputStream.write(b);
+ fileLength = fileLength - myPointer;
+ }
+ /*
+ * Read in reverse order
+ */
+ byte[] a = myByteArrayOutputStream.toByteArray();
+ int start = 0;
+ int mid = a.length / 2;
+ int end = a.length - 1;
+
+ while (start < mid) {
+ byte temp = a[end];
+ a[end] = a[start];
+ a[start] = temp;
+ start++;
+ end--;
+ }// End while-loop
+ return new String(a).trim();
+ } // End try-with-resources
+ } // End 2 try-with-resources
+
+ }
}
\ No newline at end of file
diff --git a/src/de/superx/bin/KettleExecutor.java b/src/de/superx/bin/KettleExecutor.java
index c49fbca..764682b 100644
--- a/src/de/superx/bin/KettleExecutor.java
+++ b/src/de/superx/bin/KettleExecutor.java
@@ -46,7 +46,7 @@ public class KettleExecutor {
public StringBuffer perform() throws IOException, SQLException, DBServletException {
try {
initSxJob();
- EtlStarter es = new EtlStarter();
+ EtlStarter es = new EtlStarter(this.mandantenID);
if (!isModusVorschau()) {
check_sql("vor");
}
@@ -136,7 +136,9 @@ public class KettleExecutor {
String mandantenID = GetOpts.getValue("-mandantenID");
String job_uniquename = GetOpts.getValue("-job_uniquename");
- SuperXManager.setWEB_INFPfad(GetOpts.getValue("-WEB_INF_PFAD"));
+ String webinfpfad=GetOpts.getValue("-WEB_INF_PFAD");
+ if(webinfpfad != null && !webinfpfad.equals(""))
+ SuperXManager.setWEB_INFPfad(GetOpts.getValue("-WEB_INF_PFAD"));
SuperXManager.setModuleDir(GetOpts.getValue("-MODULE_PFAD"));
String path_to_uploadfile = "";
if (GetOpts.isPresent("-path_to_uploadfile")) {
diff --git a/src/de/superx/bin/SimpleTransform.java b/src/de/superx/bin/SimpleTransform.java
index c7dad60..93f4297 100644
--- a/src/de/superx/bin/SimpleTransform.java
+++ b/src/de/superx/bin/SimpleTransform.java
@@ -63,7 +63,7 @@ public class SimpleTransform {
_out = GetOpts.getValue("-OUT");
if (GetOpts.isPresent("-method"))
method = GetOpts.getValue("-method");
- if (GetOpts.isPresent("-param"))
+ if (GetOpts.isPresent("-params"))
_params = GetOpts.getValue("-params");
//logger.initRaw(dosql.class)
diff --git a/src/de/superx/bin/fm/EtlStarter.java b/src/de/superx/bin/fm/EtlStarter.java
index 7944e07..2d32f61 100644
--- a/src/de/superx/bin/fm/EtlStarter.java
+++ b/src/de/superx/bin/fm/EtlStarter.java
@@ -38,9 +38,11 @@ public class EtlStarter implements TemplateTransformModel {
public static final String PARAM_KEY_STEP = "etl_step";
public static final String PARAM_LOGLEVEL = "log_level";
private static Logger logger = Logger.getLogger(EtlStarter.class);
-
+ private String mandantenId = "default";
private LogLevel logLevel = LogLevel.BASIC;
-
+ public EtlStarter(String mandantenId) {
+ this.mandantenId = mandantenId;
+ }
@Override
public Writer getWriter(Writer paramWriter, Map paramMap) throws TemplateModelException, IOException {
logger.info("**EtlStarter**");
@@ -210,7 +212,7 @@ public class EtlStarter implements TemplateTransformModel {
String jobfile = "file:///home/superx/data-integration/exceltest.kjb";
Map jobParams = new HashMap();
jobParams.put("PATH_TO_EXCELFILE", "/home/superx/tmp/testexcel2.xlsx");
- EtlStarter es = new EtlStarter();
+ EtlStarter es = new EtlStarter("default");
es.kettleCallEmbedded("default",jobfile, jobParams,false);
}
diff --git a/src/de/superx/common/Maske.java b/src/de/superx/common/Maske.java
index 486eadf..9524891 100644
--- a/src/de/superx/common/Maske.java
+++ b/src/de/superx/common/Maske.java
@@ -1038,8 +1038,7 @@ public class Maske extends NamedIdObject implements Serializable {
: this.navi;
StringBuffer result = new StringBuffer("\n\n\n");
diff --git a/src/de/superx/common/TemplateProcessor.java b/src/de/superx/common/TemplateProcessor.java
index c339c79..059a700 100644
--- a/src/de/superx/common/TemplateProcessor.java
+++ b/src/de/superx/common/TemplateProcessor.java
@@ -90,7 +90,7 @@ public class TemplateProcessor {
// die Formatierungseinstellung wird u.a. für sortnr gebraucht,
// sonst erscheint z.B. 1000 als 1.000
myCfg.setSetting("number_format", "###########.####");
- // TODO myCfg.setSharedVariable("etl", new EtlStarter(mandantenID));
+ myCfg.setSharedVariable("etl", new EtlStarter(mandantenID));
} catch (TemplateException e) {
System.out.println("FreeMarker warning: " + e);
e.printStackTrace();
@@ -220,7 +220,7 @@ public class TemplateProcessor {
List results = new ArrayList();
for (Iterator it = roh.iterator(); it.hasNext();) {
SxResultRow zeile = (SxResultRow) it.next();
- results.add((String) zeile.get(0));
+ results.add( zeile.get(0).toString());
}
hmap.put(varname, results);
diff --git a/src/de/superx/etl/EtlUtils.java b/src/de/superx/etl/EtlUtils.java
new file mode 100644
index 0000000..08bb8fe
--- /dev/null
+++ b/src/de/superx/etl/EtlUtils.java
@@ -0,0 +1,343 @@
+/*
+ * de.superx.etl - a package for controlling ETL routines
+ * Copyright (C) 2021 Daniel Quathamer
+ *
+ * This package is licensed under the CampusSource License;
+ * http://www.campussource.de/org/license/
+ */
+package de.superx.etl;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.Enumeration;
+import java.util.Properties;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import de.superx.servlet.SuperXManager;
+
+/**
+
+ */
+public class EtlUtils {
+public static final String NEWLINE=System.getProperty("line.separator");
+public static final String PATHSEP=File.separator;
+public static String WEBINFDIR=SuperXManager.getWEB_INFPfad();
+public static String SUPERX_DIR=(System.getProperties().containsKey("SUPERX_DIR")? System.getProperty("SUPERX_DIR"):"");
+public static String MODUL_ROOT_PFAD=(System.getProperties().containsKey("MODULE_PFAD")? System.getProperty("MODULE_PFAD"):SUPERX_DIR+PATHSEP+"db"+PATHSEP+"module");
+public static String MODUL_ROHDATEN_SUBPFAD=PATHSEP+"rohdaten"+PATHSEP;
+
+
+public static void main(String args[]) {
+
+ try {
+ String webinfdir=getWebinfDirectory();;
+ System.out.println(webinfdir);
+
+ } catch (URISyntaxException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+}
+
+public static Properties convertStringToProperty(String inp) throws IOException
+{
+ Properties myProps = new Properties();
+ myProps.load(new StringReader(inp));
+ return myProps;
+}
+/* Ein EtlJob hat vordefininierte Parameter, die zur Laufzeit durch die runTimeParams ersetzt werden.
+ * Der merge findet hier statt:
+ */
+public static Properties mergeParamProperties(Properties params, Properties runTimeParams)
+
+{
+ if(params==null) {
+ //Job ist noch unbekannt, oder hat keine vordef. Parameter
+ //in diesem Falle werden die Standardparameter übergeben (Modulpfade, SUPERX_DIR)
+ params=runTimeParams;
+ }
+ else
+ {
+ String parsedParam;
+ Enumeration runTimeParamNames = runTimeParams.propertyNames();
+ while (runTimeParamNames.hasMoreElements()) {
+ String runTimeParamName = (String)runTimeParamNames.nextElement();
+ String runTimeParamValue = runTimeParams.getProperty(runTimeParamName);
+ Enumeration paramNames = params.propertyNames();
+ while (paramNames.hasMoreElements()) {
+ String paramName=(String)paramNames.nextElement();
+ String paramValue=params.getProperty(paramName);
+ if(paramName.equals(runTimeParamName) ) {
+ paramValue=runTimeParamValue;
+ params.setProperty(paramName, paramValue);
+ }
+ if(paramValue.indexOf("$"+runTimeParamName)>-1) {
+ paramValue=de.memtext.util.StringUtils.replace(paramValue,"$"+runTimeParamName, runTimeParamValue);
+ params.setProperty(paramName, paramValue);
+ }
+
+ }
+ }
+ }
+ return params;
+
+}
+public static String parseStringAgainstParams(String targetString, Properties params)
+
+{
+ String parsedParam;
+ Enumeration paramNames = params.propertyNames();
+ while (paramNames.hasMoreElements()) {
+ String paramName = (String)paramNames.nextElement();
+ String paramValue = params.getProperty(paramName);
+ if(targetString.indexOf("$"+paramName)>-1) {
+ targetString=de.memtext.util.StringUtils.replace(targetString,"$"+paramName, paramValue);
+ }
+
+ }
+ return targetString;
+}
+public static void initJobEnvironment()
+{
+ String initVar="";
+ if(System.getProperty("WEBINFDIR") ==null)
+ {
+
+ try {
+ initVar=de.superx.servlet.SuperXManager.getWEB_INFPfad();
+ if(initVar==null || initVar.equals("."))
+ {
+ //ermittle webinfdir
+ initVar=getWebinfDirectory();
+
+ }
+ } catch (Exception e) {
+ // do nothing, try another
+ }
+
+ WEBINFDIR=initVar;
+
+ }
+
+ if(System.getProperty("SUPERX_DIR") ==null)
+ {
+ SUPERX_DIR=WEBINFDIR+PATHSEP+"conf"+PATHSEP+"edustore";
+
+ }
+
+}
+private static String getJarName()
+{
+ return new File(EtlUtils.class.getProtectionDomain()
+ .getCodeSource()
+ .getLocation()
+ .getPath())
+ .getName();
+}
+
+private static boolean runningFromJar()
+{
+ String jarName = getJarName();
+ return jarName.contains(".jar");
+}
+
+public static String getWebinfDirectory() throws URISyntaxException
+{
+ if (runningFromJar())
+ {
+ return getWebinfDirectoryFromJar();
+ } else
+ {
+ return getWebinfDirectoryFromClass();
+ }
+}
+
+private static String getWebinfDirectoryFromClass()
+{
+ File f= new File(EtlUtils.class.getProtectionDomain()
+ .getCodeSource()
+ .getLocation()
+ .getPath()+PATHSEP+"..");
+ String class_path=f.getAbsolutePath();
+ return class_path;
+
+}
+
+private static String getWebinfDirectoryFromJar() throws URISyntaxException
+{
+ File f1=new File(EtlUtils.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
+ //now we're got superx.jar, go up one level
+ File f2=new File(f1.getParent());
+ //now we've got WEB-INF/lib, return WEB-INF:
+
+ return f2.getParent();
+
+}
+/* die folgenden 2 Methoden
+ * getFileContentsWithEncoding
+ * saveFileContentsWithEncoding
+ * sind in kern5.0 in de.superx.util.FileUtils
+ * wg. abwärtskompatiblität hierhin kopiert, sollten langfristig wieder weg
+ */
+public static String getFileContentsWithEncoding(String filePath, String encoding) {
+ File f = new File(filePath);
+ if (!f.exists()) {
+ System.out.println("Fehler: Datei " + filePath + " existiert nicht.");
+ return null;
+ }
+ String fileContents = "";
+ if (encoding == null || encoding.trim().equals("")) {
+ encoding = System.getProperty("file.encoding");
+ }
+ try {
+ // --- IputStream und OutputStream generieren ---//
+ FileInputStream fis = new FileInputStream(f);
+ // Wenn Quelldatei Unicode, dann speziellen Reader nutzen
+ BufferedReader in;
+ //BufferedReader ist schneller bei großen Dateien
+ in = new BufferedReader(new InputStreamReader(fis, encoding));
+ // --- Output-Stream der temporären Datei erzeugen ---//
+ StringWriter out = new StringWriter();
+ // --- Verarbeiten der Datei ---//
+ String text;
+ text = in.readLine();
+ while (text != null) { // Datei nicht leer
+ out.write(text);
+ out.write(System.getProperty("line.separator"));
+ text = in.readLine();
+ }
+ if (!(out == null)) {
+ fileContents = out.toString();
+ }
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (UnsupportedEncodingException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return fileContents;
+ }
+
+ public static void saveFileContentsWithEncoding(String filename, String contents, String encoding) throws
+
+ FileNotFoundException,
+ IOException
+ {
+
+
+ File f = new File(filename);
+ BufferedReader in;
+ BufferedWriter out;
+
+ //Default encoding ist utf-8
+ if (encoding == null) encoding = System.getProperty("file.encoding");
+ // --- Output-Stream der temporären Datei erzeugen ---//
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), encoding));
+
+ out.write(contents);
+
+ out.close();
+
+
+ }//Ende der Methode
+
+
+/*SAX Document aus XML-Datei erzeugen */
+ public static Document buildDocumentFromXmlFile(String srcFile)
+ throws ParserConfigurationException, FileNotFoundException, SAXException, IOException {
+ Document mydomres;
+ org.xml.sax.InputSource is;
+ DocumentBuilderFactory dfactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = dfactory.newDocumentBuilder();
+ FileInputStream in = new FileInputStream(srcFile);
+
+ is=new org.xml.sax.InputSource(in);
+ mydomres = builder.newDocument();
+ mydomres = builder.parse(is);
+ return mydomres;
+ }
+ /***
+ * TODO: nach merge in master diese Methode löschen, und auf de.memtext.util.XMLUtils.parseXml(String) verweisen
+ * @author Witt This function parses XML-containing string into documents while
+ * preserving the namespaces and is primarily meant to be used withing
+ * (jUnit) test cases
+ * @param xmlString
+ * @return
+ * @throws ParserConfigurationException
+ * @throws SAXException
+ * @throws IOException
+ */
+ public static Document parseXml(String xmlString) throws ParserConfigurationException, SAXException, IOException {
+ DocumentBuilderFactory myFactory = DocumentBuilderFactory.newInstance();
+ myFactory.setNamespaceAware(true);
+ DocumentBuilder myBuilder;
+ myBuilder = myFactory.newDocumentBuilder();
+ Document myDocument = myBuilder.parse(new InputSource(new StringReader(xmlString)));
+ return myDocument;
+ }
+ /**
+ * @param src
+ * @param tidInXmlFile
+ * @return
+ * @throws ParserConfigurationException
+ * @throws SAXException
+ * @throws IOException
+ * @throws XPathExpressionException
+ */
+ public static boolean isNodeValueInXml(String src, String searchPath, String expectedValue)
+ throws ParserConfigurationException, SAXException, IOException, XPathExpressionException {
+ boolean b=false;
+ Document mydomres =de.superx.etl.EtlUtils.parseXml(src);
+ XPathFactory factory = new net.sf.saxon.xpath.XPathFactoryImpl();
+
+ XPath xPath = factory.newXPath();
+
+ Node myNode=(Node) xPath.compile(searchPath).evaluate(
+ mydomres, XPathConstants.NODE);
+ if(myNode!=null)
+ {
+ String foundValue=de.memtext.util.XMLUtils.getTheValue(myNode);
+ if(!(foundValue==null) && foundValue.trim().equals(expectedValue))
+ b=true;
+ }
+ return b;
+
+ }
+ public static String translateReturnCode(int returnCode)
+ {
+ String returnString="Fehlerhaft";
+ if(returnCode==0)
+ returnString="Erfolg";
+ return returnString;
+ }
+}
diff --git a/src/de/superx/sec/StringCheck.java b/src/de/superx/sec/StringCheck.java
new file mode 100644
index 0000000..1aceca3
--- /dev/null
+++ b/src/de/superx/sec/StringCheck.java
@@ -0,0 +1,26 @@
+package de.superx.sec;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+/*
+ * Check for letters, numbers and characters that don't cause problems in SQL when enclosed in ''
+ */
+public class StringCheck implements InputCheck {
+ public static String STRING_CHECK = "string";
+ private static String LETTER = "\\p{gc=L}"; // Unicode General Category "Letter"
+ private static String DECIMAL = "\\p{gc=Nd}"; // Unicode General Category "Decimal Number"
+ private static String SEPARATOR = "\\p{gc=Z}"; // Unicode General Category "Separator,space"
+ private static String CURRENCY = "\\p{gc=Sc}"; // Unicode General Category "Currency"
+ private static String ALLOWED_CHARS="()\\[\\]&/$§=\\,!#*~"; //Characters that don't cause problems in SQL when enclosed in ''
+ private static Pattern STRING = Pattern.compile("\\A[" + LETTER + DECIMAL + SEPARATOR + CURRENCY + ALLOWED_CHARS+"\\.:\\-\\_/\\n\\r@\\^]*\\z");
+
+ @Override
+ public boolean check(String input) {
+ if (input.equals("--leer--")) {
+ return true;
+ }
+ Matcher matcher = STRING.matcher(input);
+ return matcher.find();
+ }
+
+}
diff --git a/src/de/superx/servlet/SxPool.java b/src/de/superx/servlet/SxPool.java
index e617013..63cad46 100644
--- a/src/de/superx/servlet/SxPool.java
+++ b/src/de/superx/servlet/SxPool.java
@@ -3,6 +3,7 @@ package de.superx.servlet;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
+import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
@@ -341,7 +342,15 @@ public class SxPool extends GenericObjectPool implements NamedObjectI {
loginP = rs.getInt(1);
}
isErweitertesProtokollWanted = loginP == 1;
-
+ String myWEBINFFilePath=SuperXManager.getWEB_INFPfad();
+ //bei Kommandozerilenaufrufen ist WEB-INF nicht aus dem Servlet Manager ermittelbar:
+ if(myWEBINFFilePath==null || myWEBINFFilePath.equals("."))
+ try {
+ myWEBINFFilePath= de.superx.etl.EtlUtils.getWebinfDirectory();
+ } catch (URISyntaxException e1) {
+ // Default ist user.dir:
+ myWEBINFFilePath=System.getProperty("user.dir");
+ } //
rs = st.executeQuery("select tid from sx_repository where id='WEBINFDIR'");
int tid = 0;
while (rs.next()) {
@@ -353,7 +362,7 @@ public class SxPool extends GenericObjectPool implements NamedObjectI {
}
try (PreparedStatement pst = con
.prepareStatement("update sx_repository set content=? where id='WEBINFDIR'")) {
- pst.setString(1, SuperXManager.getWEB_INFPfad());
+ pst.setString(1, myWEBINFFilePath);
pst.execute();
}
}
diff --git a/src/de/superx/servlet/UserInitializer.java b/src/de/superx/servlet/UserInitializer.java
index 745f784..37ed633 100644
--- a/src/de/superx/servlet/UserInitializer.java
+++ b/src/de/superx/servlet/UserInitializer.java
@@ -691,7 +691,22 @@ public class UserInitializer {
+ "" + "" + "