kern/test-src/de/superx/kettle/KdsfTransformationTest.java

package de.superx.kettle;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Year;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import javax.naming.NamingException;
import javax.sql.DataSource;
import javax.xml.namespace.NamespaceContext;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.xerces.jaxp.validation.XMLSchemaFactory;
import org.junit.AfterClass;
//import org.apache.xerces.jaxp.validation.XMLSchemaFactory;
//import org.h2.tools.Server;
import org.junit.Assert;
import org.junit.Test;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.database.DataSourceProviderFactory;
import org.pentaho.di.core.database.DataSourceProviderInterface;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.logging.LogLevel;
import org.pentaho.di.core.parameters.UnknownParamException;
import org.pentaho.di.job.Job;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.metastore.DatabaseMetaStoreUtil;
import org.pentaho.metastore.api.exceptions.MetaStoreException;
import org.pentaho.metastore.stores.memory.MemoryMetaStore;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import de.memtext.util.StringUtils;
import de.memtext.util.XMLUtils;
import de.superx.db.H2InMemoryCreator;
import de.superx.util.test.KdsfNamespaceContext;
import de.superx.util.test.KettleH2TestDataSourceProvider;

public class KdsfTransformationTest {

    private static final int YEAR_OF_REFERENCE_DATA_CREATION = 2019;
    private static final String SCHEMA_PATH = String.join(File.separator, "test", "resources", "schema_files", "kdsf_xml_export", "schema");
    private static final String VIEWS_PATH = String.join(File.separator, "test", "resources", "schema_files", "kdsf_xml_export", "views");
    private static final String UNLOADS_PATH = String.join(File.separator, "test", "resources", "schema_files", "kdsf_xml_export", "unloads");
    private static final String SCHEMA_LOCATION = String.join(File.separator, "test", "resources", "schema_files", "kdsf_xml_export", "xsd", "kdsf.xsd");
    private static final String KETTLE_PLUGIN_DIR = String.join(File.separator, "superx", "WEB-INF", "kettle-plugins");

    private static final String REFERENCE_FILE_LOCATION_BASE = String.join(File.separator, "test", "resources", "xmlunittest", "kdsf_testfile_");

    private DataSource datasource = null;
    private List<String> errorList;

    @Test
    public void testKdsfTransformation() throws InstantiationException, IllegalAccessException, ClassNotFoundException, IOException, SQLException, NamingException, KettleException, MetaStoreException, XPathExpressionException, ParserConfigurationException, SAXException {
        createInMemoryTestDb();
        MemoryMetaStore metastore = new MemoryMetaStore();
        initKettleEnvironment(metastore);
        this.errorList = new ArrayList<String>();

        // attention: it seems that in memory datasource is destroyed by kettle processing if we do not keep an open connection!?
        try ( Connection conn = this.datasource.getConnection(); ) {
            verjuengeTestdaten(conn);
            exportAndValidateYear(metastore, conn, 2018);
            exportAndValidateYear(metastore, conn, 1900);
//          Server.startWebServer(conn); // uncomment for local (e.g. dev env) debugging
        }

        Assert.assertEquals(String.join("\n", this.errorList), 0, this.errorList.size());
    }

    public static void main(String args[]) throws IOException, ParserConfigurationException, SAXException, XPathExpressionException {
        if (args.length == 0) {
            printHelpAndExit();
        } else if (args.length < 2) {
            System.out.println("ERROR: awaiting two input files...");
            System.exit(-1);
        }
        String referenceFilePath = args[0];
        String testFilePath = args[1];
        compareXmlLeaveElements(referenceFilePath, testFilePath);
    }

    private static void compareXmlLeaveElements(String referenceFilePath, String testFilePath)
            throws IOException, ParserConfigurationException, SAXException, XPathExpressionException {
        Document referenceDocument = loadXMLDocument(referenceFilePath);
        Document testDocument = loadXMLDocument(testFilePath);
        validateAgainstSchema(referenceDocument);
        validateAgainstSchema(testDocument);
        List<String> missingTestLeaves = findMissingLeaves(referenceDocument, testDocument);
        if (!missingTestLeaves.isEmpty()) {
            System.out.println("the following elements from reference file are missing in export file\n" +
                String.join("\n", missingTestLeaves));
        }
    }

    private static void printHelpAndExit() {
        System.out.println("Use this program to perform a basic check on exported KDSF-Files:\n"
                + "- the first parameter is used as path for a reference file\n"
                + "- the second parameter is used as path for the file to be checked\n"
                + "- if both files conform to the KDSF XSD schema\n"
                + "- and if the XML leave elements of the reference file are contained in those of the file to be checked\n"
                + "...then the test is cosidered to be successful");
        System.exit(0);
    }

    /*
     * Im KDSF-Export wird über eine Formel die Altersgruppe ermittelt;
     * Da die Referenzdatei des Tests statisch ist, muss das Alter in den Testdaten verjüngt werden,
     * damit sich die Altersgruppe nicht ändert, wenn der Testfall zu einem späteren Zeitpunkt
     * ausgeführt wird (analog zu Datenverjüngung in HISinOne).
     */
    private static void verjuengeTestdaten(Connection conn) throws SQLException {
        Statement statement = conn.createStatement();
        int birthyearOffset = Year.now().getValue() - YEAR_OF_REFERENCE_DATA_CREATION;
        statement.execute("UPDATE res_person SET birthdate = DATEADD(YEAR, " + birthyearOffset + ", birthdate);");
    }

    private void exportAndValidateYear( MemoryMetaStore metastore, Connection conn, int year) throws SQLException, UnknownParamException, KettleXMLException, ParserConfigurationException, SAXException, IOException, XPathExpressionException {

        Job job = runKettleJob(metastore, year);
        notifyLoggedErrors(job);
        String exportContent = fetchXmlString(conn);
        Document exportDocument = XMLUtils.parseXml(exportContent);
//        writeToFile(exportDocument, year); // uncomment for debugging purposes
        validateAgainstSchema(exportDocument);
        // content check: ensure all xml leaves of reference file to appear in export file (naive, but should suffice)
        Document referenceDocument = loadXMLDocument(REFERENCE_FILE_LOCATION_BASE + year + ".xml");
        validateAgainstSchema(referenceDocument);
        List<String> missingExportLeaves = findMissingLeaves(referenceDocument, exportDocument);

        if(!missingExportLeaves.isEmpty()) {
            this.errorList.add("the following elements from reference file are missing in export file from year " + year + "\n" + String.join("\n", missingExportLeaves));
        }

    }

    private static List<String> findMissingLeaves(Document referenceDocument, Document testDocument) throws XPathExpressionException {
        NodeList exportElements = extractLeaveElements(testDocument);
        NodeList referenceElements = extractLeaveElements(referenceDocument);
        List<String> missingExportLeaves = findMissingMatches(referenceElements, exportElements);
        return missingExportLeaves;
    }

    private static List<Node> toGenericList(NodeList nodeList) {
        List<Node> result = new ArrayList<Node>();
        for (int i = 0; i < nodeList.getLength(); i++) {
            result.add(nodeList.item(i));
        }
        return result;
    }

    private static Document loadXMLDocument(String fileLocation)
            throws IOException, ParserConfigurationException, SAXException {
        File file = new File(fileLocation);
        String content = StringUtils.readFile(file, StandardCharsets.UTF_8);
        Document xmlDocument = XMLUtils.parseXml(content);
        return xmlDocument;
    }

    private static List<String> findMissingMatches(NodeList referenceElements, NodeList exportElements) {
        List<String> missingMatches = new LinkedList<String>();
        List<Node> possibleMatches = toGenericList(exportElements);
        for(int i = 0; i < referenceElements.getLength(); i++) {
            Node refNode = referenceElements.item(i);
            if (!hasMatch(refNode, possibleMatches)) {
                String missedMatch = "element name: \"" + refNode.getNodeName() + "\" content: \"" + refNode.getTextContent() + "\"";
                missingMatches.add(missedMatch);
            }
        }
        return missingMatches;
    }

    private static boolean hasMatch(Node testNode, List<Node> possibleMatches) {
        for (int j = 0; j < possibleMatches.size(); j++) {
            Node matchTestNode = possibleMatches.get(j);
            if (testNode.isEqualNode(matchTestNode)) {
                // if we match one leave in the test file, we "burn" it, so it does not cause multiple matches for reference elements
                possibleMatches.remove(j);
                return true;
            }
        }
        return false;
    }

    private static NodeList extractLeaveElements(Document document) throws XPathExpressionException {
        XPath xpath = setupXPath(document);
        XPathExpression expr = xpath.compile("//*[count(./*) = 0]");
        NodeList leaveElements = (NodeList)expr.evaluate(document, XPathConstants.NODESET);
        return leaveElements;
    }

    private static void createInMemoryTestDb() throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, NamingException {
        H2InMemoryCreator h2InMemoryCreator = new H2InMemoryCreator();
        String schemaPath = new File(SCHEMA_PATH).getAbsolutePath();
        String viewsPath = new File(VIEWS_PATH).getAbsolutePath();
        String unloadsPath = new File(UNLOADS_PATH).getAbsolutePath();
        h2InMemoryCreator.setSchemaDir(schemaPath)
                         .setViewDir(viewsPath)
                         .setUnlDir(unloadsPath)
                         .setUserName("sa")
                         .setTablePattern("res_.*")
                         .setViewPattern("skipeverything");
        h2InMemoryCreator.createH2InMemory();
    }

    private void initKettleEnvironment(MemoryMetaStore metastore) throws KettleException, MetaStoreException {
        System.setProperty("BI_KETTLE_PLUGIN_BASE_FOLDERS", KETTLE_PLUGIN_DIR);
        KettleEnvironment.init();
        DataSourceProviderInterface providerinterface = new KettleH2TestDataSourceProvider("default", null);
        DataSourceProviderFactory.setDataSourceProviderInterface(providerinterface);
        this.datasource = providerinterface.getNamedDataSource("eduetl");
        DatabaseMeta dbmeta = new DatabaseMeta("eduetl", "POSTGRESQL", "JNDI", null, "eduetl", "1521", null, null);
        DatabaseMetaStoreUtil.createDatabaseElement(metastore, dbmeta);
    }

    private static Job runKettleJob(MemoryMetaStore metastore, int year) throws KettleXMLException, UnknownParamException {
        JobMeta jobMeta = new JobMeta(null, "superx/WEB-INF/conf/edustore/db/module/res/etl/kdsf/job_kdsf.kjb", null, metastore, null);
        jobMeta.setParameterValue("release_year", "" + year);
        jobMeta.setParameterValue("table_prefix", "res_");
        jobMeta.setParameterValue("export_employment", "1");
        jobMeta.setParameterValue("export_patent", "1");
        jobMeta.setParameterValue("export_project", "1");
        jobMeta.setParameterValue("export_publication", "1");
        jobMeta.setParameterValue("export_research_infrastructure", "1");
        jobMeta.setParameterValue("export_sdcp", "1");
        jobMeta.setParameterValue("export_file_name", " ");
        Job job = new Job(null, jobMeta);
        job.setLogLevel(LogLevel.ERROR);
        job.start();
        job.waitUntilFinished();
        return job;
    }

    private static void notifyLoggedErrors(Job job) {
        Result result = job.getResult();
        String log = result.getLogText();
        boolean errorsOccured = log.contains("ERROR");
        Assert.assertFalse("Errors where found in kettle log:\n" + log, errorsOccured);
    }

    private static XPath setupXPath(Document document) {
        XPathFactory xPathfactory = XPathFactory.newInstance();
        XPath xpath = xPathfactory.newXPath();
        NamespaceContext nsc = new KdsfNamespaceContext(document);
        xpath.setNamespaceContext(nsc);
        return xpath;
    }

    private static String fetchXmlString(Connection conn) throws SQLException {
        Statement statement = conn.createStatement();
        ResultSet resultSet = statement.executeQuery("select xmlfile from res_output");
        resultSet.next();
        String xmlString = resultSet.getString("xmlfile");
        return xmlString;
    }

    private static void validateAgainstSchema(Document document) throws SAXException, IOException {
        SchemaFactory fac = new XMLSchemaFactory();
        File schemaFile = new File(SCHEMA_LOCATION);
        Schema schema = fac.newSchema(schemaFile);
        Validator validator = schema.newValidator();
        Source source = new DOMSource(document);
        validator.validate(source);
    }

    private static void writeToFile(Document document, int year) {
        try {
            Transformer transformer = TransformerFactory.newInstance().newTransformer();
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            DOMSource source = new DOMSource(document);
            // insert your output directory here
            OutputStream out = new FileOutputStream(REFERENCE_FILE_LOCATION_BASE + year + ".xml");
            Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
            StreamResult streamResult = new StreamResult(writer);
            transformer.transform(source, streamResult);
            writer.flush();
        } catch (Exception e) {
            // nothing happening here...
        }
    }
    
    @AfterClass
    public static void after() {
        KettleEnvironment.shutdown();
    }

}