package de.superx.spring.service;

import static de.memtext.util.XMLUtils.replaceMultipleUnloadJobs;
import static de.superx.util.PathAndFileUtils.getAttribute;
import static de.superx.util.PathAndFileUtils.getDbiFileForManualEtlStep;
import static de.superx.util.PathAndFileUtils.getModulePath;
import static de.superx.util.PathAndFileUtils.getSystemInfoId;
import static de.superx.util.PathAndFileUtils.getUnloadXmlFile;
import static de.superx.util.PathAndFileUtils.getXmlFile;
import static de.superx.util.PathAndFileUtils.getXslFile;
import static de.superx.util.PathAndFileUtils.readXmlFile;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

import javax.xml.transform.stream.StreamSource;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.w3c.dom.CharacterData;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import de.memtext.util.StringUtils;
import de.superx.dbt.DbtAction;
import de.superx.dbt.DbtExecutionState;
import de.superx.dbt.DbtRunRequest;
import de.superx.dbt.DbtRunResult;
import de.superx.dbt.DbtWrapper;
import de.superx.etl.biad.DBSource;
import de.superx.etl.biad.SourceTable;
import de.superx.etl.biad.SourcesYML;
import de.superx.job.ActionNode;
import de.superx.job.ContainerNode;
import de.superx.job.ContainerNode.JobSource;
import de.superx.job.ContainerNode.JobType;
import de.superx.job.ExecuteActionGroupAction;
import de.superx.job.ExecuteDbtAction;
import de.superx.job.ExecuteSQLFluffAction;
import de.superx.job.ExtractAction;
import de.superx.job.KettleAction;
import de.superx.job.LoadAction;
import de.superx.job.LoadMondrianSchemaAction;
import de.superx.job.RemoveSpringBatchHistoryAction;
import de.superx.job.SqlAction;
import de.superx.job.SqlActionNoSxPool;
import de.superx.job.SqlScriptAction;
import de.superx.job.Timestamp;
import de.superx.job.TransferAction;
import de.superx.job.UnloadAction;
import de.superx.job.UnloadParams;
import de.superx.job.UpdateMetadataAction;
import de.superx.job.UpgradeDbFormsConfigAction;
import de.superx.rest.model.job.Component;
import de.superx.util.PathAndFileUtils;
import de.superx.util.XmlUtil;

@Service
public class EntityJobDescriptionSource implements JobDescriptionSource {

    public static final String BIAD_COMPONENT_ABBREVIATION = "biad";
    public static final int SYSTEMINFO_ID_BIAD = 440;
    public static final int SYSTEMINFO_ID_ASTAT = 280;
    public static final int SYSTEMINFO_ID_SOS = 7;
    static Logger logger = LoggerFactory.getLogger(EntityJobDescriptionSource.class);
    
    @Autowired
    DbtWrapper dbtWrapper;

    @Override
    public List<ContainerNode> readJobs() {
        List<ContainerNode> jobs = new ArrayList<>();
        for (String componentAbbreviation : PathAndFileUtils.getModules()) {
            jobs.addAll(getJobsForComponent(componentAbbreviation));
        }
        return jobs;
    }
    
    @Override
    public Component getComponentWithLoadAndUnloadDetails(String abbreviation) {
        Component component = new Component(abbreviation);
        
        boolean hasLoad = false;
        Document document = PathAndFileUtils.readXml(abbreviation, true);
        if(document != null) {
            NodeList etlSteps = document.getElementsByTagName("etl-step");
            for (int i = 0; i < etlSteps.getLength(); i++) {
                Node etlStep = etlSteps.item(i);
                String type = getAttribute(etlStep.getAttributes(), "type"); 
                if(type.equals("load")) {
                    hasLoad = true;
                    break;
                }
            }
        }
        
        boolean hasUnload = false;
        Document unloadXml = PathAndFileUtils.readXml(abbreviation, false);
        if(unloadXml != null) {
            NodeList unloadJobs = unloadXml.getElementsByTagName("unload-job");
            if(unloadJobs != null && unloadJobs.getLength() > 0) {
                hasUnload = true; 
            }
        }

        component.setLoad(hasLoad);
        component.setUnload(hasUnload);
        return component;
    }
    
    @Override
    public ContainerNode getUnloadJob(String componentAbbreviation) {

        // use sources.yml for bad component
        if(componentAbbreviation.equals(BIAD_COMPONENT_ABBREVIATION)) {
            ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Unload", JobType.unload.createName(componentAbbreviation), Integer.valueOf(SYSTEMINFO_ID_BIAD));
            File file = new File(PathAndFileUtils.getSourcesYML("hisinone"));
            // make startup more robust: don't fail if sources.yml is not found #303809
            if(!file.exists()) {
                logger.error("No sources.yml for " + componentAbbreviation);
            } else {
                SourcesYML sourcesYML = SourcesYML.fromJson(file);
                for (DBSource source : sourcesYML.getSources()) {
                    if(!source.getName().startsWith("staging")) {
                       continue; 
                    }
                    for (SourceTable table : source.getTables()) {
                        String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName());
                        ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getName(), source.getSrcDatabase());
                        root.add(action);
                    }
                }
            }
            return root;
        }

        File xmlFile = getUnloadXmlFile(componentAbbreviation);
        if (! xmlFile.exists()) {
            logger.warn("No unload.xml for " + componentAbbreviation);
            return null;
        }
        Document unloadXml = null;
        try {
            unloadXml = readXmlFile(xmlFile);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        String defaultSourcesystem = getDefaultSourcesystem(unloadXml);
        Integer systemInfoId = getSystemInfoId(unloadXml);
        ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Unload", JobType.unload.createName(componentAbbreviation), systemInfoId);
        NodeList unloadJobs = unloadXml.getElementsByTagName("unload-job");
        int unloadJobCount = unloadJobs.getLength();
        if (unloadJobCount == 0) {
            return null;
        }
        for (int i = 0; i < unloadJobCount; i++) {
            Node unloadJob = unloadJobs.item(i);
            root.addAll(createUnloadStep(unloadJob, systemInfoId, componentAbbreviation));
        }
        root.getActions().add(0, new de.superx.job.UnloadParams(systemInfoId));
        return root;
    }
    
    public List<ActionNode> createUnloadStep(Node unloadStep, Integer systemInfoId, String componentAbbreviation) {

        List<ActionNode> result = new ArrayList<>();
        String id = getAttribute(unloadStep.getAttributes(), "id");
        String unloadFile = getAttribute(unloadStep.getAttributes(), "transfer-unload-file");
        String commentString = null;
        logger.info(id + " (" + (!unloadFile.isEmpty()) + ")");
        List<Node> sqlNodes = filterElements(unloadStep.getChildNodes());
        Optional<Node> comment = sqlNodes.stream()
            .filter(node -> node.getNodeName().equalsIgnoreCase("comment") || node.getNodeName().equalsIgnoreCase("description"))
            .findFirst();
        if (comment.isPresent()) {
            commentString = comment.get().getTextContent();
            sqlNodes.remove(comment.get());
        }
        sqlNodes.forEach(sqlNode -> {
            String sourcesystem = getAttribute(sqlNode.getAttributes(), "sourcesystem");
            String database = getAttribute(sqlNode.getAttributes(), "database").toLowerCase();
            String sqlvarsourcesystem = getAttribute(sqlNode.getAttributes(), "sqlVarDatabase").toLowerCase();
            String version = getAttribute(sqlNode.getAttributes(), "version");
            String sql = sqlNode.getTextContent();
            if (sql.indexOf("<sqlvars>") > -1) {
                sql = StringUtils.removeSqlVars(sql);
            }
            if (unloadFile.isEmpty()) {
                result.add(new SqlAction(id, systemInfoId, database, sourcesystem, sqlvarsourcesystem, version, sql));
            } else {
                result.add(new UnloadAction(id, sql,
                            unloadFile.startsWith("$") ?
                                            unloadFile
                            : "$" + componentAbbreviation.toUpperCase() + "_LOAD_PATH/"
                           + unloadFile,
                            sourcesystem, sqlvarsourcesystem, systemInfoId, version, database));
            }
        });
        return result;
    }

    @Override
    public ContainerNode getLoadAndTransformJob(String componentAbbreviation) {

        File xmlFile = getXmlFile(componentAbbreviation);
        Document document = null;
        try {
            document = readXmlFile(xmlFile);
        } catch (IOException e) {
            logger.error("Couldn't read " + xmlFile.getAbsolutePath(), e);
            return null;
        }
        NodeList etlSteps = document.getElementsByTagName("etl-step");
        Integer systemInfoId = getSystemInfoId(document);
        ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Load And Transform", 
                                               JobType.load_transform.createName(componentAbbreviation), systemInfoId);
        for (int i = 0; i < etlSteps.getLength(); i++) {
            Node etlStep = etlSteps.item(i);
            String name = getAttribute(etlStep.getAttributes(), "name");
            String type = getAttribute(etlStep.getAttributes(), "type"); 
            String id = getAttribute(etlStep.getAttributes(), "id");
            if ("manual".equals(type) ) {
                logger.info("etl-step " + name + " type manual. Skipping.");
                continue;
            }
            if (name.isEmpty()) {
                name = type;
            }
            logger.info("etl-step: " + name);
            ContainerNode container = createContainerNode(name, id, systemInfoId);
            List<ActionNode> stepList = createActions(filterElements(etlStep.getChildNodes()), systemInfoId, componentAbbreviation); 
            if (stepList.size() > 0) {
                container.addAll(stepList);
            }
            if (container.name.startsWith("Upload")) {
                container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_load_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation)));
                container.add(new SqlScriptAction("postgres", systemInfoId, "$" + componentAbbreviation.toUpperCase() + "_PFAD/preparation.sql"));
            }
            if (container.name.startsWith("Transformation")) {
                container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_trans_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation)));
                container.add(new SqlScriptAction("postgres", systemInfoId, "$" + componentAbbreviation.toUpperCase() + "_PFAD/finalize.sql"));
            }
            if (container.name.startsWith("Aggregation")) {
                container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_aggr_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation)));
            }
            // Keine leeren Container hinzufügen
            if (container.getActions().size() > 0) {
                root.add(container);
            }
        }
        int childrenCount = root.size();
        if (childrenCount > 0) {
            ContainerNode timestampContainer = new ContainerNode("Timestamp", "Timestamp", systemInfoId);
            timestampContainer.add(new Timestamp(root.systemInfoId));
            if (root.systemInfoId.intValue() == SYSTEMINFO_ID_SOS) {
                // for Studierende, Prüfungen HLR we also set the 
                // timestamp for Astat, yes, ugly :(
                timestampContainer.add(new Timestamp(Integer.valueOf(SYSTEMINFO_ID_ASTAT)));
            }
            root.add(timestampContainer);
        } else {
            return null;
        }
        return root;
    }
    
    @Override
    public ContainerNode getCompleteJob(String componentAbbreviation) {
        ContainerNode loadAndTransform = getLoadAndTransformJob(componentAbbreviation);
        ContainerNode unload = getUnloadJob(componentAbbreviation); 
        if (loadAndTransform == null || unload == null) {
            return null;
        }
        ContainerNode complete = createContainerNode(componentAbbreviation.toUpperCase() + 
                                                       " Unload, Load And Transform", JobType.complete.createName(componentAbbreviation), loadAndTransform.systemInfoId);
        complete.add(unload);
        complete.add(loadAndTransform);
        return complete;
    }

    @Override
    public ContainerNode getJobById(String id) {

        ContainerNode root = null;
        String componentAbbreviation = JobType.getComponentAbbreviationFromJobName(id);
        JobType type = JobType.getJobTypeForName(id);

        switch (type) {
        case install_upgrade:
           root = getInstallUpgradeJob(componentAbbreviation); 
            break;
        case uninstall:
           root =  "kern".equals(componentAbbreviation) ? null : getUninstallJob(componentAbbreviation); 
            break;
        case unload:
           root =  getUnloadJob(componentAbbreviation);
            break;
        case load_transform:
           root =  getLoadAndTransformJob(componentAbbreviation);
            break;
        case complete:
            root = getCompleteJob(componentAbbreviation); 
            break;
        case special:
            int first = id.indexOf('_');
            int last = id.lastIndexOf('_');
            root = getSpecialJob(componentAbbreviation, id.substring(first + 1, last)); 
            break;
        case unknown:
            break;
        default:
            // maybe a component abbreviation e.g. 'res' was passed,
            // in this case return the complete job
            root = getCompleteJob(id);
            break;
        }
        return root;
    }

    @Override
    public List<ContainerNode> getSpecialJobs(String componentAbbreviation) {
        Document modulDoc = readModuleXml(componentAbbreviation);
        
        List<ContainerNode> result;
        
        try {
            NodeList nodes = modulDoc.getElementsByTagName("etl-step");
            result = convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), getSystemInfoId(modulDoc));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
	    // add a UnloadParams-Action, so that Freemarkervariablen with unload_params
	    // are always available in 'manual' jobs
	    for (ContainerNode ulr : result) {
            ulr.getActions().add(0, new UnloadParams(ulr.systemInfoId));
	    }

        return result;
    }
    
    protected ContainerNode getSpecialJob(String componentAbbreviation, String jobId) {
        Document modulDoc = readModuleXml(componentAbbreviation);
        ContainerNode result = null;

        List<Node> etlSteps = XmlUtil.fromNodeList(modulDoc.getElementsByTagName("etl-step"));
        for (Node etlStep : etlSteps) {
            String id = getAttribute(etlStep.getAttributes(), "id");
            if (! jobId.equals(id)) {
                logger.info(id + " id != " + jobId);
                continue;
            }
            Integer systemInfoId = getSystemInfoId(modulDoc);
            List<ContainerNode> l = convertToContainers(componentAbbreviation, List.of(etlStep), systemInfoId);
            result = l.get(0);
            if (getUnloadXmlFile(componentAbbreviation).exists()) {
                result.getActions().add(0, new de.superx.job.UnloadParams(systemInfoId));
            }
            break;
        }
        return result;
    }

    public List<ContainerNode> convertToContainers(String componentAbbreviation, List<Node> etlSteps, Integer systemInfoId) {
        List<ContainerNode> result = new ArrayList<>();
        for (Node etlStep : etlSteps) {
            String id = getAttribute(etlStep.getAttributes(), "id");
            String name = getAttribute(etlStep.getAttributes(), "name");
            String type = getAttribute(etlStep.getAttributes(), "type"); 
            String active = getAttribute(etlStep.getAttributes(), "active");
            String visible = getAttribute(etlStep.getAttributes(), "visible"); 
            String generatedStr = getAttribute(etlStep.getAttributes(), "generateExecutable");
            boolean generated = generatedStr.isEmpty() || "true".equalsIgnoreCase(generatedStr);
            if ("manual".equalsIgnoreCase(type) 
                            && "true".equalsIgnoreCase(active) 
                            && "true".equalsIgnoreCase(visible)) { 
                if (generated) {
                    List<Node> childNodes = filterElements(etlStep.getChildNodes());
                    List<Node> desc = childNodes.stream().filter(el -> el.getNodeName().equals("description")).collect(Collectors.toList());
                    ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " " + name, JobType.special.createName(componentAbbreviation, id), systemInfoId);
                    if (desc.size() > 0) {
                        root.setDescription(desc.get(0).getTextContent().trim());
                    }
                    List<ActionNode> stepList = createActions(childNodes, systemInfoId, componentAbbreviation); 
                    if (stepList.size() > 0) {
                        root.addAll(stepList);
                    }
                    if (stepList.size() == 0 || !stepList.get(stepList.size() - 1).id.contains("_custom")) {
                        root.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_man_%s_custom.sql",
                                                                                           componentAbbreviation.toUpperCase(),
                                                                                           componentAbbreviation,
                                                                                           id)));
                    }   
                    result.add(root);
                } else {
                    File dbiFile = getDbiFileForManualEtlStep(componentAbbreviation, id, "pg");
                    File xslFile = getXslFile("dbi_ulr.xsl");
                    Map<String, Object> parameters = Map.ofEntries(
                                        Map.entry("id", id),
                                        Map.entry("name", name),
                                        Map.entry("systemInfoId", systemInfoId)
                                    );
                        Document doc = XmlUtil.xmlTransformWithXsl(new StreamSource(dbiFile), new StreamSource(xslFile), parameters);
                        NodeList nodes = doc.getElementsByTagName("etl-step");
                        result.addAll(convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), systemInfoId));
                }
            }
        }
        return result;
    }
    
    @Override
    public ContainerNode getInstallUpgradeJob(String componentAbbreviation) {
        
        File xmlFile;
        Document document;
        try {
        xmlFile = getXmlFile(componentAbbreviation);
        if(xmlFile.canRead()) {
            document = readXmlFile(xmlFile);
        }else {
            return null;
        }
        }catch(Exception e) {
            logger.info("Could not get xml file for "+ componentAbbreviation+" module. Skipping!");
            return null;
        }
        
        NodeList installSteps = document.getElementsByTagName("install-upgrade-step");
        int stepCount = installSteps.getLength();
        
        if (stepCount == 0) {
            logger.warn("No install-upgrade-steps found for component " + componentAbbreviation + " Please check " + componentAbbreviation + ".xml!");
            return null;
        }
        
        Integer systemInfoId = getSystemInfoId(document);
        
        // create containers
        ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Install/ Upgrade", 
                                               JobType.install_upgrade.createName(componentAbbreviation), systemInfoId);
        ContainerNode upgradeSchema = createContainerNode(componentAbbreviation.toUpperCase() + " Upgrade DB Schema", 
                                                        componentAbbreviation + "_upgrade_db_schema", systemInfoId);
        ContainerNode updateMetadata = createContainerNode(componentAbbreviation.toUpperCase() + "Update Module Metadata", componentAbbreviation + "_update_module_metadata", systemInfoId);
        ContainerNode scriptStepsContainer = createContainerNode(componentAbbreviation.toUpperCase() + " Upgrade-Scripts", 
                                                               componentAbbreviation + "_upgrade_scripts", systemInfoId);
        logger.debug("In module: "+componentAbbreviation+ " there were: " + stepCount+ " Install/ Upgrade Steps found!");
        List<ActionNode> scriptSteps = new ArrayList<>();
        for(int i = 0; i < stepCount; i++) {
            Node installStep = installSteps.item(i);
            String name = getAttribute(installStep.getAttributes(), "name");
            String type = getAttribute(installStep.getAttributes(), "type");
            String id = getAttribute(installStep.getAttributes(), "id");
            if("container".equals(type)) {
                try {
                    ContainerNode container = fillInstallUpgradeContainers(componentAbbreviation, name, id, systemInfoId, installStep);
                    if (container.size() == 0) {
                        // leere Container überspringen
                        continue;
                    }
                    if (container.id.contains("update_module_metadata")) {
                        updateMetadata = container;
                    } else {
                        root.add(container);
                    }
                } catch (IOException e) {
                    logger.error("couldn't read install/ upgrade job for module: "+componentAbbreviation);
                    e.printStackTrace();
                }
                continue;
            }
            List<ActionNode> stepList = createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation);
            for (ActionNode node : stepList) {
                if (node.id.endsWith("upgrade_tables_pg.sql") || node.id.endsWith("upgrade_views_pg.sql")
                                || node.id.endsWith("upgrade_fields_pg.sql")
                                || node.id.endsWith("upgrade_indexes_pg.sql")
                                || node.id.endsWith("sx_tables_fuellen.sql")
                                ) {
                    upgradeSchema.add(node);
                } else {
                    scriptSteps.add(node);
                }
            }
        }
        if (root.size() == 0) {
            logger.warn("No pre_upgrade container found in install-upgrade-steps for component " + componentAbbreviation + " Please check " + componentAbbreviation + ".xml!");
            return null;
        }
        // nach pre_upgrade hinzufügen
        root.add(1, upgradeSchema);
        scriptStepsContainer.addAll(scriptSteps);
        installSteps = document.getElementsByTagName("upgrade-step");
        if (installSteps.getLength() == 1) {
            List<ActionNode> stepList = createActions(filterElements(installSteps.item(0).getChildNodes()), systemInfoId, componentAbbreviation);
            if (stepList.size() > 0) {
                scriptStepsContainer.addAll(stepList);
            }
        }
        scriptStepsContainer.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_install_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation)));
        root.add(scriptStepsContainer);
        updateMetadata.addAll(additionalInstallActions(componentAbbreviation, systemInfoId));
        root.add(updateMetadata);
        root.add(new UpgradeDbFormsConfigAction(systemInfoId, true, componentAbbreviation));
        root.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/conf/customize.sql", componentAbbreviation.toUpperCase())));
        root.add(new SqlAction("Update DB version", systemInfoId, "postgres", "eduetl", null, 
                               "insert into db_version (his_system, version, version_hash, systeminfo_id) values ('" + componentAbbreviation + "',"
                                               + " '${version}', '${version_hash}', " + systemInfoId + ")\n"
                                               + "on conflict (his_system) do\n"
                               + "update set version='${version}', version_hash='${version_hash}'"));
        
        return stepCount != 0 ? root : null;
    }
    
    @Override
    public ContainerNode getUninstallJob(String componentAbbreviation) {
        if ("kern".equalsIgnoreCase(componentAbbreviation)) {
            return null;
        }
        Integer systemInfoId = Integer.valueOf(-1);
        try {
            systemInfoId = Integer.valueOf(getSystemInfoId(componentAbbreviation));
        } catch (Exception e) {
            logger.error("Couldn't get systemInfoId for " +componentAbbreviation, e);
        }
        File uninstallFile = new File(String.format("%s/conf/his1/edustore_install/edustore_%s_drop_pg.xml", 
                                        getModulePath(componentAbbreviation), componentAbbreviation));
        File xslFile = getXslFile("dbi_ulr.xsl");
        Map<String, Object> parameters = Map.ofEntries(
                                                       Map.entry("id", "uninstall"),
                                                       Map.entry("name", componentAbbreviation + "_uninstall"),
                                                       Map.entry("systemInfoId", systemInfoId)
                        );
        Document doc = XmlUtil.xmlTransformWithXsl(new StreamSource(uninstallFile), new StreamSource(xslFile), parameters);
		ContainerNode uninstall = new ContainerNode("Uninstall_" + componentAbbreviation.toLowerCase(), JobType.uninstall.createName(componentAbbreviation.toLowerCase()), systemInfoId);
		NodeList nodes = doc.getElementsByTagName("etl-step");
        List<ContainerNode> converted = convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), systemInfoId);
        uninstall.addAll(converted.get(0).getActions());
        uninstall.name = componentAbbreviation.toUpperCase() + " Uninstall";
        uninstall.addAll(additionalInstallActions(componentAbbreviation, systemInfoId));
        return uninstall;
    }
    
    private static List<ActionNode> additionalInstallActions(String componentAbbreviation, Integer systemInfoId) {
        List<ActionNode> result = new ArrayList<>();
        //result.add(new LoadEtlRoutinesAction("load_etl_routines", null, systemInfoId, true,
        //                                     false,  componentAbbreviation));
        result.add(new LoadMondrianSchemaAction("load_mondrian_schema", null, systemInfoId, true,
                                                Integer.valueOf(0), componentAbbreviation));
        return result;
    }
    
    private ContainerNode fillInstallUpgradeContainers(String componentAbbreviation, String containerName, String containerId, Integer systemInfoId, Node installStep) throws IOException {
        ContainerNode container = createContainerNode(componentAbbreviation.toUpperCase() + " " + containerName, containerId, systemInfoId);
    
        if("pre_upgrade".equals(containerName)) {
            container.addAll(checkForPreUpgrade(componentAbbreviation, systemInfoId));
        }else if("install_functions".equals(containerName)) {
            container.addAll(installFunctions(componentAbbreviation, systemInfoId));
        }else if("install_masks".equals(containerName)) {
            container.addAll(installMasks(componentAbbreviation, systemInfoId));
        } else if("Update Module Metadata".equals(containerName)) {
            List<ActionNode> actionNodes = metadaten(componentAbbreviation, systemInfoId, installStep);
            container.addAll(actionNodes);    
        } else {
            // generic container
            container.addAll(createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation));
        }
        
        return container;
    }
    
    private List<ActionNode> metadaten(String componentAbbreviation, Integer systemInfoId, Node installStep) {
        return createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation);
    }
    
    private List<ActionNode> installMasks(String componentAbbreviation, Integer systemInfoId) throws IOException {
        List<ActionNode> actions = new ArrayList<>();
        File xmlFile = getXmlFile(componentAbbreviation);
        Document document = readXmlFile(xmlFile);

        NodeList masks = document.getElementsByTagName("maske");
        int maskCount = masks.getLength();
        for(int i = 0; i < maskCount; i++) {
            Node mask = masks.item(i);
            String id = getAttribute(mask.getAttributes(), "tid");
            String name = getAttribute(mask.getAttributes(), "name");
            String path = null;

            NodeList srcTags = ((Element) mask).getElementsByTagName("src");
            for (int x = 0; x < srcTags.getLength(); x++) {
               Element e = (Element) srcTags.item(x);

               String pathText = e.getElementsByTagName("path").item(0).getTextContent();
               String dbSystemText = null; // null means no dbsystem tag -> POSTGRES by default

               NodeList dbSystemNode = e.getElementsByTagName("dbsystem");
               if(dbSystemNode.getLength() > 0) {
                   dbSystemText = dbSystemNode.item(0).getTextContent();
               }
               if(dbSystemText == null || dbSystemText.isBlank() || "POSTGRES".equals(dbSystemText)) {
                  path =  pathText;
               }
            }

            ContainerNode container = createContainerNode("Install mask: (" + id + ") " + name, "install_mask_tid_"+id, systemInfoId);

            container.add(new SqlAction("sql_del_maskeninfo_"+id, systemInfoId, "postgres", "eduetl", null, String.format(
                    "delete from maskeninfo where tid = %s;",id)));
            container.add(new SqlAction("sql_del_felderinfo_"+id, systemInfoId, "postgres", "eduetl", null, String.format(
                    "delete from felderinfo where tid in (select felderinfo_id from masken_felder_bez where maskeninfo_id = %s);",id)));
            container.add(new SqlAction("sql_del_masken_felder_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format(
                    "delete from masken_felder_bez where maskeninfo_id = %s;",id)));
            container.add(new SqlAction("sql_del_sachgeb_maske_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format(
                    "delete from sachgeb_maske_bez where maskeninfo_id = %s;",id)));
            container.add(new SqlAction("sql_del_maske_sys_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format(
                    "delete from maske_system_bez where maskeninfo_id = %s;",id)));

            String filePath = path + "/" + id + "_";
            container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "maskeninfo", filePath + "maskeninfo.unl"));
            container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "felderinfo", filePath + "felderinfo.unl"));
            container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "masken_felder_bez", filePath + "masken_felder_bez.unl"));
            container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "sachgeb_maske_bez", filePath + "sachgeb_maske_bez.unl"));
            container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "maske_system_bez", filePath + "maske_system_bez.unl"));

            actions.add(container);
        }
        return actions;
    }
    
    private static ContainerNode createContainerNode(String name, String id, Integer systemInfoId) {
        ContainerNode node = new ContainerNode(name, id, systemInfoId);
        node.setJobSource(JobSource.FILE);
        return node;
    }

    private List<ActionNode> installFunctions(String componentAbbreviation, Integer systemInfoId) throws IOException {
        File xmlFile = getXmlFile(componentAbbreviation);
        Document document = readXmlFile(xmlFile);
        List<ActionNode> functionActions = new ArrayList<>();
        
        NodeList functions = document.getElementsByTagName("function");
        int functionsCount = functions.getLength();
        for(int i = 0; i < functionsCount; i++) {
            Element function = (Element)functions.item(i);
            NodeList srcTags = function.getElementsByTagName("src");
            int srcTagsCount = srcTags.getLength();
            for(int k = 0; k < srcTagsCount; k++) {
                Element src = (Element)srcTags.item(k);
                //String file = src.getElementsByTagName("file").item(0).getTextContent();
                NodeList file = src.getElementsByTagName("file");
                NodeList body = src.getElementsByTagName("body");
                
                String dbsystem = src.getElementsByTagName("dbsystem").item(0).getTextContent();
                
                if(file.getLength() > 0) {
                    functionActions.add(installFunctionFileTag(file, dbsystem, systemInfoId));
                }else if(body.getLength() > 0) {
                    functionActions.add(installFunctionBodyTag(body, dbsystem, systemInfoId));
                }
            }
        }
        
        return functionActions;
    }

    private static ActionNode installFunctionBodyTag(NodeList body, String dbsystem, Integer systemInfoId) {
        
        Node cdata = body.item(0).getFirstChild();
        if(cdata instanceof CharacterData) {
            CharacterData cd = (CharacterData) cdata;
            return new SqlAction(cd.getData().toLowerCase().substring(0, 10).replace(" ", "_")+"_function_from_"+systemInfoId.toString(),
                    systemInfoId, dbsystem, "eduetl", null, cd.getData());
        }
        
        return null;
    }

    private static ActionNode installFunctionFileTag(NodeList file, String dbsystem, Integer systemInfoId) {
        String f = file.item(0).getTextContent();
        
        if(dbsystem.equals("INFORMIX")) {
            return new SqlScriptAction("informix", systemInfoId, f);
        }
        
        return new SqlScriptAction("postgres", systemInfoId, f);
    }

    private static List<ActionNode> checkForPreUpgrade(String componentAbbreviation, Integer systemInfoId) throws IOException {
        List<ActionNode> actions = new ArrayList<>();
        
        String moduleUpgradeDirPath = getModulePath(componentAbbreviation)+"/upgrade";
        File moduleUpgradeDir = new File(moduleUpgradeDirPath);
        File[] preUpgradeFiles = moduleUpgradeDir.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                String filename = name.strip().toLowerCase();
                if(filename.contains("pre_upgrade_pg") && filename.endsWith("sql") && !filename.contains("tmp")) {
                    return true;
                }
                return false;
            }
        });
        
        for(int i = 0; i < preUpgradeFiles.length; i++) {
            actions.add(new SqlScriptAction("postgres", systemInfoId, String.format(
                    "$%s_PFAD/upgrade/%s", componentAbbreviation.toUpperCase(), preUpgradeFiles[i].getName())));
        }
        
        return actions;
    }

    private List<ActionNode> createActions(List<Node> actionNodes, Integer systemInfoId, String componentAbbreviation) {
        List<ActionNode> result = new ArrayList<>();
        actionNodes.forEach(node -> {
            logger.info("action :");
            List<Node> actions = filterElements(node.getChildNodes());
            for (Node action : actions) {
                String name = action.getNodeName();
                NamedNodeMap attributes = action.getAttributes();
                if ("nativeaction".equals(name)) {
                    String database = getAttribute(attributes, "database");
                    String container = getAttribute(attributes, "container");
                    ActionNode actionStep = createActionStep(attributes, systemInfoId);
                    if (actionStep != null) {
                        if (container != null && container.equalsIgnoreCase("true")) {
                            // TODO: Container sollten generischer gelöst werden! 
                            // Das ist hier ein Workaround, damit man die Ausführung einer
                            // Action in einen Container packen kann. Damit wird ein Neueinlesen
                            // der Freemarkervariablen getriggert.
                            // Z.B. notwendig für sos install, damit sos_modul_upgrade_man.sql
                            // auf die Variablen im davor ausgeführten Schritt zugreifen kann.
                            ContainerNode containerNode = createContainerNode( name + " Container", actionStep.id + "_container", systemInfoId);
                            containerNode.add(actionStep);
                            result.add(containerNode);
                        } else {
                            result.add(actionStep);
                        }
                    }
                } else if ("loadtable".equals(name)) {
                    ActionNode loadStep = createLoadStep(action, attributes, systemInfoId);
                    result.add(loadStep);
                } else if ("transferTable".equals(name)) {
                    ActionNode transferStep = createTransferStep(attributes, systemInfoId);
                    result.add(transferStep);
                } else if ("kettle-job-embedded".equals(name)) {
                    ActionNode kettleStep = createKettleStep(attributes, systemInfoId);
                    result.add(kettleStep);
                } else if ("unload-job".equals(name)) {
                    result.addAll(createUnloadStep(action, systemInfoId, componentAbbreviation));
                } else if("execute-unload-scripts".equals(name)) {
                    result.addAll(createLoadAfterExtract(componentAbbreviation, systemInfoId));
                } else if("biad_unload_and_load".equals(name)) {
                    result.add(getUnloadJob(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION));
                    List<ActionNode> nodes = createLoadAfterExtract(componentAbbreviation, Integer.valueOf(SYSTEMINFO_ID_BIAD));
                    result.addAll(nodes);
                } else if("evaluate-dbt-json".equals(name)) {
                    String noGroupActionAttr = getAttribute(attributes, "noGroupAction");
                    boolean groupAction = !Boolean.parseBoolean(noGroupActionAttr);
                    result.addAll(createDbtExecutionStep(attributes, systemInfoId));
                    result.add(createMetaimportJsonStep(attributes, systemInfoId));
                    if (groupAction) {
                        result.add(new ExecuteActionGroupAction(systemInfoId));
                    }
                } else if("evaluate-metadata".equals(name)) {
                    result.add(createMetaimportJsonStep(attributes, systemInfoId));
                }  else if("evaluate-dbt-test".equals(name)) {
                    result.addAll(createDbtTestStep(attributes, systemInfoId));
                } else if("remove-spring-batch-history".equals(name)) {
                    result.add(new RemoveSpringBatchHistoryAction(systemInfoId));
                } else if ("loadtableinfo".equals(name)) {
                    // this exists in cob.xml but is obviously not used
                    // so skipping here!
                } else {
                    logger.error("Unknown action type: " + name);
//                    throw new RuntimeException("Unknown action type: " + name);
                }
            }
        });
        return result;
    }
    
    private List<ActionNode> createDbtTestStep(NamedNodeMap attributes, Integer systemInfoId) {
        List<ActionNode> nodes = new ArrayList<>();
        String dbtProject = getAttribute(attributes, "project");
        String dbtSelect = getAttribute(attributes, "select");
        nodes.add(new ExecuteDbtAction(dbtProject, dbtSelect, systemInfoId, true));
        nodes.add(new ExecuteSQLFluffAction(dbtProject, "Lint DBT Models using SQLFluff", "sqlfluff_action", Integer.valueOf(SYSTEMINFO_ID_BIAD), true, Integer.valueOf(0)));
        return nodes;
    }

    private List<ActionNode> createDbtExecutionStep(NamedNodeMap attributes, Integer systemInfoId) {
        List<ActionNode> nodes = new ArrayList<>();
        String dbtProject = getAttribute(attributes, "project");
        String dbtSelect = getAttribute(attributes, "select");
        String sourceSystem = getAttribute(attributes, "sourcesystem");
        if(sourceSystem == null || sourceSystem.isBlank()) {
           sourceSystem = "hisinone";
        }

        if(dbtSelect != null && !dbtSelect.isBlank()) {
            String id = getAttribute(attributes, "id");
            ContainerNode extractLoad = createContainerNode("Extract and Load", "extract_load_" + id, systemInfoId);

            DbtRunRequest request = new DbtRunRequest(DbtAction.STAGING_TABLES, dbtProject, dbtSelect);
            DbtRunResult result = dbtWrapper.execute(request);
            if(!result.getState().equals(DbtExecutionState.SUCCESS)) {
                logger.error("Could not get specific Staging Tables.", String.join("\n", result.getErrorMessages()));
            }
            List<String> stagingTables = result.getOutputMessages();

            if(stagingTables != null && !stagingTables.isEmpty()) {
                extractLoad.addAll(createExtractAndLoadStep(stagingTables, systemInfoId, sourceSystem));
                nodes.add(extractLoad);
            }
        }

        nodes.add(new ExecuteDbtAction(dbtProject, dbtSelect, systemInfoId));
        return nodes;
    }

    private static ActionNode createMetaimportJsonStep(NamedNodeMap attributes, Integer systemInfoId) {
        String dbtProject = getAttribute(attributes, "project");
        return new UpdateMetadataAction(dbtProject, systemInfoId);
    }

    private static List<ActionNode> createExtractAndLoadStep(List<String> tables, Integer systemInfoId, String sourceSystem) {
        List<ActionNode> result = new ArrayList<>();
        SourcesYML sourcesYML = SourcesYML.fromJson(new File(PathAndFileUtils.getSourcesYML("hisinone")));
        List<DBSource> sources = sourcesYML.getSources();
        for (DBSource source : sources) {
            if(!source.getName().startsWith("staging")) {
               continue; 
            }
            for (SourceTable table : source.getTables()) {
                if(!tables.contains(table.getName())) {
                   continue; 
                }

                String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName());

                ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getSchema(), sourceSystem);
                result.add(action);
                String unloadPath = PathAndFileUtils.getUnloadPath("biad")
	                + File.separator + "tables" + File.separator + table.getName() + ".sql";

                SqlScriptAction createTableAction = new SqlScriptAction("postgres", systemInfoId, unloadPath);
                result.add(createTableAction);
                
                LoadAction loadAction = new LoadAction(systemInfoId, false, ",", false,
                        "eduetl", source.getSchema() + "." + table.getName(), unlFile, true);
                result.add(loadAction);
                logger.info("\tloadtable: tabname=" + action.sourceTable + " path=" + action.unlFile);
            }
        }
        return result;
    }

    private List<ActionNode> createLoadAfterExtract(String componentAbbreviation, Integer systemInfoId) {
        List<ActionNode> result = new ArrayList<>();
        SourcesYML sourcesYML = SourcesYML.fromJson(new File(PathAndFileUtils.getSourcesYML("hisinone")));
        for (DBSource source : sourcesYML.getSources()) {
            if(!source.getName().startsWith("staging")) {
               continue; 
            }
            for (SourceTable table : source.getTables()) {
                String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName());
                ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getSchema(), source.getSrcDatabase());

                String unloadPath = PathAndFileUtils.getUnloadPath("biad")
	                + File.separator + "tables" + File.separator + table.getName() + ".sql";
                SqlScriptAction createTableAction = new SqlScriptAction("postgres", systemInfoId, unloadPath);

                LoadAction loadAction = new LoadAction(systemInfoId, false, ",", false,
                        "eduetl", source.getSchema() + "." + table.getName(), unlFile, true);

                result.add(createTableAction);
                result.add(loadAction);
                logger.info("\tloadtable: tabname=" + action.sourceTable + " path=" + action.unlFile);
            }
        }
        return result;
    }

    private ActionNode createKettleStep(NamedNodeMap attributes, Integer systemInfoId) {
        String name = getAttribute(attributes, "name");
        String file = getAttribute(attributes, "file");
        logger.info("\tcreating kettle job " + name + " in path " + file);
        KettleAction kettleAction = new KettleAction(name, systemInfoId, file);
        return kettleAction;
    }

    private ActionNode createTransferStep(NamedNodeMap attributes, Integer systemInfoId) {
        String fromTable = getAttribute(attributes, "src");
        String toTable = getAttribute(attributes, "target");
        logger.info("\ttransfer table " + fromTable + " to " + toTable);
        TransferAction transferAction = new TransferAction(systemInfoId, fromTable, toTable);
        return transferAction;
    }

    private ActionNode createLoadStep(Node action, NamedNodeMap attributes, Integer systemInfoId) {
        String refresh = getAttribute(attributes, "refresh");
        String checkEmpty = getAttribute(attributes, "checkEmpty");
        String tabName = getAttribute(attributes, "tabname");
        String delimiter = getAttribute(attributes, "delimiter");
        String header = getAttribute(attributes, "header");
        String database = getAttribute(attributes, "database");
        Node fileNode = filterElements(action.getChildNodes()).get(0);
        String unloadFileAttribute = getAttribute(fileNode.getAttributes(), "path");
        logger.info("\tloadtable: tabname=" + tabName + " path=" + unloadFileAttribute);
        LoadAction loadAction = new LoadAction(
                systemInfoId,
                Boolean.parseBoolean(refresh),
                delimiter,
                Boolean.parseBoolean(header),
                database.trim().isEmpty() ? "eduetl" : database ,
                tabName,
                unloadFileAttribute
        );
        if (!checkEmpty.isEmpty()) {
            loadAction.checkEmpty = Boolean.parseBoolean(checkEmpty);
        }
        return loadAction;
    }

    private ActionNode createActionStep(NamedNodeMap attributes, Integer systemInfoId) {
        ActionNode step = null;
        String sql = getAttribute(attributes, "sql");
        String scriptfile = getAttribute(attributes, "scriptfile");
        String dataSource = getAttribute(attributes, "dataSource");
        String databaseSystem = getAttribute(attributes, "database");
        if (dataSource.isEmpty()) {
            dataSource = "eduetl";
        }
        String sqlVardataSource = getAttribute(attributes, "sqlVarDatabase");
        if (sqlVardataSource.isEmpty()) {
            sqlVardataSource = "eduetl";
        }
        if (!sql.isEmpty()) {
            logger.info("\tnativeaction: " + sql);
            String[] parts = sql.split(":", 2);
            String database = "eduetl";
            if (parts.length == 2) {
                database = parts[0];
                sql = parts[1].trim();
            } else if (parts.length != 1) {
                 logger.error("Expecting format '<database>:<sql-code> or '<sql-code>', but got '" + sql + "'");
                 return null;
            }
            if (sql.indexOf("<sqlvars>") > -1) {
                sql = StringUtils.removeSqlVars(sql);
            }
            step = new SqlAction(null, systemInfoId, databaseSystem, database, null, sql);
        } else {
            logger.info("\tnativeaction: file=" + scriptfile);
            
            step = new SqlScriptAction(databaseSystem, dataSource, sqlVardataSource ,systemInfoId, scriptfile);
        }
        return step;
    }
    
    private static String getDefaultSourcesystem(org.w3c.dom.Document unloadXml) {
        String result = "hisinone";
        NodeList sourcesystems = unloadXml.getElementsByTagName("sourcesystem");
        int count = sourcesystems.getLength();
        for (int i = 0; i < count; i++) {
            Node sourcesystem = sourcesystems.item(i);
            String def = getAttribute(sourcesystem.getAttributes(), "default");
            if (!def.isEmpty()) {
                result = getAttribute(sourcesystem.getAttributes(), "name");
                break;
            }
        }
        return result;
    }
    
    public static List<Node> filterElements(NodeList nodeList) {
        List<Node> nodes = new ArrayList<>();
        int count = nodeList.getLength();
        for (int i = 0; i < count; i++) {
            if (! (nodeList.item(i) instanceof Element) ) {
                continue;
            }
            nodes.add(nodeList.item(i));
        }
        return nodes;
    }

    @Override
	public List<ContainerNode> readInstallJobs(List<String> components) {
    	logger.debug("Reading install jobs from file system");
    	List<ContainerNode> jobs = new ArrayList<>();
    	for (String componentAbbreviation : PathAndFileUtils.getModules()) {
        	ContainerNode job = getInstallUpgradeJob(componentAbbreviation);
        	if (job != null) {
            	jobs.add(job);
        	}
    	}
    	logger.debug("Count of jobs: " + jobs.size());
    	return jobs;
	}

    public static ContainerNode getPreKernInstallJob() {
        ContainerNode root = createContainerNode("kern_pre_install", "kern_pre_install", Integer.valueOf(9));
        List<String> tables = Arrays.asList(
            "db_version",           // edustore manager
            "xdummy",           // one row 
            "felderinfo",
            "hochschulinfo",    // one row
            "konstanten",
            "userinfo",         // gang
            "groupinfo",        // gang
            "aggregierung",     // gang and fin
            "user_group_bez",   // gang and fin
            "db_tabellen",
            "sx_repository",
            "fm_templates",
            "sichten",
            "etl_job",
            "etl_job_param",
            "etl_step",
            "etl_step_property",
            "etl_step_relation",
            "etl_step_type",
            "etl_step_type_param",
            "orgunit_mapping",
            "sx_captions",
            "macro_feld_wert",
            "graphicformat",    // empty table
            "dbconnections",     // empty table
            "themenbaum"
        );

        for (String table : tables) {

            String unlFile = "$SUPERX_DIR/db/install/schluesseltabellen/release_unloads/" + table + ".unl";
            String tabFile = "$SUPERX_DIR/db/install/conf/his1_cli/dbconv/sch.eduetl/" + table + ".tab";

            SqlActionNoSxPool action = new SqlActionNoSxPool(tabFile, "eduetl", Integer.valueOf(9));
            root.add(action);

            LoadAction loadAction = new LoadAction(Integer.valueOf(9), false, "^", false, "eduetl", table, unlFile);
            root.add(loadAction);
        }

        return root;
    }

    @Override
    public List<ContainerNode> getJobsForComponent(String componentAbbreviation){
        
        List<ContainerNode> jobs = new ArrayList<>();
            
        ContainerNode installUpgrade = getInstallUpgradeJob(componentAbbreviation);
        if(installUpgrade != null) {
           jobs.add(installUpgrade); 
           ContainerNode uninstallJob = getUninstallJob(componentAbbreviation);
           if(uninstallJob != null) {
              jobs.add(uninstallJob); 
           }
        }

	    ContainerNode root = getLoadAndTransformJob(componentAbbreviation);
	    if (root != null) {
	        jobs.add(root);
	    }

	    List<ContainerNode> ulrs = getSpecialJobs(componentAbbreviation);
	    for (ContainerNode ulr : ulrs) {
	        jobs.add(ulr);
	    }

	    ContainerNode rootUnl = getUnloadJob(componentAbbreviation); 
	    if (rootUnl != null) {
	        jobs.add(rootUnl);
	    }

        ContainerNode rootComplete = getCompleteJob(componentAbbreviation);
        if (rootComplete != null) {
	        jobs.add(rootComplete);
        }

       return jobs;
    }

    @Override
    public List<ContainerNode> getSpecialJobsOverview(String abbreviation) {
        List<ContainerNode> result = new ArrayList<>();
        Document modulDoc = readModuleXml(abbreviation);
        if(modulDoc == null) {
            logger.error("Could not get SpecialJob Overview for module " + abbreviation);
            return result;
        }
        
        NodeList etlSteps = modulDoc.getElementsByTagName("etl-step");
        Integer systemInfoId = getSystemInfoId(modulDoc);
        int count = etlSteps.getLength();
        for (int i = 0; i < count; i++) {
            Node etlStep = etlSteps.item(i);
            List<Node> childNodes = filterElements(etlStep.getChildNodes());
            List<Node> desc = childNodes.stream().filter(el -> el.getNodeName().equals("description")).collect(Collectors.toList());
            String id = getAttribute(etlStep.getAttributes(), "id");
            String name = getAttribute(etlStep.getAttributes(), "name");
            String type = getAttribute(etlStep.getAttributes(), "type"); 
            String active = getAttribute(etlStep.getAttributes(), "active");
            String visible = getAttribute(etlStep.getAttributes(), "visible"); 
            if ("manual".equalsIgnoreCase(type) 
             && "true".equalsIgnoreCase(active) 
             && "true".equalsIgnoreCase(visible)) { 
                ContainerNode root = createContainerNode(name, JobType.special.createName(abbreviation, id), systemInfoId);
                if (desc.size() > 0) {
                    root.setDescription(desc.get(0).getTextContent().trim());
                }
                result.add(root);
            }
        }
        return result;
    }
    
    private static Document readModuleXml(String componentAbbreviation) {
        Document modulDoc = null;
        File xmlFile = null;
        try {
            xmlFile = getXmlFile(componentAbbreviation);
            if (getUnloadXmlFile(componentAbbreviation).exists()) {
                File xmlUnloadFile = getUnloadXmlFile(componentAbbreviation);
                modulDoc = replaceMultipleUnloadJobs(xmlFile, xmlUnloadFile);
            } else {
                modulDoc = readXmlFile(xmlFile);
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        if (modulDoc == null) {
            throw new RuntimeException("Couldn't read module xml for " + componentAbbreviation);
        }
        return modulDoc;
    }

}