You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1152 lines
56 KiB
1152 lines
56 KiB
package de.superx.spring.service; |
|
|
|
import static de.memtext.util.XMLUtils.replaceMultipleUnloadJobs; |
|
import static de.superx.util.PathAndFileUtils.getAttribute; |
|
import static de.superx.util.PathAndFileUtils.getDbiFileForManualEtlStep; |
|
import static de.superx.util.PathAndFileUtils.getModulePath; |
|
import static de.superx.util.PathAndFileUtils.getSystemInfoId; |
|
import static de.superx.util.PathAndFileUtils.getUnloadXmlFile; |
|
import static de.superx.util.PathAndFileUtils.getXmlFile; |
|
import static de.superx.util.PathAndFileUtils.getXslFile; |
|
import static de.superx.util.PathAndFileUtils.readXmlFile; |
|
|
|
import java.io.File; |
|
import java.io.FilenameFilter; |
|
import java.io.IOException; |
|
import java.util.ArrayList; |
|
import java.util.Arrays; |
|
import java.util.List; |
|
import java.util.Map; |
|
import java.util.Optional; |
|
import java.util.stream.Collectors; |
|
|
|
import javax.xml.transform.stream.StreamSource; |
|
|
|
import org.slf4j.Logger; |
|
import org.slf4j.LoggerFactory; |
|
import org.springframework.beans.factory.annotation.Autowired; |
|
import org.springframework.stereotype.Service; |
|
import org.w3c.dom.CharacterData; |
|
import org.w3c.dom.Document; |
|
import org.w3c.dom.Element; |
|
import org.w3c.dom.NamedNodeMap; |
|
import org.w3c.dom.Node; |
|
import org.w3c.dom.NodeList; |
|
|
|
import de.memtext.util.StringUtils; |
|
import de.superx.dbt.DbtAction; |
|
import de.superx.dbt.DbtExecutionState; |
|
import de.superx.dbt.DbtRunRequest; |
|
import de.superx.dbt.DbtRunResult; |
|
import de.superx.dbt.DbtWrapper; |
|
import de.superx.etl.biad.DBSource; |
|
import de.superx.etl.biad.SourceTable; |
|
import de.superx.etl.biad.SourcesYML; |
|
import de.superx.job.ActionNode; |
|
import de.superx.job.ContainerNode; |
|
import de.superx.job.ContainerNode.JobSource; |
|
import de.superx.job.ContainerNode.JobType; |
|
import de.superx.job.ExecuteActionGroupAction; |
|
import de.superx.job.ExecuteDbtAction; |
|
import de.superx.job.ExecuteSQLFluffAction; |
|
import de.superx.job.ExtractAction; |
|
import de.superx.job.KettleAction; |
|
import de.superx.job.LoadAction; |
|
import de.superx.job.LoadMondrianSchemaAction; |
|
import de.superx.job.RemoveSpringBatchHistoryAction; |
|
import de.superx.job.SqlAction; |
|
import de.superx.job.SqlActionNoSxPool; |
|
import de.superx.job.SqlScriptAction; |
|
import de.superx.job.Timestamp; |
|
import de.superx.job.TransferAction; |
|
import de.superx.job.UnloadAction; |
|
import de.superx.job.UnloadParams; |
|
import de.superx.job.UpdateMetadataAction; |
|
import de.superx.job.UpgradeDbFormsConfigAction; |
|
import de.superx.rest.model.job.Component; |
|
import de.superx.util.PathAndFileUtils; |
|
import de.superx.util.XmlUtil; |
|
|
|
@Service |
|
public class EntityJobDescriptionSource implements JobDescriptionSource { |
|
|
|
public static final String BIAD_COMPONENT_ABBREVIATION = "biad"; |
|
public static final int SYSTEMINFO_ID_BIAD = 440; |
|
public static final int SYSTEMINFO_ID_ASTAT = 280; |
|
public static final int SYSTEMINFO_ID_SOS = 7; |
|
static Logger logger = LoggerFactory.getLogger(EntityJobDescriptionSource.class); |
|
|
|
@Autowired |
|
DbtWrapper dbtWrapper; |
|
|
|
@Override |
|
public List<ContainerNode> readJobs() { |
|
List<ContainerNode> jobs = new ArrayList<>(); |
|
for (String componentAbbreviation : PathAndFileUtils.getModules()) { |
|
jobs.addAll(getJobsForComponent(componentAbbreviation)); |
|
} |
|
return jobs; |
|
} |
|
|
|
@Override |
|
public Component getComponentWithLoadAndUnloadDetails(String abbreviation) { |
|
Component component = new Component(abbreviation); |
|
|
|
boolean hasLoad = false; |
|
Document document = PathAndFileUtils.readXml(abbreviation, true); |
|
if(document != null) { |
|
NodeList etlSteps = document.getElementsByTagName("etl-step"); |
|
for (int i = 0; i < etlSteps.getLength(); i++) { |
|
Node etlStep = etlSteps.item(i); |
|
String type = getAttribute(etlStep.getAttributes(), "type"); |
|
if(type.equals("load")) { |
|
hasLoad = true; |
|
break; |
|
} |
|
} |
|
} |
|
|
|
boolean hasUnload = false; |
|
Document unloadXml = PathAndFileUtils.readXml(abbreviation, false); |
|
if(unloadXml != null) { |
|
NodeList unloadJobs = unloadXml.getElementsByTagName("unload-job"); |
|
if(unloadJobs != null && unloadJobs.getLength() > 0) { |
|
hasUnload = true; |
|
} |
|
} |
|
|
|
component.setLoad(hasLoad); |
|
component.setUnload(hasUnload); |
|
return component; |
|
} |
|
|
|
@Override |
|
public ContainerNode getUnloadJob(String componentAbbreviation) { |
|
|
|
// use sources.yml for bad component |
|
if(componentAbbreviation.equals(BIAD_COMPONENT_ABBREVIATION)) { |
|
ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Unload", JobType.unload.createName(componentAbbreviation), Integer.valueOf(SYSTEMINFO_ID_BIAD)); |
|
File file = new File(PathAndFileUtils.getSourcesYML("hisinone")); |
|
// make startup more robust: don't fail if sources.yml is not found #303809 |
|
if(!file.exists()) { |
|
logger.error("No sources.yml for " + componentAbbreviation); |
|
} else { |
|
SourcesYML sourcesYML = SourcesYML.fromJson(file); |
|
for (DBSource source : sourcesYML.getSources()) { |
|
if(!source.getName().startsWith("staging")) { |
|
continue; |
|
} |
|
for (SourceTable table : source.getTables()) { |
|
String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName()); |
|
ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getName(), source.getSrcDatabase()); |
|
root.add(action); |
|
} |
|
} |
|
} |
|
return root; |
|
} |
|
|
|
File xmlFile = getUnloadXmlFile(componentAbbreviation); |
|
if (! xmlFile.exists()) { |
|
logger.warn("No unload.xml for " + componentAbbreviation); |
|
return null; |
|
} |
|
Document unloadXml = null; |
|
try { |
|
unloadXml = readXmlFile(xmlFile); |
|
} catch (IOException e) { |
|
// TODO Auto-generated catch block |
|
e.printStackTrace(); |
|
} |
|
String defaultSourcesystem = getDefaultSourcesystem(unloadXml); |
|
Integer systemInfoId = getSystemInfoId(unloadXml); |
|
ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Unload", JobType.unload.createName(componentAbbreviation), systemInfoId); |
|
NodeList unloadJobs = unloadXml.getElementsByTagName("unload-job"); |
|
int unloadJobCount = unloadJobs.getLength(); |
|
if (unloadJobCount == 0) { |
|
return null; |
|
} |
|
for (int i = 0; i < unloadJobCount; i++) { |
|
Node unloadJob = unloadJobs.item(i); |
|
root.addAll(createUnloadStep(unloadJob, systemInfoId, componentAbbreviation)); |
|
} |
|
root.getActions().add(0, new de.superx.job.UnloadParams(systemInfoId)); |
|
return root; |
|
} |
|
|
|
public List<ActionNode> createUnloadStep(Node unloadStep, Integer systemInfoId, String componentAbbreviation) { |
|
|
|
List<ActionNode> result = new ArrayList<>(); |
|
String id = getAttribute(unloadStep.getAttributes(), "id"); |
|
String unloadFile = getAttribute(unloadStep.getAttributes(), "transfer-unload-file"); |
|
String commentString = null; |
|
logger.info(id + " (" + (!unloadFile.isEmpty()) + ")"); |
|
List<Node> sqlNodes = filterElements(unloadStep.getChildNodes()); |
|
Optional<Node> comment = sqlNodes.stream() |
|
.filter(node -> node.getNodeName().equalsIgnoreCase("comment") || node.getNodeName().equalsIgnoreCase("description")) |
|
.findFirst(); |
|
if (comment.isPresent()) { |
|
commentString = comment.get().getTextContent(); |
|
sqlNodes.remove(comment.get()); |
|
} |
|
sqlNodes.forEach(sqlNode -> { |
|
String sourcesystem = getAttribute(sqlNode.getAttributes(), "sourcesystem"); |
|
String database = getAttribute(sqlNode.getAttributes(), "database").toLowerCase(); |
|
String sqlvarsourcesystem = getAttribute(sqlNode.getAttributes(), "sqlVarDatabase").toLowerCase(); |
|
String version = getAttribute(sqlNode.getAttributes(), "version"); |
|
String sql = sqlNode.getTextContent(); |
|
if (sql.indexOf("<sqlvars>") > -1) { |
|
sql = StringUtils.removeSqlVars(sql); |
|
} |
|
if (unloadFile.isEmpty()) { |
|
result.add(new SqlAction(id, systemInfoId, database, sourcesystem, sqlvarsourcesystem, version, sql)); |
|
} else { |
|
result.add(new UnloadAction(id, sql, |
|
unloadFile.startsWith("$") ? |
|
unloadFile |
|
: "$" + componentAbbreviation.toUpperCase() + "_LOAD_PATH/" |
|
+ unloadFile, |
|
sourcesystem, sqlvarsourcesystem, systemInfoId, version, database)); |
|
} |
|
}); |
|
return result; |
|
} |
|
|
|
@Override |
|
public ContainerNode getLoadAndTransformJob(String componentAbbreviation) { |
|
|
|
File xmlFile = getXmlFile(componentAbbreviation); |
|
Document document = null; |
|
try { |
|
document = readXmlFile(xmlFile); |
|
} catch (IOException e) { |
|
logger.error("Couldn't read " + xmlFile.getAbsolutePath(), e); |
|
return null; |
|
} |
|
NodeList etlSteps = document.getElementsByTagName("etl-step"); |
|
Integer systemInfoId = getSystemInfoId(document); |
|
ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Load And Transform", |
|
JobType.load_transform.createName(componentAbbreviation), systemInfoId); |
|
for (int i = 0; i < etlSteps.getLength(); i++) { |
|
Node etlStep = etlSteps.item(i); |
|
String name = getAttribute(etlStep.getAttributes(), "name"); |
|
String type = getAttribute(etlStep.getAttributes(), "type"); |
|
String id = getAttribute(etlStep.getAttributes(), "id"); |
|
if ("manual".equals(type) ) { |
|
logger.info("etl-step " + name + " type manual. Skipping."); |
|
continue; |
|
} |
|
if (name.isEmpty()) { |
|
name = type; |
|
} |
|
logger.info("etl-step: " + name); |
|
ContainerNode container = createContainerNode(name, id, systemInfoId); |
|
List<ActionNode> stepList = createActions(filterElements(etlStep.getChildNodes()), systemInfoId, componentAbbreviation); |
|
if (stepList.size() > 0) { |
|
container.addAll(stepList); |
|
} |
|
if (container.name.startsWith("Upload")) { |
|
container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_load_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation))); |
|
container.add(new SqlScriptAction("postgres", systemInfoId, "$" + componentAbbreviation.toUpperCase() + "_PFAD/preparation.sql")); |
|
} |
|
if (container.name.startsWith("Transformation")) { |
|
container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_trans_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation))); |
|
container.add(new SqlScriptAction("postgres", systemInfoId, "$" + componentAbbreviation.toUpperCase() + "_PFAD/finalize.sql")); |
|
} |
|
if (container.name.startsWith("Aggregation")) { |
|
container.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_aggr_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation))); |
|
} |
|
// Keine leeren Container hinzufügen |
|
if (container.getActions().size() > 0) { |
|
root.add(container); |
|
} |
|
} |
|
int childrenCount = root.size(); |
|
if (childrenCount > 0) { |
|
ContainerNode timestampContainer = new ContainerNode("Timestamp", "Timestamp", systemInfoId); |
|
timestampContainer.add(new Timestamp(root.systemInfoId)); |
|
if (root.systemInfoId.intValue() == SYSTEMINFO_ID_SOS) { |
|
// for Studierende, Prüfungen HLR we also set the |
|
// timestamp for Astat, yes, ugly :( |
|
timestampContainer.add(new Timestamp(Integer.valueOf(SYSTEMINFO_ID_ASTAT))); |
|
} |
|
root.add(timestampContainer); |
|
} else { |
|
return null; |
|
} |
|
return root; |
|
} |
|
|
|
@Override |
|
public ContainerNode getCompleteJob(String componentAbbreviation) { |
|
ContainerNode loadAndTransform = getLoadAndTransformJob(componentAbbreviation); |
|
ContainerNode unload = getUnloadJob(componentAbbreviation); |
|
if (loadAndTransform == null || unload == null) { |
|
return null; |
|
} |
|
ContainerNode complete = createContainerNode(componentAbbreviation.toUpperCase() + |
|
" Unload, Load And Transform", JobType.complete.createName(componentAbbreviation), loadAndTransform.systemInfoId); |
|
complete.add(unload); |
|
complete.add(loadAndTransform); |
|
return complete; |
|
} |
|
|
|
@Override |
|
public ContainerNode getJobById(String id) { |
|
|
|
ContainerNode root = null; |
|
String componentAbbreviation = JobType.getComponentAbbreviationFromJobName(id); |
|
JobType type = JobType.getJobTypeForName(id); |
|
|
|
switch (type) { |
|
case install_upgrade: |
|
root = getInstallUpgradeJob(componentAbbreviation); |
|
break; |
|
case uninstall: |
|
root = "kern".equals(componentAbbreviation) ? null : getUninstallJob(componentAbbreviation); |
|
break; |
|
case unload: |
|
root = getUnloadJob(componentAbbreviation); |
|
break; |
|
case load_transform: |
|
root = getLoadAndTransformJob(componentAbbreviation); |
|
break; |
|
case complete: |
|
root = getCompleteJob(componentAbbreviation); |
|
break; |
|
case special: |
|
int first = id.indexOf('_'); |
|
int last = id.lastIndexOf('_'); |
|
root = getSpecialJob(componentAbbreviation, id.substring(first + 1, last)); |
|
break; |
|
case unknown: |
|
break; |
|
default: |
|
// maybe a component abbreviation e.g. 'res' was passed, |
|
// in this case return the complete job |
|
root = getCompleteJob(id); |
|
break; |
|
} |
|
return root; |
|
} |
|
|
|
@Override |
|
public List<ContainerNode> getSpecialJobs(String componentAbbreviation) { |
|
Document modulDoc = readModuleXml(componentAbbreviation); |
|
|
|
List<ContainerNode> result; |
|
|
|
try { |
|
NodeList nodes = modulDoc.getElementsByTagName("etl-step"); |
|
result = convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), getSystemInfoId(modulDoc)); |
|
} catch (Exception ex) { |
|
throw new RuntimeException(ex); |
|
} |
|
// add a UnloadParams-Action, so that Freemarkervariablen with unload_params |
|
// are always available in 'manual' jobs |
|
for (ContainerNode ulr : result) { |
|
ulr.getActions().add(0, new UnloadParams(ulr.systemInfoId)); |
|
} |
|
|
|
return result; |
|
} |
|
|
|
protected ContainerNode getSpecialJob(String componentAbbreviation, String jobId) { |
|
Document modulDoc = readModuleXml(componentAbbreviation); |
|
ContainerNode result = null; |
|
|
|
List<Node> etlSteps = XmlUtil.fromNodeList(modulDoc.getElementsByTagName("etl-step")); |
|
for (Node etlStep : etlSteps) { |
|
String id = getAttribute(etlStep.getAttributes(), "id"); |
|
if (! jobId.equals(id)) { |
|
logger.info(id + " id != " + jobId); |
|
continue; |
|
} |
|
Integer systemInfoId = getSystemInfoId(modulDoc); |
|
List<ContainerNode> l = convertToContainers(componentAbbreviation, List.of(etlStep), systemInfoId); |
|
result = l.get(0); |
|
if (getUnloadXmlFile(componentAbbreviation).exists()) { |
|
result.getActions().add(0, new de.superx.job.UnloadParams(systemInfoId)); |
|
} |
|
break; |
|
} |
|
return result; |
|
} |
|
|
|
public List<ContainerNode> convertToContainers(String componentAbbreviation, List<Node> etlSteps, Integer systemInfoId) { |
|
List<ContainerNode> result = new ArrayList<>(); |
|
for (Node etlStep : etlSteps) { |
|
String id = getAttribute(etlStep.getAttributes(), "id"); |
|
String name = getAttribute(etlStep.getAttributes(), "name"); |
|
String type = getAttribute(etlStep.getAttributes(), "type"); |
|
String active = getAttribute(etlStep.getAttributes(), "active"); |
|
String visible = getAttribute(etlStep.getAttributes(), "visible"); |
|
String generatedStr = getAttribute(etlStep.getAttributes(), "generateExecutable"); |
|
boolean generated = generatedStr.isEmpty() || "true".equalsIgnoreCase(generatedStr); |
|
if ("manual".equalsIgnoreCase(type) |
|
&& "true".equalsIgnoreCase(active) |
|
&& "true".equalsIgnoreCase(visible)) { |
|
if (generated) { |
|
List<Node> childNodes = filterElements(etlStep.getChildNodes()); |
|
List<Node> desc = childNodes.stream().filter(el -> el.getNodeName().equals("description")).collect(Collectors.toList()); |
|
ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " " + name, JobType.special.createName(componentAbbreviation, id), systemInfoId); |
|
if (desc.size() > 0) { |
|
root.setDescription(desc.get(0).getTextContent().trim()); |
|
} |
|
List<ActionNode> stepList = createActions(childNodes, systemInfoId, componentAbbreviation); |
|
if (stepList.size() > 0) { |
|
root.addAll(stepList); |
|
} |
|
if (stepList.size() == 0 || !stepList.get(stepList.size() - 1).id.contains("_custom")) { |
|
root.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_man_%s_custom.sql", |
|
componentAbbreviation.toUpperCase(), |
|
componentAbbreviation, |
|
id))); |
|
} |
|
result.add(root); |
|
} else { |
|
File dbiFile = getDbiFileForManualEtlStep(componentAbbreviation, id, "pg"); |
|
File xslFile = getXslFile("dbi_ulr.xsl"); |
|
Map<String, Object> parameters = Map.ofEntries( |
|
Map.entry("id", id), |
|
Map.entry("name", name), |
|
Map.entry("systemInfoId", systemInfoId) |
|
); |
|
Document doc = XmlUtil.xmlTransformWithXsl(new StreamSource(dbiFile), new StreamSource(xslFile), parameters); |
|
NodeList nodes = doc.getElementsByTagName("etl-step"); |
|
result.addAll(convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), systemInfoId)); |
|
} |
|
} |
|
} |
|
return result; |
|
} |
|
|
|
@Override |
|
public ContainerNode getInstallUpgradeJob(String componentAbbreviation) { |
|
|
|
File xmlFile; |
|
Document document; |
|
try { |
|
xmlFile = getXmlFile(componentAbbreviation); |
|
if(xmlFile.canRead()) { |
|
document = readXmlFile(xmlFile); |
|
}else { |
|
return null; |
|
} |
|
}catch(Exception e) { |
|
logger.info("Could not get xml file for "+ componentAbbreviation+" module. Skipping!"); |
|
return null; |
|
} |
|
|
|
NodeList installSteps = document.getElementsByTagName("install-upgrade-step"); |
|
int stepCount = installSteps.getLength(); |
|
|
|
if (stepCount == 0) { |
|
logger.warn("No install-upgrade-steps found for component " + componentAbbreviation + " Please check " + componentAbbreviation + ".xml!"); |
|
return null; |
|
} |
|
|
|
Integer systemInfoId = getSystemInfoId(document); |
|
|
|
// create containers |
|
ContainerNode root = createContainerNode(componentAbbreviation.toUpperCase() + " Install/ Upgrade", |
|
JobType.install_upgrade.createName(componentAbbreviation), systemInfoId); |
|
ContainerNode upgradeSchema = createContainerNode(componentAbbreviation.toUpperCase() + " Upgrade DB Schema", |
|
componentAbbreviation + "_upgrade_db_schema", systemInfoId); |
|
ContainerNode updateMetadata = createContainerNode(componentAbbreviation.toUpperCase() + "Update Module Metadata", componentAbbreviation + "_update_module_metadata", systemInfoId); |
|
ContainerNode scriptStepsContainer = createContainerNode(componentAbbreviation.toUpperCase() + " Upgrade-Scripts", |
|
componentAbbreviation + "_upgrade_scripts", systemInfoId); |
|
logger.debug("In module: "+componentAbbreviation+ " there were: " + stepCount+ " Install/ Upgrade Steps found!"); |
|
List<ActionNode> scriptSteps = new ArrayList<>(); |
|
for(int i = 0; i < stepCount; i++) { |
|
Node installStep = installSteps.item(i); |
|
String name = getAttribute(installStep.getAttributes(), "name"); |
|
String type = getAttribute(installStep.getAttributes(), "type"); |
|
String id = getAttribute(installStep.getAttributes(), "id"); |
|
if("container".equals(type)) { |
|
try { |
|
ContainerNode container = fillInstallUpgradeContainers(componentAbbreviation, name, id, systemInfoId, installStep); |
|
if (container.size() == 0) { |
|
// leere Container überspringen |
|
continue; |
|
} |
|
if (container.id.contains("update_module_metadata")) { |
|
updateMetadata = container; |
|
} else { |
|
root.add(container); |
|
} |
|
} catch (IOException e) { |
|
logger.error("couldn't read install/ upgrade job for module: "+componentAbbreviation); |
|
e.printStackTrace(); |
|
} |
|
continue; |
|
} |
|
List<ActionNode> stepList = createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation); |
|
for (ActionNode node : stepList) { |
|
if (node.id.endsWith("upgrade_tables_pg.sql") || node.id.endsWith("upgrade_views_pg.sql") |
|
|| node.id.endsWith("upgrade_fields_pg.sql") |
|
|| node.id.endsWith("upgrade_indexes_pg.sql") |
|
|| node.id.endsWith("sx_tables_fuellen.sql") |
|
) { |
|
upgradeSchema.add(node); |
|
} else { |
|
scriptSteps.add(node); |
|
} |
|
} |
|
} |
|
if (root.size() == 0) { |
|
logger.warn("No pre_upgrade container found in install-upgrade-steps for component " + componentAbbreviation + " Please check " + componentAbbreviation + ".xml!"); |
|
return null; |
|
} |
|
// nach pre_upgrade hinzufügen |
|
root.add(1, upgradeSchema); |
|
scriptStepsContainer.addAll(scriptSteps); |
|
installSteps = document.getElementsByTagName("upgrade-step"); |
|
if (installSteps.getLength() == 1) { |
|
List<ActionNode> stepList = createActions(filterElements(installSteps.item(0).getChildNodes()), systemInfoId, componentAbbreviation); |
|
if (stepList.size() > 0) { |
|
scriptStepsContainer.addAll(stepList); |
|
} |
|
} |
|
scriptStepsContainer.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/%s_install_custom.sql", componentAbbreviation.toUpperCase(), componentAbbreviation))); |
|
root.add(scriptStepsContainer); |
|
updateMetadata.addAll(additionalInstallActions(componentAbbreviation, systemInfoId)); |
|
root.add(updateMetadata); |
|
root.add(new UpgradeDbFormsConfigAction(systemInfoId, true, componentAbbreviation)); |
|
root.add(new SqlScriptAction("postgres", systemInfoId, String.format("$%s_PFAD/conf/customize.sql", componentAbbreviation.toUpperCase()))); |
|
root.add(new SqlAction("Update DB version", systemInfoId, "postgres", "eduetl", null, |
|
"insert into db_version (his_system, version, version_hash, systeminfo_id) values ('" + componentAbbreviation + "'," |
|
+ " '${version}', '${version_hash}', " + systemInfoId + ")\n" |
|
+ "on conflict (his_system) do\n" |
|
+ "update set version='${version}', version_hash='${version_hash}'")); |
|
|
|
return stepCount != 0 ? root : null; |
|
} |
|
|
|
@Override |
|
public ContainerNode getUninstallJob(String componentAbbreviation) { |
|
if ("kern".equalsIgnoreCase(componentAbbreviation)) { |
|
return null; |
|
} |
|
Integer systemInfoId = Integer.valueOf(-1); |
|
try { |
|
systemInfoId = Integer.valueOf(getSystemInfoId(componentAbbreviation)); |
|
} catch (Exception e) { |
|
logger.error("Couldn't get systemInfoId for " +componentAbbreviation, e); |
|
} |
|
File uninstallFile = new File(String.format("%s/conf/his1/edustore_install/edustore_%s_drop_pg.xml", |
|
getModulePath(componentAbbreviation), componentAbbreviation)); |
|
File xslFile = getXslFile("dbi_ulr.xsl"); |
|
Map<String, Object> parameters = Map.ofEntries( |
|
Map.entry("id", "uninstall"), |
|
Map.entry("name", componentAbbreviation + "_uninstall"), |
|
Map.entry("systemInfoId", systemInfoId) |
|
); |
|
Document doc = XmlUtil.xmlTransformWithXsl(new StreamSource(uninstallFile), new StreamSource(xslFile), parameters); |
|
ContainerNode uninstall = new ContainerNode("Uninstall_" + componentAbbreviation.toLowerCase(), JobType.uninstall.createName(componentAbbreviation.toLowerCase()), systemInfoId); |
|
NodeList nodes = doc.getElementsByTagName("etl-step"); |
|
List<ContainerNode> converted = convertToContainers(componentAbbreviation, XmlUtil.fromNodeList(nodes), systemInfoId); |
|
uninstall.addAll(converted.get(0).getActions()); |
|
uninstall.name = componentAbbreviation.toUpperCase() + " Uninstall"; |
|
uninstall.addAll(additionalInstallActions(componentAbbreviation, systemInfoId)); |
|
return uninstall; |
|
} |
|
|
|
private static List<ActionNode> additionalInstallActions(String componentAbbreviation, Integer systemInfoId) { |
|
List<ActionNode> result = new ArrayList<>(); |
|
//result.add(new LoadEtlRoutinesAction("load_etl_routines", null, systemInfoId, true, |
|
// false, componentAbbreviation)); |
|
result.add(new LoadMondrianSchemaAction("load_mondrian_schema", null, systemInfoId, true, |
|
Integer.valueOf(0), componentAbbreviation)); |
|
return result; |
|
} |
|
|
|
private ContainerNode fillInstallUpgradeContainers(String componentAbbreviation, String containerName, String containerId, Integer systemInfoId, Node installStep) throws IOException { |
|
ContainerNode container = createContainerNode(componentAbbreviation.toUpperCase() + " " + containerName, containerId, systemInfoId); |
|
|
|
if("pre_upgrade".equals(containerName)) { |
|
container.addAll(checkForPreUpgrade(componentAbbreviation, systemInfoId)); |
|
}else if("install_functions".equals(containerName)) { |
|
container.addAll(installFunctions(componentAbbreviation, systemInfoId)); |
|
}else if("install_masks".equals(containerName)) { |
|
container.addAll(installMasks(componentAbbreviation, systemInfoId)); |
|
} else if("Update Module Metadata".equals(containerName)) { |
|
List<ActionNode> actionNodes = metadaten(componentAbbreviation, systemInfoId, installStep); |
|
container.addAll(actionNodes); |
|
} else { |
|
// generic container |
|
container.addAll(createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation)); |
|
} |
|
|
|
return container; |
|
} |
|
|
|
private List<ActionNode> metadaten(String componentAbbreviation, Integer systemInfoId, Node installStep) { |
|
return createActions(filterElements(installStep.getChildNodes()), systemInfoId, componentAbbreviation); |
|
} |
|
|
|
private List<ActionNode> installMasks(String componentAbbreviation, Integer systemInfoId) throws IOException { |
|
List<ActionNode> actions = new ArrayList<>(); |
|
File xmlFile = getXmlFile(componentAbbreviation); |
|
Document document = readXmlFile(xmlFile); |
|
|
|
NodeList masks = document.getElementsByTagName("maske"); |
|
int maskCount = masks.getLength(); |
|
for(int i = 0; i < maskCount; i++) { |
|
Node mask = masks.item(i); |
|
String id = getAttribute(mask.getAttributes(), "tid"); |
|
String name = getAttribute(mask.getAttributes(), "name"); |
|
String path = null; |
|
|
|
NodeList srcTags = ((Element) mask).getElementsByTagName("src"); |
|
for (int x = 0; x < srcTags.getLength(); x++) { |
|
Element e = (Element) srcTags.item(x); |
|
|
|
String pathText = e.getElementsByTagName("path").item(0).getTextContent(); |
|
String dbSystemText = null; // null means no dbsystem tag -> POSTGRES by default |
|
|
|
NodeList dbSystemNode = e.getElementsByTagName("dbsystem"); |
|
if(dbSystemNode.getLength() > 0) { |
|
dbSystemText = dbSystemNode.item(0).getTextContent(); |
|
} |
|
if(dbSystemText == null || dbSystemText.isBlank() || "POSTGRES".equals(dbSystemText)) { |
|
path = pathText; |
|
} |
|
} |
|
|
|
ContainerNode container = createContainerNode("Install mask: (" + id + ") " + name, "install_mask_tid_"+id, systemInfoId); |
|
|
|
container.add(new SqlAction("sql_del_maskeninfo_"+id, systemInfoId, "postgres", "eduetl", null, String.format( |
|
"delete from maskeninfo where tid = %s;",id))); |
|
container.add(new SqlAction("sql_del_felderinfo_"+id, systemInfoId, "postgres", "eduetl", null, String.format( |
|
"delete from felderinfo where tid in (select felderinfo_id from masken_felder_bez where maskeninfo_id = %s);",id))); |
|
container.add(new SqlAction("sql_del_masken_felder_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format( |
|
"delete from masken_felder_bez where maskeninfo_id = %s;",id))); |
|
container.add(new SqlAction("sql_del_sachgeb_maske_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format( |
|
"delete from sachgeb_maske_bez where maskeninfo_id = %s;",id))); |
|
container.add(new SqlAction("sql_del_maske_sys_bez_"+id, systemInfoId, "postgres", "eduetl", null, String.format( |
|
"delete from maske_system_bez where maskeninfo_id = %s;",id))); |
|
|
|
String filePath = path + "/" + id + "_"; |
|
container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "maskeninfo", filePath + "maskeninfo.unl")); |
|
container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "felderinfo", filePath + "felderinfo.unl")); |
|
container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "masken_felder_bez", filePath + "masken_felder_bez.unl")); |
|
container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "sachgeb_maske_bez", filePath + "sachgeb_maske_bez.unl")); |
|
container.add(new LoadAction(systemInfoId, false, "^", false, "eduetl", "maske_system_bez", filePath + "maske_system_bez.unl")); |
|
|
|
actions.add(container); |
|
} |
|
return actions; |
|
} |
|
|
|
private static ContainerNode createContainerNode(String name, String id, Integer systemInfoId) { |
|
ContainerNode node = new ContainerNode(name, id, systemInfoId); |
|
node.setJobSource(JobSource.FILE); |
|
return node; |
|
} |
|
|
|
private List<ActionNode> installFunctions(String componentAbbreviation, Integer systemInfoId) throws IOException { |
|
File xmlFile = getXmlFile(componentAbbreviation); |
|
Document document = readXmlFile(xmlFile); |
|
List<ActionNode> functionActions = new ArrayList<>(); |
|
|
|
NodeList functions = document.getElementsByTagName("function"); |
|
int functionsCount = functions.getLength(); |
|
for(int i = 0; i < functionsCount; i++) { |
|
Element function = (Element)functions.item(i); |
|
NodeList srcTags = function.getElementsByTagName("src"); |
|
int srcTagsCount = srcTags.getLength(); |
|
for(int k = 0; k < srcTagsCount; k++) { |
|
Element src = (Element)srcTags.item(k); |
|
//String file = src.getElementsByTagName("file").item(0).getTextContent(); |
|
NodeList file = src.getElementsByTagName("file"); |
|
NodeList body = src.getElementsByTagName("body"); |
|
|
|
String dbsystem = src.getElementsByTagName("dbsystem").item(0).getTextContent(); |
|
|
|
if(file.getLength() > 0) { |
|
functionActions.add(installFunctionFileTag(file, dbsystem, systemInfoId)); |
|
}else if(body.getLength() > 0) { |
|
functionActions.add(installFunctionBodyTag(body, dbsystem, systemInfoId)); |
|
} |
|
} |
|
} |
|
|
|
return functionActions; |
|
} |
|
|
|
private static ActionNode installFunctionBodyTag(NodeList body, String dbsystem, Integer systemInfoId) { |
|
|
|
Node cdata = body.item(0).getFirstChild(); |
|
if(cdata instanceof CharacterData) { |
|
CharacterData cd = (CharacterData) cdata; |
|
return new SqlAction(cd.getData().toLowerCase().substring(0, 10).replace(" ", "_")+"_function_from_"+systemInfoId.toString(), |
|
systemInfoId, dbsystem, "eduetl", null, cd.getData()); |
|
} |
|
|
|
return null; |
|
} |
|
|
|
private static ActionNode installFunctionFileTag(NodeList file, String dbsystem, Integer systemInfoId) { |
|
String f = file.item(0).getTextContent(); |
|
|
|
if(dbsystem.equals("INFORMIX")) { |
|
return new SqlScriptAction("informix", systemInfoId, f); |
|
} |
|
|
|
return new SqlScriptAction("postgres", systemInfoId, f); |
|
} |
|
|
|
private static List<ActionNode> checkForPreUpgrade(String componentAbbreviation, Integer systemInfoId) throws IOException { |
|
List<ActionNode> actions = new ArrayList<>(); |
|
|
|
String moduleUpgradeDirPath = getModulePath(componentAbbreviation)+"/upgrade"; |
|
File moduleUpgradeDir = new File(moduleUpgradeDirPath); |
|
File[] preUpgradeFiles = moduleUpgradeDir.listFiles(new FilenameFilter() { |
|
|
|
@Override |
|
public boolean accept(File dir, String name) { |
|
String filename = name.strip().toLowerCase(); |
|
if(filename.contains("pre_upgrade_pg") && filename.endsWith("sql") && !filename.contains("tmp")) { |
|
return true; |
|
} |
|
return false; |
|
} |
|
}); |
|
|
|
for(int i = 0; i < preUpgradeFiles.length; i++) { |
|
actions.add(new SqlScriptAction("postgres", systemInfoId, String.format( |
|
"$%s_PFAD/upgrade/%s", componentAbbreviation.toUpperCase(), preUpgradeFiles[i].getName()))); |
|
} |
|
|
|
return actions; |
|
} |
|
|
|
private List<ActionNode> createActions(List<Node> actionNodes, Integer systemInfoId, String componentAbbreviation) { |
|
List<ActionNode> result = new ArrayList<>(); |
|
actionNodes.forEach(node -> { |
|
logger.info("action :"); |
|
List<Node> actions = filterElements(node.getChildNodes()); |
|
for (Node action : actions) { |
|
String name = action.getNodeName(); |
|
NamedNodeMap attributes = action.getAttributes(); |
|
if ("nativeaction".equals(name)) { |
|
String database = getAttribute(attributes, "database"); |
|
String container = getAttribute(attributes, "container"); |
|
ActionNode actionStep = createActionStep(attributes, systemInfoId); |
|
if (actionStep != null) { |
|
if (container != null && container.equalsIgnoreCase("true")) { |
|
// TODO: Container sollten generischer gelöst werden! |
|
// Das ist hier ein Workaround, damit man die Ausführung einer |
|
// Action in einen Container packen kann. Damit wird ein Neueinlesen |
|
// der Freemarkervariablen getriggert. |
|
// Z.B. notwendig für sos install, damit sos_modul_upgrade_man.sql |
|
// auf die Variablen im davor ausgeführten Schritt zugreifen kann. |
|
ContainerNode containerNode = createContainerNode( name + " Container", actionStep.id + "_container", systemInfoId); |
|
containerNode.add(actionStep); |
|
result.add(containerNode); |
|
} else { |
|
result.add(actionStep); |
|
} |
|
} |
|
} else if ("loadtable".equals(name)) { |
|
ActionNode loadStep = createLoadStep(action, attributes, systemInfoId); |
|
result.add(loadStep); |
|
} else if ("transferTable".equals(name)) { |
|
ActionNode transferStep = createTransferStep(attributes, systemInfoId); |
|
result.add(transferStep); |
|
} else if ("kettle-job-embedded".equals(name)) { |
|
ActionNode kettleStep = createKettleStep(attributes, systemInfoId); |
|
result.add(kettleStep); |
|
} else if ("unload-job".equals(name)) { |
|
result.addAll(createUnloadStep(action, systemInfoId, componentAbbreviation)); |
|
} else if("execute-unload-scripts".equals(name)) { |
|
result.addAll(createLoadAfterExtract(componentAbbreviation, systemInfoId)); |
|
} else if("biad_unload_and_load".equals(name)) { |
|
result.add(getUnloadJob(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION)); |
|
List<ActionNode> nodes = createLoadAfterExtract(componentAbbreviation, Integer.valueOf(SYSTEMINFO_ID_BIAD)); |
|
result.addAll(nodes); |
|
} else if("evaluate-dbt-json".equals(name)) { |
|
String noGroupActionAttr = getAttribute(attributes, "noGroupAction"); |
|
boolean groupAction = !Boolean.parseBoolean(noGroupActionAttr); |
|
result.addAll(createDbtExecutionStep(attributes, systemInfoId)); |
|
result.add(createMetaimportJsonStep(attributes, systemInfoId)); |
|
if (groupAction) { |
|
result.add(new ExecuteActionGroupAction(systemInfoId)); |
|
} |
|
} else if("evaluate-metadata".equals(name)) { |
|
result.add(createMetaimportJsonStep(attributes, systemInfoId)); |
|
} else if("evaluate-dbt-test".equals(name)) { |
|
result.addAll(createDbtTestStep(attributes, systemInfoId)); |
|
} else if("remove-spring-batch-history".equals(name)) { |
|
result.add(new RemoveSpringBatchHistoryAction(systemInfoId)); |
|
} else if ("loadtableinfo".equals(name)) { |
|
// this exists in cob.xml but is obviously not used |
|
// so skipping here! |
|
} else { |
|
logger.error("Unknown action type: " + name); |
|
// throw new RuntimeException("Unknown action type: " + name); |
|
} |
|
} |
|
}); |
|
return result; |
|
} |
|
|
|
private List<ActionNode> createDbtTestStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
List<ActionNode> nodes = new ArrayList<>(); |
|
String dbtProject = getAttribute(attributes, "project"); |
|
String dbtSelect = getAttribute(attributes, "select"); |
|
nodes.add(new ExecuteDbtAction(dbtProject, dbtSelect, systemInfoId, true)); |
|
nodes.add(new ExecuteSQLFluffAction(dbtProject, "Lint DBT Models using SQLFluff", "sqlfluff_action", Integer.valueOf(SYSTEMINFO_ID_BIAD), true, Integer.valueOf(0))); |
|
return nodes; |
|
} |
|
|
|
private List<ActionNode> createDbtExecutionStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
List<ActionNode> nodes = new ArrayList<>(); |
|
String dbtProject = getAttribute(attributes, "project"); |
|
String dbtSelect = getAttribute(attributes, "select"); |
|
String sourceSystem = getAttribute(attributes, "sourcesystem"); |
|
if(sourceSystem == null || sourceSystem.isBlank()) { |
|
sourceSystem = "hisinone"; |
|
} |
|
|
|
if(dbtSelect != null && !dbtSelect.isBlank()) { |
|
String id = getAttribute(attributes, "id"); |
|
ContainerNode extractLoad = createContainerNode("Extract and Load", "extract_load_" + id, systemInfoId); |
|
|
|
DbtRunRequest request = new DbtRunRequest(DbtAction.STAGING_TABLES, dbtProject, dbtSelect); |
|
DbtRunResult result = dbtWrapper.execute(request); |
|
if(!result.getState().equals(DbtExecutionState.SUCCESS)) { |
|
logger.error("Could not get specific Staging Tables.", String.join("\n", result.getErrorMessages())); |
|
} |
|
List<String> stagingTables = result.getOutputMessages(); |
|
|
|
if(stagingTables != null && !stagingTables.isEmpty()) { |
|
extractLoad.addAll(createExtractAndLoadStep(stagingTables, systemInfoId, sourceSystem)); |
|
nodes.add(extractLoad); |
|
} |
|
} |
|
|
|
nodes.add(new ExecuteDbtAction(dbtProject, dbtSelect, systemInfoId)); |
|
return nodes; |
|
} |
|
|
|
private static ActionNode createMetaimportJsonStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
String dbtProject = getAttribute(attributes, "project"); |
|
return new UpdateMetadataAction(dbtProject, systemInfoId); |
|
} |
|
|
|
private static List<ActionNode> createExtractAndLoadStep(List<String> tables, Integer systemInfoId, String sourceSystem) { |
|
List<ActionNode> result = new ArrayList<>(); |
|
SourcesYML sourcesYML = SourcesYML.fromJson(new File(PathAndFileUtils.getSourcesYML("hisinone"))); |
|
List<DBSource> sources = sourcesYML.getSources(); |
|
for (DBSource source : sources) { |
|
if(!source.getName().startsWith("staging")) { |
|
continue; |
|
} |
|
for (SourceTable table : source.getTables()) { |
|
if(!tables.contains(table.getName())) { |
|
continue; |
|
} |
|
|
|
String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName()); |
|
|
|
ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getSchema(), sourceSystem); |
|
result.add(action); |
|
String unloadPath = PathAndFileUtils.getUnloadPath("biad") |
|
+ File.separator + "tables" + File.separator + table.getName() + ".sql"; |
|
|
|
SqlScriptAction createTableAction = new SqlScriptAction("postgres", systemInfoId, unloadPath); |
|
result.add(createTableAction); |
|
|
|
LoadAction loadAction = new LoadAction(systemInfoId, false, ",", false, |
|
"eduetl", source.getSchema() + "." + table.getName(), unlFile, true); |
|
result.add(loadAction); |
|
logger.info("\tloadtable: tabname=" + action.sourceTable + " path=" + action.unlFile); |
|
} |
|
} |
|
return result; |
|
} |
|
|
|
private List<ActionNode> createLoadAfterExtract(String componentAbbreviation, Integer systemInfoId) { |
|
List<ActionNode> result = new ArrayList<>(); |
|
SourcesYML sourcesYML = SourcesYML.fromJson(new File(PathAndFileUtils.getSourcesYML("hisinone"))); |
|
for (DBSource source : sourcesYML.getSources()) { |
|
if(!source.getName().startsWith("staging")) { |
|
continue; |
|
} |
|
for (SourceTable table : source.getTables()) { |
|
String unlFile = PathAndFileUtils.buildUnlFile(EntityJobDescriptionSource.BIAD_COMPONENT_ABBREVIATION, table.getName()); |
|
ExtractAction action = ExtractAction.createFromSourceTable(table, unlFile, source.getSchema(), source.getSrcDatabase()); |
|
|
|
String unloadPath = PathAndFileUtils.getUnloadPath("biad") |
|
+ File.separator + "tables" + File.separator + table.getName() + ".sql"; |
|
SqlScriptAction createTableAction = new SqlScriptAction("postgres", systemInfoId, unloadPath); |
|
|
|
LoadAction loadAction = new LoadAction(systemInfoId, false, ",", false, |
|
"eduetl", source.getSchema() + "." + table.getName(), unlFile, true); |
|
|
|
result.add(createTableAction); |
|
result.add(loadAction); |
|
logger.info("\tloadtable: tabname=" + action.sourceTable + " path=" + action.unlFile); |
|
} |
|
} |
|
return result; |
|
} |
|
|
|
private ActionNode createKettleStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
String name = getAttribute(attributes, "name"); |
|
String file = getAttribute(attributes, "file"); |
|
logger.info("\tcreating kettle job " + name + " in path " + file); |
|
KettleAction kettleAction = new KettleAction(name, systemInfoId, file); |
|
return kettleAction; |
|
} |
|
|
|
private ActionNode createTransferStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
String fromTable = getAttribute(attributes, "src"); |
|
String toTable = getAttribute(attributes, "target"); |
|
logger.info("\ttransfer table " + fromTable + " to " + toTable); |
|
TransferAction transferAction = new TransferAction(systemInfoId, fromTable, toTable); |
|
return transferAction; |
|
} |
|
|
|
private ActionNode createLoadStep(Node action, NamedNodeMap attributes, Integer systemInfoId) { |
|
String refresh = getAttribute(attributes, "refresh"); |
|
String checkEmpty = getAttribute(attributes, "checkEmpty"); |
|
String tabName = getAttribute(attributes, "tabname"); |
|
String delimiter = getAttribute(attributes, "delimiter"); |
|
String header = getAttribute(attributes, "header"); |
|
String database = getAttribute(attributes, "database"); |
|
Node fileNode = filterElements(action.getChildNodes()).get(0); |
|
String unloadFileAttribute = getAttribute(fileNode.getAttributes(), "path"); |
|
logger.info("\tloadtable: tabname=" + tabName + " path=" + unloadFileAttribute); |
|
LoadAction loadAction = new LoadAction( |
|
systemInfoId, |
|
Boolean.parseBoolean(refresh), |
|
delimiter, |
|
Boolean.parseBoolean(header), |
|
database.trim().isEmpty() ? "eduetl" : database , |
|
tabName, |
|
unloadFileAttribute |
|
); |
|
if (!checkEmpty.isEmpty()) { |
|
loadAction.checkEmpty = Boolean.parseBoolean(checkEmpty); |
|
} |
|
return loadAction; |
|
} |
|
|
|
private ActionNode createActionStep(NamedNodeMap attributes, Integer systemInfoId) { |
|
ActionNode step = null; |
|
String sql = getAttribute(attributes, "sql"); |
|
String scriptfile = getAttribute(attributes, "scriptfile"); |
|
String dataSource = getAttribute(attributes, "dataSource"); |
|
String databaseSystem = getAttribute(attributes, "database"); |
|
if (dataSource.isEmpty()) { |
|
dataSource = "eduetl"; |
|
} |
|
String sqlVardataSource = getAttribute(attributes, "sqlVarDatabase"); |
|
if (sqlVardataSource.isEmpty()) { |
|
sqlVardataSource = "eduetl"; |
|
} |
|
if (!sql.isEmpty()) { |
|
logger.info("\tnativeaction: " + sql); |
|
String[] parts = sql.split(":", 2); |
|
String database = "eduetl"; |
|
if (parts.length == 2) { |
|
database = parts[0]; |
|
sql = parts[1].trim(); |
|
} else if (parts.length != 1) { |
|
logger.error("Expecting format '<database>:<sql-code> or '<sql-code>', but got '" + sql + "'"); |
|
return null; |
|
} |
|
if (sql.indexOf("<sqlvars>") > -1) { |
|
sql = StringUtils.removeSqlVars(sql); |
|
} |
|
step = new SqlAction(null, systemInfoId, databaseSystem, database, null, sql); |
|
} else { |
|
logger.info("\tnativeaction: file=" + scriptfile); |
|
|
|
step = new SqlScriptAction(databaseSystem, dataSource, sqlVardataSource ,systemInfoId, scriptfile); |
|
} |
|
return step; |
|
} |
|
|
|
private static String getDefaultSourcesystem(org.w3c.dom.Document unloadXml) { |
|
String result = "hisinone"; |
|
NodeList sourcesystems = unloadXml.getElementsByTagName("sourcesystem"); |
|
int count = sourcesystems.getLength(); |
|
for (int i = 0; i < count; i++) { |
|
Node sourcesystem = sourcesystems.item(i); |
|
String def = getAttribute(sourcesystem.getAttributes(), "default"); |
|
if (!def.isEmpty()) { |
|
result = getAttribute(sourcesystem.getAttributes(), "name"); |
|
break; |
|
} |
|
} |
|
return result; |
|
} |
|
|
|
public static List<Node> filterElements(NodeList nodeList) { |
|
List<Node> nodes = new ArrayList<>(); |
|
int count = nodeList.getLength(); |
|
for (int i = 0; i < count; i++) { |
|
if (! (nodeList.item(i) instanceof Element) ) { |
|
continue; |
|
} |
|
nodes.add(nodeList.item(i)); |
|
} |
|
return nodes; |
|
} |
|
|
|
@Override |
|
public List<ContainerNode> readInstallJobs(List<String> components) { |
|
logger.debug("Reading install jobs from file system"); |
|
List<ContainerNode> jobs = new ArrayList<>(); |
|
for (String componentAbbreviation : PathAndFileUtils.getModules()) { |
|
ContainerNode job = getInstallUpgradeJob(componentAbbreviation); |
|
if (job != null) { |
|
jobs.add(job); |
|
} |
|
} |
|
logger.debug("Count of jobs: " + jobs.size()); |
|
return jobs; |
|
} |
|
|
|
public static ContainerNode getPreKernInstallJob() { |
|
ContainerNode root = createContainerNode("kern_pre_install", "kern_pre_install", Integer.valueOf(9)); |
|
List<String> tables = Arrays.asList( |
|
"db_version", // edustore manager |
|
"xdummy", // one row |
|
"felderinfo", |
|
"hochschulinfo", // one row |
|
"konstanten", |
|
"userinfo", // gang |
|
"groupinfo", // gang |
|
"aggregierung", // gang and fin |
|
"user_group_bez", // gang and fin |
|
"db_tabellen", |
|
"sx_repository", |
|
"fm_templates", |
|
"sichten", |
|
"etl_job", |
|
"etl_job_param", |
|
"etl_step", |
|
"etl_step_property", |
|
"etl_step_relation", |
|
"etl_step_type", |
|
"etl_step_type_param", |
|
"orgunit_mapping", |
|
"sx_captions", |
|
"macro_feld_wert", |
|
"graphicformat", // empty table |
|
"dbconnections", // empty table |
|
"themenbaum" |
|
); |
|
|
|
for (String table : tables) { |
|
|
|
String unlFile = "$SUPERX_DIR/db/install/schluesseltabellen/release_unloads/" + table + ".unl"; |
|
String tabFile = "$SUPERX_DIR/db/install/conf/his1_cli/dbconv/sch.eduetl/" + table + ".tab"; |
|
|
|
SqlActionNoSxPool action = new SqlActionNoSxPool(tabFile, "eduetl", Integer.valueOf(9)); |
|
root.add(action); |
|
|
|
LoadAction loadAction = new LoadAction(Integer.valueOf(9), false, "^", false, "eduetl", table, unlFile); |
|
root.add(loadAction); |
|
} |
|
|
|
return root; |
|
} |
|
|
|
@Override |
|
public List<ContainerNode> getJobsForComponent(String componentAbbreviation){ |
|
|
|
List<ContainerNode> jobs = new ArrayList<>(); |
|
|
|
ContainerNode installUpgrade = getInstallUpgradeJob(componentAbbreviation); |
|
if(installUpgrade != null) { |
|
jobs.add(installUpgrade); |
|
ContainerNode uninstallJob = getUninstallJob(componentAbbreviation); |
|
if(uninstallJob != null) { |
|
jobs.add(uninstallJob); |
|
} |
|
} |
|
|
|
ContainerNode root = getLoadAndTransformJob(componentAbbreviation); |
|
if (root != null) { |
|
jobs.add(root); |
|
} |
|
|
|
List<ContainerNode> ulrs = getSpecialJobs(componentAbbreviation); |
|
for (ContainerNode ulr : ulrs) { |
|
jobs.add(ulr); |
|
} |
|
|
|
ContainerNode rootUnl = getUnloadJob(componentAbbreviation); |
|
if (rootUnl != null) { |
|
jobs.add(rootUnl); |
|
} |
|
|
|
ContainerNode rootComplete = getCompleteJob(componentAbbreviation); |
|
if (rootComplete != null) { |
|
jobs.add(rootComplete); |
|
} |
|
|
|
return jobs; |
|
} |
|
|
|
@Override |
|
public List<ContainerNode> getSpecialJobsOverview(String abbreviation) { |
|
List<ContainerNode> result = new ArrayList<>(); |
|
Document modulDoc = readModuleXml(abbreviation); |
|
if(modulDoc == null) { |
|
logger.error("Could not get SpecialJob Overview for module " + abbreviation); |
|
return result; |
|
} |
|
|
|
NodeList etlSteps = modulDoc.getElementsByTagName("etl-step"); |
|
Integer systemInfoId = getSystemInfoId(modulDoc); |
|
int count = etlSteps.getLength(); |
|
for (int i = 0; i < count; i++) { |
|
Node etlStep = etlSteps.item(i); |
|
List<Node> childNodes = filterElements(etlStep.getChildNodes()); |
|
List<Node> desc = childNodes.stream().filter(el -> el.getNodeName().equals("description")).collect(Collectors.toList()); |
|
String id = getAttribute(etlStep.getAttributes(), "id"); |
|
String name = getAttribute(etlStep.getAttributes(), "name"); |
|
String type = getAttribute(etlStep.getAttributes(), "type"); |
|
String active = getAttribute(etlStep.getAttributes(), "active"); |
|
String visible = getAttribute(etlStep.getAttributes(), "visible"); |
|
if ("manual".equalsIgnoreCase(type) |
|
&& "true".equalsIgnoreCase(active) |
|
&& "true".equalsIgnoreCase(visible)) { |
|
ContainerNode root = createContainerNode(name, JobType.special.createName(abbreviation, id), systemInfoId); |
|
if (desc.size() > 0) { |
|
root.setDescription(desc.get(0).getTextContent().trim()); |
|
} |
|
result.add(root); |
|
} |
|
} |
|
return result; |
|
} |
|
|
|
private static Document readModuleXml(String componentAbbreviation) { |
|
Document modulDoc = null; |
|
File xmlFile = null; |
|
try { |
|
xmlFile = getXmlFile(componentAbbreviation); |
|
if (getUnloadXmlFile(componentAbbreviation).exists()) { |
|
File xmlUnloadFile = getUnloadXmlFile(componentAbbreviation); |
|
modulDoc = replaceMultipleUnloadJobs(xmlFile, xmlUnloadFile); |
|
} else { |
|
modulDoc = readXmlFile(xmlFile); |
|
} |
|
} catch (Exception ex) { |
|
throw new RuntimeException(ex); |
|
} |
|
if (modulDoc == null) { |
|
throw new RuntimeException("Couldn't read module xml for " + componentAbbreviation); |
|
} |
|
return modulDoc; |
|
} |
|
|
|
}
|
|
|