kern/src/de/superx/bin/ComponentAdminCLI.java

package de.superx.bin;

import static de.superx.servlet.SxSQL_Server.DEFAULT_MANDANTEN_ID;

import java.io.File;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;

import javax.sql.DataSource;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.springframework.beans.BeansException;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.support.GenericApplicationContext;

import de.superx.job.ContainerNode;
import de.superx.rest.EtlJobApi;
import de.superx.rest.model.job.Component;
import de.superx.rest.model.job.JobExecutionStatus;
import de.superx.servlet.SuperXManager;
import de.superx.servlet.SxPools;
import de.superx.spring.HisInOneConfiguration;
import de.superx.spring.batch.His1DataSources;
import de.superx.spring.batch.tasklet.ExecuteSqlScriptTasklet;
import de.superx.spring.cli.config.CLIConfig;
import de.superx.spring.config.BatchConfig;
import de.superx.spring.config.DataJdbcConfiguration;
import de.superx.spring.config.ServiceConfig;
import de.superx.spring.service.BatchJobDescriptionAdapter;
import de.superx.spring.service.EntityJobDescriptionSource;
import de.superx.util.PathAndFileUtils;

/***
 * This class provides functionality to run the component actions of the
 * 'modernized component administration'
 * of the HISinOne-BI via command line. That includes
 * - listing ETL jobs
 * - installing components
 * - updating components
 * - deinstalling components
 * - running "Hauptladeroutine" ETL jobs
 * - running "Unterladeroutine" ETL jobs
 * The class is just meant to be a frontend, so it uses the same implementation as the web application
 * (which happens to be the "EtlJobApi" bean)
 * However, there are to be some things to be considered:
 * - The "jobLauncher" bean from the "BatchConfig" class works asynchronously, which is fine for the
 * web application context, but in the CLI context, it is overridden by a synchronous implementation
 * (in fact the tweaks to get the configurations ready for the CLI context, are placed in the
 * "CLIConfig" class).
 * - In the web application context, the HISinOne-BI code in the HISinOne web application writes the
 * database configuration to a file, which is then consumed by the "superx" application (the actual
 * HISinOne-BI application). The command line application needs this file too, so before using it, you
 * should start the web application one time, in order to have the file in place.
 * @author witt
 *
 */
public class ComponentAdminCLI {

    private static GenericApplicationContext APPLICATION_CONTEXT = null;

    private static String HELP_STRING = "Use this tool to run component actions via command line. "
                    + "It needs the config file 'his1_databases.properties' inside the classpath; "
                    + "this file gets written automatically when starting the web application.";

    private static boolean FILESYSTEM = false;
    private enum ExitStatus { SUCCESS, FAILURE };

    static Logger logger = Logger.getLogger(ComponentAdminCLI.class);

    public static void main(String[] args) {
        ExitStatus exitStatus = ExitStatus.SUCCESS;
        BasicConfigurator.configure(); // initializes console logging to stdout
        Logger.getRootLogger().setLevel(Level.INFO);
        System.setProperty(SuperXManager.SUPER_X_HISINONE_VERSION, "non-empty-value");
        Options options = createOptions();
        CommandLine parsedArgs = parseArgs(args, options);
        logOptions(parsedArgs);

        if (parsedArgs.hasOption("h")) {
            printHelp(options);
        } else { 

            // Optionen
            if(parsedArgs.hasOption("f")) {
                FILESYSTEM = true;
            } 
            initSuperXManager();
            if(parsedArgs.hasOption("s")) {
                initTablesForEmptyDB();
            } 
            if(parsedArgs.hasOption("m")) {
                ExecuteSqlScriptTasklet.FAIL_ON_MISSING_SCRIPT = true;
            } 
            if(parsedArgs.hasOption("lg")) {
                setBatchLoggerToOneFile();
            } 

            // Ausführung
            SuperXManager.initKettleEnv(createContext());
            if (parsedArgs.hasOption("la")) {
                printAllJobs();
            } else if (parsedArgs.hasOption("li")) {
                printInstallableJobs();
            } else if (parsedArgs.hasOption("le")) {
                printEtlJobs();
            } else if (parsedArgs.hasOption("i")) {
                exitStatus = installComponents(parsedArgs);
            } else if (parsedArgs.hasOption("d")) {
                exitStatus = deinstallComponent(parsedArgs);
            } else if (parsedArgs.hasOption("u")) {
                exitStatus = upgradeComponents(parsedArgs);
            } else if (parsedArgs.hasOption("ua")) {
                exitStatus = upgradeAll();
            } else if (parsedArgs.hasOption("e")) {
                exitStatus = etlJobs(parsedArgs);
            } else if (parsedArgs.hasOption("r")) {
                exitStatus = reloadModule();
            } else if (parsedArgs.hasOption("if")) {
                exitStatus = installFunctions(parsedArgs);
            } else {
                printHelp(options);
            }
        }
        System.exit(exitStatus.ordinal());
    }

    private static void logOptions(CommandLine parsedArgs) {
        logger.info("Starting  with the following options:");
        for (Option opt : parsedArgs.getOptions()) {
            logger.info(opt);
            if(opt.getValues() != null && opt.getValues().length > 0)  {
                logger.info("Values for option " + opt.getOpt() + ": " +  Arrays.asList(opt.getValues()));
            }
        }
    }

    private static void initTablesForEmptyDB() {
        try {
            GenericApplicationContext context = createContext();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            ContainerNode root = EntityJobDescriptionSource.getPreKernInstallJob();
            componentApi.executeJob("eduetl", root);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void setBatchLoggerToOneFile() {
        try {
            GenericApplicationContext context = createContext();
            BatchJobDescriptionAdapter bjda = context.getBean(BatchJobDescriptionAdapter.class);
            bjda.setLogJobToFile(false);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static ExitStatus installComponents(CommandLine parsedArgs) {
        String[] components = parsedArgs.getOptionValues("i");
        String currentComp = null;
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            for (String comp : components) {
                currentComp = comp;
                Long jobStartStatus = Long.valueOf(-1);
                if(!FILESYSTEM) {
                    logger.info("EXECUTING: install job from database");
                    jobStartStatus = componentApi.executeInstall(comp);
                } else {
                    // disable mondrian step
                    logger.info("EXECUTING: install job from filesystem for " + currentComp);
                    jobStartStatus = componentApi.executeInstallForQAMuster(comp);
                    logger.info("EXECUTING: workaround upgrade job from filesystem for " + currentComp);
                    jobStartStatus = componentApi.executeUpgradeForQAMuster(comp);
                }
                handleStartResult(jobStartStatus, componentApi);
                if(comp.equals("kern") && FILESYSTEM) {
                    DataSource dataSource = context.getBean(His1DataSources.class).get("eduetl");
                    SuperXManager.setWebInfFilePath(dataSource);
                }
            }
        } catch (BeansException be) {
            handleBeansException(be);
            return ExitStatus.FAILURE;
        } catch (Exception e) {
            handleJobException(e, currentComp);
            return ExitStatus.FAILURE;
        }
        return ExitStatus.SUCCESS;
    }

    private static ExitStatus reloadModule() {
    	try (GenericApplicationContext context = createContext()){
            initSxPools();
            initSuperXManager();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            componentApi.writeJobsToDb();
        } catch (Exception e) {
            logger.error("Error reloading modules:", e);
            return ExitStatus.FAILURE;
        }
    	return ExitStatus.SUCCESS;
    }

    private static void printHelp(Options options) {
        HelpFormatter help = new HelpFormatter();
        help.printHelp(HELP_STRING, options);
    }

    private static void printInstallableJobs() {
        try (GenericApplicationContext context = createContext()) {
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            List<Component> installJobs = componentApi.getInstallJobs();
            for (Component comp : installJobs) {
                printDetails(comp);
                System.out.println();
            }
        }
    }

    private static void printEtlJobs() {
        try (GenericApplicationContext context = createContext()) {
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            List<Component> etlJobs = componentApi.getEtlJobs();
            for (Component etlJob : etlJobs) {
                printDetails(etlJob);
                System.out.println();
            }
        }
    }

    private static void printDetails(Component comp) {
        System.out.println("abbrev:        " + comp.getAbbreviation());
        System.out.println("name:          " + comp.getName());
        System.out.println("database:      " + comp.getDatabase());
        System.out.println("systeminfo_id: " + comp.getSysteminfoId());
        System.out.println("installed:     " + comp.isInstalled());
    }

    private static ExitStatus deinstallComponent(CommandLine parsedArgs) {
        String comp = parsedArgs.getOptionValue("d");
        System.out.println(comp);
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            Long jobStartStatus = componentApi.executeUninstall(comp);
            handleStartResult(jobStartStatus, componentApi);
        } catch (BeansException be) {
            handleBeansException(be);
            return ExitStatus.FAILURE;
        } catch (Exception e) {
            handleJobException(e, comp);
            return ExitStatus.FAILURE;
        }
        return ExitStatus.SUCCESS;
    }

    private static ExitStatus upgradeComponents(CommandLine parsedArgs) {
        String[] components = parsedArgs.getOptionValues("u");
        String currentComp = null;
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            for (String comp : components) {
                currentComp = comp;
                Long jobStartStatus = componentApi.executeUpgrade(comp);
                if (jobStartStatus.longValue() == -1) {
                    logger.warn(comp + " not installed. Skipping upgrade.");
                    continue;
                }
                handleStartResult(jobStartStatus, componentApi);
            }
        } catch (BeansException be) {
            handleBeansException(be);
            return ExitStatus.FAILURE;
        } catch (Exception e) {
            handleJobException(e, currentComp);
            return ExitStatus.FAILURE;
        }
        return ExitStatus.SUCCESS;
    }
    
    private static ExitStatus installFunctions(CommandLine parsedArgs) {
        String[] components = parsedArgs.getOptionValues("if");
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            // No components given: install 
            // functions for all installed components
            if (components.length == 1 && "all".equals(components[0])) {
                List<Component> installedComponents = componentApi.getInstallJobs();
                components = installedComponents.stream().map(
                    c -> c.getAbbreviation()).collect(Collectors.toList()).toArray(new String[] {});
            }
            boolean exitFailure = false;
            for (String comp : components) {
                exitFailure = componentApi.installModuleFunctions(comp);
            }
            if(exitFailure) {
                logger.error(("Beim Ausführen einer Aktion ist ein Fehler aufgetreten."));
                return ExitStatus.FAILURE;
            }
        } catch (BeansException be) {
            handleBeansException(be);
                return ExitStatus.FAILURE;
        }
        return ExitStatus.SUCCESS;
    }

    private static ExitStatus upgradeAll() {
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            Long jobStartStatus = componentApi.executeUpgradeAll();
            handleStartResult(jobStartStatus, componentApi);
        } catch (BeansException be) {
            handleBeansException(be);
            return ExitStatus.FAILURE;
        } catch (Exception e) {
            handleJobException(e, null);
            return ExitStatus.FAILURE;
        }
        return ExitStatus.FAILURE;
    }
    
    private static ExitStatus etlJobs(CommandLine parsedArgs) {
        String[] jobIds = parsedArgs.getOptionValues("e");
        String currentJobId = null;
        try (GenericApplicationContext context = createContext()) {
            initSxPools();
            EtlJobApi componentApi = context.getBean(EtlJobApi.class);
            Long jobStartStatus;
            for (String jobId : jobIds) {
                try {
                    currentJobId = jobId;
                    if (isHauptladeroutine(jobId, componentApi)) {
                        jobStartStatus = componentApi.complete(jobId);
                    } else if (isLoadTransform(jobId, componentApi)) {
                        jobStartStatus = componentApi.load(jobId);
                    } else {
                        jobStartStatus = componentApi.executeJob(null, jobId);
                    }
                    handleStartResult(jobStartStatus, componentApi);
                } catch (Exception e) {
                    logger.error("ERROR executing job " + jobId, e);
                }
            }
        } catch (BeansException be) {
            handleBeansException(be);
            return ExitStatus.FAILURE;
        } catch (Exception e) {
            handleJobException(e, currentJobId);
            return ExitStatus.FAILURE;
        }
        return ExitStatus.SUCCESS;
    }

    private static void handleStartResult(Long jobStartStatus, EtlJobApi componentApi) {
        if (jobStartStatus.intValue() == -1) {
            System.err.println("Aktion konnte nicht gestartet werden: Es läuft bereits eine Aktion");
        }
        try {
            JobExecutionStatus es = componentApi.getStatus(jobStartStatus);
            if ("FAILED".equals(es.exitStatus.getExitCode())) {
                EtlJobApi.outputErrorSummary(es, System.err);
                System.exit(1);
            }
        } catch (Exception e) {
            System.err.println(("Beim Ausführen der Aktion ist ein Fehler aufgetreten:"));
            e.printStackTrace();
            System.exit(1);
        }
    }

    private static void handleJobException(Exception e, String jobName) {
        System.err.println("error while executing the job '" + jobName + "'");
        e.printStackTrace();
    }

    private static void handleBeansException(BeansException be) {
        System.err.println("configuration error or error with resolving the bean '" + EtlJobApi.class.getCanonicalName() + "'");
        be.printStackTrace();
    }

    private static boolean isHauptladeroutine(String comp, EtlJobApi etlJob) {
        Component comp_meta = etlJob.getComponentAndConnectorDetails(comp);
        return comp_meta != null && comp_meta.isDatabaseConnected();
    }

    private static boolean isLoadTransform(String comp, EtlJobApi etlJob) {
        Component comp_meta = etlJob.getComponentAndConnectorDetails(comp);
        return comp_meta !=null && !comp_meta.isDatabaseConnected();
    }
    
    private static void printAllJobs() {
        try (GenericApplicationContext context = createContext()) {
            EtlJobApi etlJob = context.getBean(EtlJobApi.class);
            List<ContainerNode> allJobs = etlJob.getAllJobs();
            for (ContainerNode cn : allJobs) {
                printDetails(cn);
                System.out.println();
            }
        }
    }

    private static GenericApplicationContext createContext() {
        /*
         * https://docs.spring.io/spring-framework/docs/current/javadoc-api/org/springframework/context/annotation/AnnotationConfigApplicationContext.html
         * quote:
         * "In case of multiple @Configuration classes, @Bean methods defined in later classes will override those defined in earlier classes.
         * This can be leveraged to deliberately override certain bean definitions via an extra @Configuration class."
         * - so it's alright to override some beans via "CLIConfig"
         */
        if (APPLICATION_CONTEXT == null) {
            APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(BatchConfig.class, DataJdbcConfiguration.class, CLIConfig.class, ServiceConfig.class);
            HisInOneConfiguration.configSuperXDbformsXML("pg", SuperXManager.getWEB_INFPfad() + File.separator + "..");
        }
        // Override the JobDescriptionSource Bean if the -f flag is passed.
        if(FILESYSTEM) {
            EtlJobApi etlJob = APPLICATION_CONTEXT.getBean(EtlJobApi.class);
            EntityJobDescriptionSource entityJobDescriptionSource = APPLICATION_CONTEXT.getBean(EntityJobDescriptionSource.class);
            etlJob.setJobDescriptionSource(entityJobDescriptionSource);
        }
        return APPLICATION_CONTEXT;
    }

    private static void printDetails(ContainerNode cn) {
        System.out.println("id:            " + cn.id);
        System.out.println("name:          " + cn.name);
        System.out.println("systeminfo_id: " + cn.systemInfoId);
        System.out.println("tid:           " + cn.tid);
        System.out.println("type:          " + cn.type);
    }

    private static Options createOptions() {
        Options options = new Options();
        Option opt;
        opt = new Option("h", "help", false, "get help");
        options.addOption(opt);
        opt = new Option("f", "filesystem", false, "load jobs from filesystem");
        options.addOption(opt);
        opt = new Option("s", "setup", false, "setup minimal list of tables necessary for kern install");
        options.addOption(opt);
        opt = new Option("m", "fail-on-missing-script", false, "Fail if a referenced SQL script is missing during execution");
        options.addOption(opt);
        opt = new Option("la", "list-all", false, "list all available components");
        options.addOption(opt);
        opt = new Option("li", "list-installables", false, "list all installable components");
        options.addOption(opt);
        opt = new Option("le", "list-etl", false, "list all etl components");
        options.addOption(opt);
        opt = new Option("i", "install", true, "install components");
        opt.setArgs(Option.UNLIMITED_VALUES);
        options.addOption(opt);
        opt = new Option("if", "install-functions", true, "install database functions for components (use 'all' to install functions for all installed components)");
        options.addOption(opt);
        opt.setArgs(Option.UNLIMITED_VALUES);
        opt = new Option("d", "deinstall", true, "de-/uninstall component");
        options.addOption(opt);
        opt = new Option("u", "upgrade", true, "upgrade components");
        opt.setArgs(Option.UNLIMITED_VALUES);
        options.addOption(opt);
        opt = new Option("ua", "upgrade-all", false, "upgrade all installed components");
        options.addOption(opt);
        opt = new Option("e", "etl", true, "run etl jobs");
        opt.setArgs(Option.UNLIMITED_VALUES);
        options.addOption(opt);
        opt = new Option("r", "reload-modules", false, "reload modules");
        options.addOption(opt);
        //opt = new Option("re", "reload-module-etl", true, "reload module etl");
        //options.addOption(opt);
        opt = new Option("db", "database", true, "database system");
        options.addOption(opt);
        opt = new Option("lg", "log-to-stdout", false, "log only to stdout and not to individual job files");
        options.addOption(opt);
        return options;
    }

    private static CommandLine parseArgs(String[] args, Options options) {
        CommandLineParser parser = new GnuParser();
        try {
            return parser.parse(options, args, false);
        } catch (ParseException e) {
            System.out.println("error while reading the command line parameters:");
            e.printStackTrace();
            System.exit(1);
        }
        return null;
    }
    
    // some actions require the SuperXManager, this is the place for initializing its static class attributes when needed
    private static void initSuperXManager() {
    	try {
    		SuperXManager.setWEB_INFPfad(PathAndFileUtils.getWebinfPath());
    		SuperXManager.setModuleDir(PathAndFileUtils.getWebinfPath() + File.separator + PathAndFileUtils.MODULE_PATH);
    	} catch(Exception e) {
            System.out.println("error while initialising the SuperXManger:");
            e.printStackTrace();
            System.exit(1);
    	}
    }

    // sxPools need to be initialized, because spring batch ETL uses them to look up the database connections
    private static void initSxPools() {
        try {
            List<String> mandantenNamen = new LinkedList<String>();
            mandantenNamen.add(DEFAULT_MANDANTEN_ID);
            SxPools.closeAll();
            SxPools.init(mandantenNamen);
            SxPools.get(DEFAULT_MANDANTEN_ID).init();
            SxPools.get(DEFAULT_MANDANTEN_ID).initLogging(true, Level.DEBUG);
            // also init kettle env, set plugin dir
            SuperXManager.initKettleEnv(APPLICATION_CONTEXT);
        } catch (Exception e) {
            System.out.println("error while initialising the SuperX pools:");
            e.printStackTrace();
            System.exit(1);
        }
    }

}