kern/src/de/superx/bianalysis/metadata/MetadataImporter.java

package de.superx.bianalysis.metadata;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

import javax.sql.DataSource;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.log4j.Logger;
import org.springframework.jdbc.core.JdbcTemplate;

import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;

import de.superx.bianalysis.FaultyMetadataException;
import de.superx.bianalysis.StoredReport;
import de.superx.bianalysis.metadata.models.json.MetaDimension;
import de.superx.bianalysis.metadata.models.json.MetaDimensionAttribute;
import de.superx.bianalysis.metadata.models.json.MetaFact;
import de.superx.bianalysis.metadata.models.json.MetaObject;
import de.superx.bianalysis.metadata.models.yml.MetaYml;
import de.superx.bianalysis.metadata.models.yml.MetaYmlModel;
import de.superx.bianalysis.metadata.models.yml.MetaYmlModelColumns;
import de.superx.bianalysis.service.DbMetaAdapter;
import de.superx.util.PathAndFileUtils;

/**
 * Provides functionality for updating the tables in the metadata schema.
 * The tables are updated by reading the metadata information from various
 * metaimport.json files and transforming that information into executable sql.
 *
 * The BIAnalysis Tool uses the tables to read information about the different
 * meta objects and more importantly to figure out their relationships, e.g. what
 * dimension is part of which facttable or which attribute belongs to which 
 * dimension.
 * 
 * To learn more about the metadata concept for the BIAnalysis Tool see: 
 * doc\bi_analysis\metadaten.adoc 
 * 
 */
public final class MetadataImporter {

	/**
	 * Each file containing metadata information must have the following file suffix.
	 */
    private static final String METAIMPORT_FILE_SUFFIX = "_metaimport.json";

    protected static final String CONFORMED_DIMENSIONS_FILE_SUFFIX = "conformed_dimensions" + METAIMPORT_FILE_SUFFIX;

	/**
	 * Holds all Metaimport objects with which this instance was initalized.
	 * (One MetaImport object corresponds to exactly one deserialized json file)
	 */
	private List<MetaJson> metaImports = new ArrayList<>();

	public List<String> errorMessages = new ArrayList<>();

	/**
	 * SQL String for deleting from all metadata tables except 'custom' releases. 
	 */
	public static final String TRUNCATE_METADATA_SQL = 
	        "DELETE FROM metadata.facttable WHERE is_custom = false; " +
	        "DELETE FROM metadata.measure WHERE is_custom = false; " +
	        "DELETE FROM metadata.measure_filter WHERE is_custom = false; " +
	        "DELETE FROM metadata.dimension WHERE is_custom = false; " +
	        "DELETE FROM metadata.dimension_attribute WHERE is_custom = false; ";

	private static Logger log = Logger.getLogger(MetadataImporter.class);

	private boolean shouldReadYMLDoc = true;
	
	private String ymlDir = "";

    public MetadataImporter() {}
    
    public MetadataImporter(String ymlDir) {this.ymlDir = ymlDir;}

    public void deserializeMetadataFromStrings(String... values) {
        ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
        List<MetaImportConformedDimensions> conformedDimension = new ArrayList<>();

        for (String value : values) {
                MetaJson meta = null;
                try{
                    if(value.contains("conformed_dimensions")) {
                        meta = mapper.readValue(value, MetaImportConformedDimensions.class);
                        conformedDimension.add((MetaImportConformedDimensions) meta);
                    } else {
                        meta = mapper.readValue(value, MetaImport.class);
                    }
                } catch(Exception e) {
                    throw(new RuntimeException(e));
                }
                if(meta != null) {
                    meta.setFile(null);
                    metaImports.add(meta);
                }
        }

        // gather all conformed dimensions
        List<MetaDimension> confDims = new ArrayList<>();
        for (MetaImportConformedDimensions conf : conformedDimension) {
            confDims.addAll(conf.conformedDimensions);
        }

        // resolve conformed references ('ref_to' attributes)
        for (MetaJson metaJson : metaImports) {
            if (conformedDimension.size() > 0  && metaJson instanceof MetaImport) {
                ((MetaImport) metaJson).setConformedDimensions(confDims);
            }
            try {
                metaJson.init();
                metaJson.setNamespaceToMetaObjects();
            } catch (Exception e) {
                errorMessages.add(ExceptionUtils.getFullStackTrace(e));
            }
        }

        if(shouldReadYMLDoc) {
            addDescriptionsFromYMLFiles();
        }
    }

    /**
     * Calling this method initalizes the MetadataImporter by deserializing all unique meta objects
     * from the provided json files. Faulty json files are ignored.
     * 
     * @param paths Path(s) to the metadata file(s). A path can point to a directory or a file.
     *              Multiple paths and/or directories can be provided.
     */
    public void deserializeMetadataFromJsonFiles(String... paths) {

        ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
        List<MetaImportConformedDimensions> conformedDimension = new ArrayList<>();

        for (String path : paths) {
            List<File> metaFiles = readMetaImportFiles(path);
            for (File file : metaFiles) {
                MetaJson meta = null;
                try{
                    if(file.getName().endsWith(CONFORMED_DIMENSIONS_FILE_SUFFIX)) {
                        meta = mapper.readValue(file, MetaImportConformedDimensions.class);
                        conformedDimension.add((MetaImportConformedDimensions) meta);
                    } else {
                        meta = mapper.readValue(file, MetaImport.class);
                    }
                } catch(JsonMappingException e) {
                    String message = "Could not deserialize metadata from file: " + file.getName() + "\n";
                    message += e.getMessage();
                    errorMessages.add(message);
                } catch(Exception e) {
                    errorMessages.add(ExceptionUtils.getFullStackTrace(e));
                }
                if(meta != null) {
                    log.info("Read metadata from file: " + file.getName());
                    meta.setFile(file);
                    metaImports.add(meta);
                }
            }
        }

        // gather all conformed dimensions
        List<MetaDimension> confDims = new ArrayList<>();
        for (MetaImportConformedDimensions conf : conformedDimension) {
            confDims.addAll(conf.conformedDimensions);
        }

        // resolve conformed references ('ref_to' attributes)
        for (MetaJson metaJson : metaImports) {
            if (conformedDimension.size() > 0  && metaJson instanceof MetaImport) {
                ((MetaImport) metaJson).setConformedDimensions(confDims);
            }
            try {
                metaJson.init();
                metaJson.setNamespaceToMetaObjects();
            } catch (Exception e) {
                errorMessages.add(ExceptionUtils.getFullStackTrace(e));
            }
        }

        if(shouldReadYMLDoc) {
            addDescriptionsFromYMLFiles();
        }

    }

    public List<String> readStoredReports() {
        List<String> result = new ArrayList<>();
        try {
            String dir = PathAndFileUtils.getStoredReportDir("hisinone");
            File[] files = new File(dir).listFiles();
            if(files == null) {
               return result; 
            }
            for (File file : files) {
                try {
                    ObjectMapper mapper = JsonMapper.builder().findAndAddModules().build();
                    StoredReport report = mapper.readValue(file, StoredReport.class);
                    UpsertStringBuilder builder = new UpsertStringBuilder()
                        .forTable("metadata", "rw_report_definitions")
                        .withIntCol("id", Integer.valueOf(report.id))
                        .withStringCol("name", report.name)
                        .withStringCol("definition", report.definition)
                        .withIntCol("show_total_column", Integer.valueOf(report.showTotalColumn));
                    result.add(builder.build(true));
                } catch (JsonMappingException e) {
                    String message = "Could not deserialize stored report from file: " + file.getName() + "\n";
                    message += e.getMessage();
                    errorMessages.add(message);
                }
            }
            // After inserting the stored reports with a fixed id we need to re-sync the
            // id column of the rw_report_definitions table
            if(result.size() != 0) {
                result.add("SELECT setval(pg_get_serial_sequence('metadata.rw_report_definitions', 'id'),"
                                    + "(SELECT max(id) FROM metadata.rw_report_definitions ));");
            }
        } catch(Exception e) {
            errorMessages.add("Unable to read stored report:\n");
            errorMessages.add(ExceptionUtils.getFullStackTrace(e));
        }
        return result;
    }

    public void addDescriptionsFromYMLFiles() {
        String dir = ymlDir;
        if(ymlDir == null || ymlDir.isBlank()) {
            dir = PathAndFileUtils.getDbtModelsDirectory("hisinone");
        }
        HashMap<String, String> map = getMarkdownDefinitions(dir);
        addYMLDescriptionsToMetaObjects(dir, map);
    }

    public void addYMLDescriptionsToMetaObjects(String ymlDir, HashMap<String, String> mdDefs){
        log.info("Adding descriptions from yml files");
        HashMap<String, String> descriptions = createDescriptions(new File(ymlDir), mdDefs);
        List<MetaObject> objs = getAllMetaObjectsWithConformed();
        for (MetaObject metaObj : objs ) {
            String docIdentifier = null;
            try {
                docIdentifier = metaObj.getDocIdentifier();
            } catch (FaultyMetadataException e) {
                errorMessages.add(ExceptionUtils.getFullStackTrace(e));
            }
            if(docIdentifier == null || docIdentifier.isBlank()) {
               continue;
            }
            // only use yml doc if json description does not exist 
            if(metaObj.getDescription() == null || metaObj.getDescription().isBlank()) {
                String desc = descriptions.get(docIdentifier);
                if(desc == null) {
                   log.warn("Missing yml description for: " + docIdentifier);
                } else {
                    metaObj.setDescription(desc); 
                    if(desc.isBlank()) {
                       log.warn("Empty yml description for MetaObject: " + docIdentifier);
                    }
                }
            }
        }
    }

    private HashMap<String, String> createDescriptions(File startDir, HashMap<String, String> mdDefs){
        HashMap<String, String> result = new HashMap<>();
        for (MetaYml yml : getDescriptionYMLs(startDir)) {
            for (MetaYmlModel model : yml.getModels()) {
                String modelName = model.getName();
                String modelDesc = model.getDescription();
                result.put(modelName, getDescription(modelDesc, mdDefs));
                for (MetaYmlModelColumns column : model.getColumns()) {
                    String colName = column.getName();
                    String colDesc = column.getDescription();
                    result.put(modelName + "." + colName, getDescription(colDesc, mdDefs));
                }
            }
        }
        return result;
    }

    private static String getDescription(String desc, HashMap<String, String> mdDefs) {
        if(desc == null) {
           return ""; 
        }
        if(desc.startsWith("{{")) {
            String[] parts = desc.split("\"");
            String docRef = parts[1];
            return mdDefs.get(docRef);
        }
        return desc;
    }

    private List<MetaYml> getDescriptionYMLs(File startDir){
        List<MetaYml> ymls = new ArrayList<>();
        List<String> files = getFiles(startDir, "", ".yml");
        ObjectMapper mapperYml = new ObjectMapper(new YAMLFactory());
        for (String f : files) {
            File file = new File(startDir + File.separator + f);
            MetaYml doc = null;
            try {
                doc = mapperYml.readValue(file, MetaYml.class);
            } catch (Exception e) {
                String message = "Could not read documentation from file: " + file.getName() + "\n";
                errorMessages.add(message);
                errorMessages.add(ExceptionUtils.getFullStackTrace(e));
            }
            if(doc != null) {
                log.info("Read documentation from file: " + file.getName());
                ymls.add(doc);
            }
        }
        return ymls;
    }


    /**
     * Gathers all metadata json files.
     * 
     * @param path A path to a metadata json file or a directory containing metadata json files.
     * @return A list of files matching the metadata json suffix.
     */
    private static List<File> readMetaImportFiles(String path) {
        File metaimportPath = new File(PathAndFileUtils.getDbtJsonPath(path));
        List<File> metaimportFiles = new ArrayList<>();
        if (metaimportPath.isDirectory()) {
            metaimportPath.list(new FilenameFilter() {
                @Override
                public boolean accept(File dir, String name) {
                    if (name.endsWith(METAIMPORT_FILE_SUFFIX)) {
                        File file = new File(dir.getAbsolutePath() + File.separator + name);
                        metaimportFiles.add(file);
                        return true;
                    }
                    return false;
                }
            });
        } else {
            metaimportFiles.add(metaimportPath);
        }
        return metaimportFiles;
    }

    private static List<String> getFiles(File startDir, String subDir, String extension) {
        List<String> filtered = new ArrayList<String>();
        for (File file : startDir.listFiles()) {
            String name = file.getName();
            if(file.isDirectory()) {
                filtered.addAll(getFiles(file, subDir + File.separator + name, extension));
            }
            String filename = name.strip().toLowerCase();
            if(filename.endsWith(extension)) {
                filtered.add(subDir + File.separator + name);
            }
        }
        return filtered;
    }

    /*

    /**
     * Generates the sql upsert strings for all unique, deserialized MetaObjects.
     * 
     * @param hasOnConflictConstruct If set to true generates upsert strings with the postgres-specific "ON CONFLICT" clause.
     * @return All the generated upserts from the metadata files.
     */
    public List<String> getAllUpsertStrings(boolean hasOnConflictConstruct) {

        List<String> upsertStmts = new ArrayList<>();
        List<Identifier> ids = new ArrayList<>();

        for (MetaJson meta : metaImports) {
            for(MetaObject obj: meta.getMetaObjects()) {
                Identifier id = obj.getId();
                if(id == null) {
                    String message = String.format("Missing ID for Element '%s' in file: %s.", obj.getCaption(), meta.getFile().getAbsolutePath());
                    errorMessages.add(message);
                    continue;
                }
                if(ids.contains(id)) {
                    String message = String.format("Duplicate ID '%s'. Ignoring Element '%s'.", obj.getCaption(), obj.getId().composedId);
                    errorMessages.add(message);
                    continue;
                }
                ids.add(obj.getId());
                String stmt = obj.getUpsertBuilder().build(hasOnConflictConstruct);
                upsertStmts.add(stmt);
            }
        }
        return upsertStmts;
    }

    public void updateMetadataForH2Database(DataSource dataSource) throws Exception {
		String metaFilesDir = String.join(File.separator, new String[] {"test", "resources", "db", "fixtures", "bianalysis", "metadata"}); 
		deserializeMetadataFromJsonFiles(metaFilesDir);
	  	JdbcTemplate jt = new JdbcTemplate(dataSource);
		String upserts = String.join("\n", getAllUpsertStrings(false).toString());
        jt.execute(upserts);
	}

	/**
	 * Updates tables in the metadata schema.
	 * 
	 * @param metaPath Location of the metadata file or directory.
	 * @param dataSource The datasource on which the sql is executed.
	 * @throws Exception
	 */
    public void updateMetadataSchema(String project, DataSource dataSource) throws Exception {
        String metaFilesDir = PathAndFileUtils.getReportGeneratorDir(project);
        deserializeMetadataFromJsonFiles(metaFilesDir);
        
        try (Connection con = dataSource.getConnection()) {
            try (Statement st = con.createStatement()) {
                log.info("Update Metadata for BIAnalysis.");
                st.execute(TRUNCATE_METADATA_SQL);
                List<String> upserts = getAllUpsertStrings(true);
                upserts.addAll(readStoredReports());
                for (String sql : upserts) {
                    log.info(sql);
                    try (Statement stUpsert = con.createStatement()) {
                        stUpsert.execute(sql);
                    } catch (Exception e) {
                        throw e;
                    }
                }
            }
            
            
            // execute sql in "attributes_sql" in metadata files to build the attributes
            // dynamically 

            List<Identifier> ids = getAllIds();
            
            List<MetaDimensionAttribute> generatedAttributes = new ArrayList<>();
            List<MetaDimensionAttribute> co = new ArrayList<>();

            for (MetaJson i : this.metaImports) {
                for (MetaObject obj : i.getMetaObjects()) {
                    if (!(obj instanceof MetaDimension)) continue;
                    MetaDimension dim = (MetaDimension) obj;
                    
                    String attributesSql = "";
                    if (dim.getAttributesSql() != null && !dim.getAttributesSql().isEmpty()) {
                        attributesSql = dim.getAttributesSql();
                    } else if((dim.getRefTo() != null && dim.getConformedDimension().getAttributesSql() != null)) {
                       attributesSql = dim.getConformedDimension().getAttributesSql(); 
                    } else {
                        continue;
                    }
                    
                    try (Statement stAttr = con.createStatement(); ResultSet rs = stAttr.executeQuery(attributesSql)) {
                        int numAttributes = 0;
                        while (rs.next()) {

                            MetaDimensionAttribute attribute = new MetaDimensionAttribute();
                            attribute.setDimension(dim);
                            attribute.setCaption(rs.getString("caption"));
                            attribute.setDimColumn(rs.getString("dim_column"));
                            attribute.setDefaultRelease(DbMetaAdapter.getCustomDefaultReleaseForCurrentDate());
                            attribute.setPosition(Integer.valueOf(numAttributes));

                            // create a new 'on the fly' identifier for the new metadata
                            // attribute
                            Identifier id = Identifier.getNewIdentifierValue(ids, dim.getNamespace());
                            ids.add(id);
                            Integer val = Integer.valueOf(id.value.intValue() + numAttributes);
                            attribute.setId(new Identifier(dim.getNamespace() + ":" +val));
                            attribute.setNamespace(id.namespace);
                            numAttributes++;

                            generatedAttributes.add(attribute);

                            if(attribute.getDimension().getConformedDimension() == null) {
                                co.add(attribute);
                            }
                        }
                    } 
                }
            }
            
            // TODO figure out better solution for orgunits
            for (MetaDimensionAttribute metaDimensionAttribute : generatedAttributes) {
                if(metaDimensionAttribute.getDimension().getConformedDimension() != null) {
                    for (MetaDimensionAttribute confA : co) {
                        if(confA.getDimColumn().equals(metaDimensionAttribute.getDimColumn())) {
                            metaDimensionAttribute.setConfDimAttrRef(confA);
                        }
                    } 
                }
                String stmt = metaDimensionAttribute.getUpsertBuilder().build(true);
                try (Statement stUpsert = con.createStatement()) {
                    stUpsert.execute(stmt);
                }
            }

        }
    }
    
    
    public static void writeYmlToFile(MetaYml yml, File file) {
        YAMLFactory yf = new YAMLFactory()
            .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)
            .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER);
        ObjectMapper mapper = new ObjectMapper(yf);
        DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter("  ", DefaultIndenter.SYS_LF);
        DefaultPrettyPrinter printer = new DefaultPrettyPrinter();
        printer.indentObjectsWith(indenter);
        printer.indentArraysWith(indenter);
        try {
            mapper.writer(printer).writeValue(file, yml);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

	public static String writeYmlToString(MetaYml yml) {
	    YAMLFactory yf = new YAMLFactory()
	            .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)
	            .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER);
       	ObjectMapper mapper = new ObjectMapper(yf);
       	DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter("  ", DefaultIndenter.SYS_LF);
       	DefaultPrettyPrinter printer = new DefaultPrettyPrinter();
       	printer.indentObjectsWith(indenter);
       	printer.indentArraysWith(indenter);
		try {
			//mapper.writer(printer).writeValue(file, yml);
			return mapper.writer(printer).writeValueAsString(yml);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	public String getPrintableErrorMessages() {
	    String output = "";
        if(!errorMessages.isEmpty()) {
            output += "The following errors occured:\n";
            for (String message : errorMessages) {
               output += message + "\n";
            }
        }
        return output;
	}

	public HashMap<String, String> getMarkdownDefinitions(String ymlDir) {
        List<String> files = getFiles(new File(ymlDir), "", ".md");
        HashMap<String, String> map = new HashMap<>();
        for (String file : files) {
            try {
                try (BufferedReader br = new BufferedReader(new FileReader(ymlDir + File.separator + file))) {
                    String line;
                    String key = null;
                    boolean readHeading = false;
                    while ((line = br.readLine()) != null) {
                        if (line.startsWith("{% docs ")) {
                            int startIndex = line.indexOf("docs"); // find "docs"
                            if(startIndex == -1) {
                                throw new RuntimeException("docs not found");
                            }
                            startIndex += "docs".length(); // move start index to after "docs"
                            int endIndex = line.indexOf('%', startIndex); // find next % starting from "docs"
                            if(endIndex == -1) {
                                throw new RuntimeException("No % found after docs");
                            }
                            key = line.substring(startIndex, endIndex).trim();
                            map.put(key, "");
                        } else if(key != null && line.startsWith("# ")) {
                            readHeading = true;
                        } else {
                            if(readHeading && !line.isBlank()) {
                                map.put(key, line);
                                key = null;
                                readHeading = false;
                            }
                        }
                    }
                }
            } catch (Exception e) {
                String message = "ERROR getting markdown definitions from file: " + file + "\n";
                errorMessages.add(message);
                errorMessages.add(ExceptionUtils.getFullStackTrace(e));
            }
        }
        return map;
	}

	public Optional<MetaImport> getMetaImport(String fileName) {
        return metaImports.stream()
                .filter(json -> (json instanceof MetaImport) && json.file.getName().equals(fileName))
                .map(json -> (MetaImport) json)
                .findFirst();
    }
	
	   public Optional<MetaJson> getMetaJson(String fileName) {
	        return metaImports.stream()
	                .filter(json -> json.file.getName().equals(fileName))
	                .findFirst();
	    }

	public List<MetaImport> getMetaImports() {
        return metaImports.stream()
                .filter(json -> (json instanceof MetaImport))
                .map(json -> (MetaImport) json)
                .collect(Collectors.toList());
    }

	public List<MetaImportConformedDimensions> getMetaImportsConformed() {
        return metaImports.stream()
                .filter(json -> (json instanceof MetaImportConformedDimensions))
                .map(json -> (MetaImportConformedDimensions) json)
                .collect(Collectors.toList());
    }

	public List<MetaJson> getMetaJsons() {
        return metaImports.stream().collect(Collectors.toList());
    }

	public List<MetaObject> getAllMetaObjects(){
        return metaImports.stream()
                .filter(json -> (json instanceof MetaImport))
                .map(meta -> ((MetaImport) meta).getMetaObjects())
                .flatMap(List::stream)
                .collect(Collectors.toList());
	}

	public List<MetaObject> getAllMetaObjectsWithConformed(){
        return metaImports.stream()
                .map(meta -> meta.getMetaObjects())
                .flatMap(List::stream)
                .collect(Collectors.toList());
	}

	public List<MetaFact> getAllFactTables(){
        return metaImports.stream()
                .filter(json -> (json instanceof MetaImport))
                .map(meta -> ((MetaImport) meta).facts)
                .flatMap(List::stream)
                .collect(Collectors.toList());
	}

	public void setShouldReadYMLDoc(boolean shouldReadYMLDoc) {
        this.shouldReadYMLDoc = shouldReadYMLDoc;
    }

    public List<Identifier> getAllIds() {
        return metaImports.stream()
                .map(meta -> meta.getIds())
                .flatMap(List::stream)
                .collect(Collectors.toList());
    }


}