You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
347 lines
14 KiB
347 lines
14 KiB
package de.superx.servlet; |
|
|
|
import static org.junit.Assert.assertEquals; |
|
import static org.junit.Assert.assertTrue; |
|
import static org.junit.Assert.fail; |
|
|
|
import java.io.File; |
|
import java.io.IOException; |
|
import java.nio.file.Files; |
|
import java.nio.file.Path; |
|
import java.nio.file.StandardCopyOption; |
|
import java.sql.Connection; |
|
import java.sql.DriverManager; |
|
import java.sql.ResultSet; |
|
import java.sql.SQLException; |
|
import java.sql.Statement; |
|
import java.util.Iterator; |
|
|
|
import org.apache.commons.io.FileUtils; |
|
import org.junit.AfterClass; |
|
import org.junit.BeforeClass; |
|
import org.junit.Test; |
|
|
|
/** |
|
* |
|
* Tests für CSV Updater - andere Headerreihenfolge als Zieltabelle geht nicht - |
|
* weniger Spalten als in Zieltabelle geht nicht |
|
* |
|
* @author MB |
|
* |
|
*/ |
|
public class CSVUploaderTest { |
|
private static Connection con; |
|
private static String tmpDbPath; |
|
String mandantenId = "default"; |
|
boolean isXml = false; |
|
private String userid="4"; |
|
String tabelle = "tmp_csv_test"; |
|
|
|
String delim = "^"; |
|
String encoding = "ISO-8859-1"; |
|
|
|
String insertMode = "false"; |
|
|
|
@BeforeClass |
|
public static void setUp() throws Exception { |
|
System.out.println("SETUP"); |
|
Class.forName("org.h2.Driver"); |
|
tmpDbPath = System.getProperty("java.io.tmpdir") + File.separator + "csv_updater_test"; |
|
File srcDb = new File("test/resources/db/head_eduetl.mv.db"); |
|
if (!srcDb.exists()) { |
|
fail("eduetl src Test-DB not found: " + srcDb); |
|
} |
|
Files.copy(srcDb.toPath(), (new File(tmpDbPath + ".mv.db")).toPath(), |
|
StandardCopyOption.REPLACE_EXISTING); |
|
con = DriverManager |
|
.getConnection("jdbc:h2:" + tmpDbPath + ";MODE=POSTGRESQL;DATABASE_TO_LOWER=TRUE;NON_KEYWORDS=KEY,VALUE", |
|
"sa", ""); |
|
try (Statement stm = con.createStatement()) { |
|
stm.executeUpdate("create table tmp_csv_test (hsnr integer,stugkey varchar(10),value double precision)"); |
|
stm.executeUpdate("create table tmp_csv_hsnr (hsnr integer,name varchar(100))"); |
|
stm.executeUpdate("insert into tmp_csv_hsnr values (1000,'testhochschule');"); |
|
stm.executeUpdate("create table tmp_csv_stugkey (stugkey varchar(10),name varchar(100))"); |
|
stm.executeUpdate("insert into tmp_csv_stugkey values ('100','test 100');"); |
|
stm.executeUpdate("insert into tmp_csv_stugkey values ('101','test 101');"); |
|
stm.executeUpdate("insert into tmp_csv_stugkey values ('102','test 102');"); |
|
String sql = "delete from sx_fields where table_name='tmp_csv_test'; \n" |
|
+ "insert into sx_fields (tid,table_name,name,field_type,field_size,field_not_null,foreignkey_tab,foreignkey_col) values (-1,'tmp_csv_test','hsnr','INTEGER',10,0,'tmp_csv_hsnr','hsnr');\n"+ |
|
"insert into sx_fields (tid,table_name,name,field_type,field_size,field_not_null,foreignkey_tab,foreignkey_col) values (-2,'tmp_csv_test','stugkey','VARCHAR',10,0,'tmp_csv_stugkey','stugkey')"; |
|
stm.executeUpdate(sql); |
|
stm.executeUpdate("CREATE TABLE tmp_escape_test (text_col_1 TEXT, text_col_2 TEXT);"); |
|
stm.executeUpdate("CREATE TABLE tmp_serial_test (id SERIAL, text_col_1 TEXT, text_col_2 TEXT);"); |
|
} |
|
|
|
} |
|
|
|
private static File writeTempFile(String prefix, String suffix, String content) throws IOException { |
|
Path p = Files.createTempFile(prefix, suffix); |
|
File f = p.toFile(); |
|
f.deleteOnExit(); |
|
FileUtils.writeStringToFile(f, content); |
|
return f; |
|
} |
|
|
|
private static void dropTempTable() throws SQLException { |
|
try (Statement stm = con.createStatement()) { |
|
stm.executeUpdate("drop table if exists tmp_tmp_csv_test"); |
|
} |
|
} |
|
|
|
@Test |
|
public void testOK1() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testok","csv", |
|
"hsnr^stugkey^value\n1000^100^1.2\n1000^101^2.2\n1000^102^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid, this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
} |
|
|
|
@Test |
|
public void xxxtestNichtAlleSpalten() throws IOException, |
|
SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testnichtallespalten","csv", |
|
"hsnr^stugkey\n1000^100\n1000^101\n1000^102").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
CSVResultReport rr = cu.getResultReport(); |
|
for (Iterator<String> it = rr.getMessages().iterator(); it.hasNext();) { |
|
System.out.println(it.next()); |
|
} |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
} |
|
|
|
@Test |
|
public void xxxtestOKColSort() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testokcolsort","csv", |
|
"value^hsnr^stugkey\n1.1^1000^100\n2.2^1000^100\n3.3^1000^102").getAbsolutePath(); |
|
|
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
CSVResultReport rr = cu.getResultReport(); |
|
for (Iterator<String> it = rr.getMessages().iterator(); it.hasNext();) { |
|
System.out.println(it.next()); |
|
} |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
} |
|
|
|
@Test |
|
public void testOK2() throws IOException, SQLException { |
|
boolean withHeader = false; |
|
String dateiPfad = writeTempFile("testok2","csv", |
|
"1000^100^1.2\n1000^101^2.3\n1000^102^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
CSVResultReport rr = cu.getResultReport(); |
|
for (Iterator<String> it = rr.getMessages().iterator(); it.hasNext();) { |
|
System.out.println(it.next()); |
|
} |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
} |
|
|
|
@Test |
|
public void testTooManyHeaders() throws IOException, |
|
SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testtoomanyheaders","csv", |
|
"hsnr^stugkey^value^value2\n1000^100^1.2\n1000^101^2.2\n1000^102^3.3").getAbsolutePath(); |
|
|
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
boolean isIOException = false; |
|
try { |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
} catch (Exception e) { |
|
isIOException = true; |
|
} finally { |
|
dropTempTable(); |
|
} |
|
assertTrue(isIOException); |
|
|
|
} |
|
|
|
@Test |
|
public void testTooManyDelim() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testtoomanydelim","csv", |
|
"hsnr^stugkey^value\n1000^100^1.2\n1000^101^2.2^\n1000^102^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
assertEquals(1, cu.getResultReport().getErrorCount()); |
|
/* System.out.println(cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(0) |
|
.toString()); |
|
*/ |
|
int pos = cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(0) |
|
.toString() |
|
.indexOf( |
|
"#Error in Load-File Record 2: CSV-Datensatz hat 4 Spalten, aber Zieltabelle hat 3 Spalten."); |
|
|
|
assertTrue(pos > -1); |
|
} |
|
|
|
@Test |
|
public void testEscaping() throws IOException, SQLException { |
|
boolean withHeader = false; |
|
String dateiPfad = writeTempFile("testescape","csv", |
|
"bla\\^bla^bla").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, "tmp_escape_test", dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
Statement stmt = CSVUploaderTest.con.createStatement(); |
|
ResultSet rs = stmt.executeQuery("SELECT * FROM tmp_tmp_escape_test;"); |
|
while (rs.next()) { |
|
String inserted = rs.getString(1); |
|
assertEquals("String with escaped separator should be found in target table.", "bla^bla", inserted); |
|
} |
|
} |
|
|
|
@Test |
|
public void testSerial() throws IOException, SQLException { |
|
boolean withHeader = false; |
|
String dateiPfad = writeTempFile("testserial","csv", |
|
"10^wert_1^wert_2").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, "tmp_serial_test", dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
assertEquals(0, cu.getResultReport().getErrorCount()); |
|
Statement stmt = CSVUploaderTest.con.createStatement(); |
|
ResultSet rs = stmt.executeQuery("SELECT * FROM tmp_tmp_serial_test;"); |
|
while (rs.next()) { |
|
String inserted = rs.getString(1); |
|
assertEquals("Field in 'temp' table that corresponds to SERIAL/AUTO_INCREMENT in target table, should be ignored in INSERTs", null, inserted); |
|
} |
|
} |
|
|
|
// TODO: Disabled because foreign keys have to be correctly defined in database! |
|
// @Test |
|
public void testFK() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testfk","csv", |
|
"hsnr^stugkey^value\n100^100^1.2\n1000^1010^2.2\n2000^1020^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
|
|
if (cu.getResultReport().getMessages().size() > 0) |
|
System.out.println(cu.getResultReport().getMessages().get(2) |
|
.toString()); |
|
|
|
assertEquals(3, cu.getResultReport().getErrorCount()); |
|
|
|
int pos = cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(0) |
|
.toString() |
|
.indexOf( |
|
"Error in Load-File Line 1: Error hsnr (Spalte 1) enthält Wert (100), der nicht in foreign keys vorkommt"); |
|
assertTrue(pos > -1); |
|
pos= |
|
cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(1) |
|
.toString() |
|
.indexOf( |
|
"Error in Load-File Line 2: Error stugkey (Spalte 2) enthält Wert (1010), der nicht in foreign keys vorkommt"); |
|
assertTrue(pos > -1); |
|
|
|
pos= |
|
cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(2) |
|
.toString() |
|
.indexOf( |
|
"Error in Load-File Line 3: Error hsnr (Spalte 1) enthält Wert (2000), der nicht in foreign keys vorkommt Error stugkey (Spalte 2) enthält Wert (1020), der nicht in foreign keys vorkommt"); |
|
assertTrue(pos > -1); |
|
|
|
// |
|
|
|
|
|
} |
|
|
|
// TODO: Disabled because foreign keys have to be correctly defined in database! |
|
// @Test |
|
public void testDatatype2() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testdatatype2","csv", |
|
"hsnr^stugkey^value\n1000^100^1.2\n1000^101^2,2\n1000^102^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
/* |
|
* if (cu.getResultReport().getMessages().size() > 0) |
|
* System.out.println(cu.getResultReport().getMessages().get(0) |
|
* .toString()); |
|
*/ |
|
assertEquals(1, cu.getResultReport().getErrorCount()); |
|
|
|
int pos = cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(0) |
|
.toString() |
|
.indexOf( |
|
"Error in Load-File Line 2: Error in Column 3: java.lang.NumberFormatException: For input string: \"2,2\""); |
|
assertTrue(pos > -1); |
|
|
|
} |
|
|
|
// TODO: Disabled because foreign keys have to be correctly defined in database! |
|
// @Test |
|
public void testDatatype1() throws IOException, SQLException { |
|
boolean withHeader = true; |
|
String dateiPfad = writeTempFile("testdatatype1","csv", |
|
"hsnr^stugkey^value\n1000x^100^1.2\n1000^101^2.2\n1000^102^3.3").getAbsolutePath(); |
|
CSVUploader cu = new CSVUploader(this.userid,this.mandantenId, this.tabelle, dateiPfad, |
|
this.delim, this.encoding, withHeader, this.insertMode, this.isXml, "exclude-row", false); |
|
cu.upload(true, true, CSVUploaderTest.con); |
|
dropTempTable(); |
|
// if (cu.getResultReport().getMessages().size() > 0) |
|
// System.out.println(cu.getResultReport().getMessages().get(0) |
|
// .toString()); |
|
|
|
assertEquals(1, cu.getResultReport().getErrorCount()); |
|
|
|
int pos = cu |
|
.getResultReport() |
|
.getMessages() |
|
.get(0) |
|
.toString() |
|
.indexOf( |
|
"Error in Load-File Line 1: Error hsnr (Spalte 1) enthält Wert (1000x), der nicht in foreign keys vorkommt Error in Column 1: java.lang.NumberFormatException: For input string: \"1000x\""); |
|
assertTrue(pos > -1); |
|
|
|
} |
|
|
|
@AfterClass |
|
public static void tearDown() throws Exception { |
|
System.out.println("TEAR DOWN"); |
|
con.close(); |
|
Files.deleteIfExists((new File(tmpDbPath + ".mv.db")).toPath()); |
|
} |
|
|
|
}
|
|
|