Add init-db

Add save-regex
This commit is contained in:
Steve Schafer 2026-01-17 09:50:26 -07:00
parent 72d424a3f8
commit ea3ebbb298
20 changed files with 1125 additions and 754 deletions

2
.gitignore vendored
View file

@ -11,6 +11,7 @@
/budget/ /budget/
/build/ /build/
/categorized.lst /categorized.lst
/category-regex-*.lst
/chromedriver /chromedriver
/databases.lst /databases.lst
/explore*.sql /explore*.sql
@ -27,3 +28,4 @@
/test-urls /test-urls
/test_files /test_files
/upload.sql /upload.sql
*.java_*

View file

@ -3,4 +3,4 @@ db-url=jdbc:mysql://localhost:3306/budget_2023?serverTimezone=UTC&useSSL=false
db-username=elephant db-username=elephant
db-password=zsKRtBw6gPi0B0hMg1c2 db-password=zsKRtBw6gPi0B0hMg1c2
trilium-url=http://nuc1:8080/etapi trilium-url=http://nuc1:8080/etapi
trilium-password=scale-daughter-twiddling-educate-afraid trilium-password=$cale-D@ughter-TWenty5

File diff suppressed because it is too large Load diff

7
run
View file

@ -1,3 +1,8 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
java -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@" if java -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +1,8 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
./run categorize budget.properties if ./run categorize budget.properties > run-categorize.log 2> run-categorize.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +1,9 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
JVM_ARGS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8004" JVM_ARGS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8004"
java $JVM_ARGS -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@" if java $JVM_ARGS -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +0,0 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run-debug load-csv budget.properties trilium Budget

9
run-debug-load-csv Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run-debug load-csv budget.properties trilium Budget > run-debug-load-csv.log 2> run-debug-load-csv.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

9
run-init-db Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run init-db budget.properties > run-init-db.log 2> run-init-db.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +1,8 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
./run load-csv budget.properties trilium Budget if ./run load-csv budget.properties trilium Budget > run-load-csv.log 2> run-load-csv.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +1,9 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
./run load-regex budget.properties if ./run load-regex budget.properties > run-load-regex.log 2> run-load-regex.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -1,4 +1,8 @@
#!/bin/bash #!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
./run process budget.properties if ./run process budget.properties > run-process.log 2> run-process.err.log; then
echo "success"
else
echo "failure"
return 1
fi

8
run-save-regex Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run save-regex budget.properties > run-save-regex.log 2> run-save-regex.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

View file

@ -82,6 +82,31 @@ public class CatRegex {
this.year = year; this.year = year;
} }
public String toLine() {
final StringBuilder sb = new StringBuilder();
sb.append(category);
sb.append(", ");
sb.append(pattern.pattern());
sb.append(", ");
String flags = "";
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
flags += "i";
}
if ((pattern.flags() & Pattern.MULTILINE) != 0) {
flags += "m";
}
sb.append(flags);
sb.append(", ");
sb.append(source == null ? "" : source);
sb.append(", ");
sb.append(priority == 0 ? "" : String.valueOf(priority));
sb.append(", ");
sb.append(extraDescription);
sb.append(", ");
sb.append(year == null ? "" : year.toString());
return sb.toString();
}
public String getSource() { public String getSource() {
return source; return source;
} }

View file

@ -18,6 +18,7 @@ class Category {
if (parentId != null) { if (parentId != null) {
final Category parentCategory = categories.get(parentId); final Category parentCategory = categories.get(parentId);
sb.append(parentCategory.getFullName(categories)); sb.append(parentCategory.getFullName(categories));
sb.append(".");
} }
sb.append(name); sb.append(name);
return sb.toString(); return sb.toString();

View file

@ -9,6 +9,7 @@ import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.PrintStream; import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Reader; import java.io.Reader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.math.BigDecimal; import java.math.BigDecimal;
@ -26,9 +27,12 @@ import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.sql.Types; import java.sql.Types;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collections; import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar; import java.util.GregorianCalendar;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -115,17 +119,31 @@ public class Schema {
final Schema schema = new Schema(argsHelper.get("properties file")); final Schema schema = new Schema(argsHelper.get("properties file"));
schema.loadRegex(); schema.loadRegex();
}); });
PHASES.put("init-db", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file"));
schema.initDb();
});
PHASES.put("save-regex", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file"));
schema.saveRegex();
});
PHASES.put("load-csv", argsHelper -> { PHASES.put("load-csv", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file")); final Schema schema = new Schema(argsHelper.get("properties file"));
final String source = argsHelper.get("source"); final String source = argsHelper.get("source");
final String name = argsHelper.get("name"); final String name = argsHelper.get("name");
final Set<String> years = new HashSet<>(); final Set<String> years = new HashSet<>();
final DatabaseInitializer databaseInitializer = year -> {
if (!years.contains(year)) {
years.add(year);
schema.initializeDatabase(year);
}
};
if ("file".equals(source)) { if ("file".equals(source)) {
years.addAll(schema.generate(null, 1, new File(name))); schema.generate(null, 1, new File(name), databaseInitializer);
} }
else if ("trilium".equals(source)) { else if ("trilium".equals(source)) {
schema.triliumLogin(schema.config.getTriliumPassword()); schema.triliumLogin(schema.config.getTriliumPassword());
years.addAll(schema.generate(null, 1, name)); schema.generate(null, 1, name, databaseInitializer);
} }
else { else {
System.out.println("First argument must be 'file' or 'trilium'"); System.out.println("First argument must be 'file' or 'trilium'");
@ -144,6 +162,10 @@ public class Schema {
}); });
} }
private interface DatabaseInitializer {
void initializeDatabase(String year) throws SQLException;
}
private void categorize() private void categorize()
throws SQLException, UnsupportedEncodingException, FileNotFoundException, IOException { throws SQLException, UnsupportedEncodingException, FileNotFoundException, IOException {
final Map<String, List<Detail>> categoryMap = new HashMap<>(); final Map<String, List<Detail>> categoryMap = new HashMap<>();
@ -175,7 +197,9 @@ public class Schema {
final java.sql.Date date = rs.getDate(2); final java.sql.Date date = rs.getDate(2);
final String source = rs.getString(3); final String source = rs.getString(3);
final String description = rs.getString(4); final String description = rs.getString(4);
final BigDecimal amount = rs.getBigDecimal(5); final BigDecimal tmpAmount = rs.getBigDecimal(5);
final BigDecimal amount = tmpAmount == null ? new BigDecimal(0)
: tmpAmount;
final Detail detail = new Detail(); final Detail detail = new Detail();
detail.transactionId = transactionId; detail.transactionId = transactionId;
detail.source = source; detail.source = source;
@ -258,8 +282,8 @@ public class Schema {
"${catRegex}", catRegex.toString())); "${catRegex}", catRegex.toString()));
try (PreparedStatement insertStmt = connection.prepareStatement( try (PreparedStatement insertStmt = connection.prepareStatement(
insertSql)) { insertSql)) {
insertStmt.setInt(1, transactionId); insertStmt.setInt(1, catRegex.getId());
insertStmt.setInt(2, catRegex.getId()); insertStmt.setInt(2, transactionId);
insertStmt.execute(); insertStmt.execute();
} }
} }
@ -297,14 +321,58 @@ public class Schema {
} }
} }
private void initializeDatabase(final String year) throws SQLException {
try (Connection connection = pool.getConnection(8, false, true)) {
System.out.println("Initializing database budget_" + year);
dbExecute(connection, "drop database if exists budget_" + year);
dbExecute(connection, "create database budget_" + year);
}
}
private void saveRegex() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
final List<CatRegex> catRegexes = getCatRegexes(connection);
final File catRegexFile = new File("category-regex.lst");
if (catRegexFile.exists()) {
final DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
final String newFilename = "category-regex-${timestamp}.lst" //
.replace("${timestamp}", df.format(new Date()));
catRegexFile.renameTo(new File(newFilename));
}
try (PrintWriter pw = new PrintWriter(new FileOutputStream(catRegexFile))) {
for (final CatRegex catRegex : catRegexes) {
pw.println(catRegex.toLine());
}
}
}
}
private void initDb() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists " + "budget.category");
dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") //
.replace("${databaseName}", "budget"));
final Set<String> years = getYears(connection);
for (final String year : years) {
dbExecute(connection, Util.getResourceAsString("dropBudgetAmountTable.sql") //
.replace("${databaseName}", "budget_" + year));
dbExecute(connection, Util.getResourceAsString("createBudgetAmountTable.sql") //
.replace("${databaseName}", "budget_" + year));
}
dbExecute(connection, "drop table if exists budget.regex");
dbExecute(connection, Util.getResourceAsString("createRegexTable.sql") //
.replace("${databaseName}", "budget"));
}
}
private void loadRegex() throws SQLException, IOException { private void loadRegex() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) { try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists " + "budget.category"); dbExecute(connection, "drop table if exists " + "budget.category");
dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") // dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") //
.replace("${databaseName}", "budget")); .replace("${databaseName}", "budget"));
dbExecute(connection, "drop table if exists budget.regex"); dbExecute(connection, "drop table if exists budget.regex");
dbExecute(connection, Util.getResourceAsString("createRegexTable.sql").replace( dbExecute(connection, Util.getResourceAsString("createRegexTable.sql") //
"${databaseName}", "budget")); .replace("${databaseName}", "budget"));
final List<CatRegex> catRegexes = new ArrayList<>(); final List<CatRegex> catRegexes = new ArrayList<>();
final File catRegexesFile = new File("category-regex.lst"); final File catRegexesFile = new File("category-regex.lst");
try (BufferedReader reader = new BufferedReader( try (BufferedReader reader = new BufferedReader(
@ -324,8 +392,8 @@ public class Schema {
Map<Integer, Category> loadCategories(final Connection connection) Map<Integer, Category> loadCategories(final Connection connection)
throws IOException, SQLException { throws IOException, SQLException {
final Map<Integer, Category> categories = new HashMap<>(); final Map<Integer, Category> categories = new HashMap<>();
final String sql = Util.getResourceAsString("getCategories.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getCategories.sql") //
"budget"); .replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -346,8 +414,8 @@ public class Schema {
List<CatRegex> getCatRegexes(final Connection connection) throws IOException, SQLException { List<CatRegex> getCatRegexes(final Connection connection) throws IOException, SQLException {
final List<CatRegex> catRegexes = new ArrayList<>(); final List<CatRegex> catRegexes = new ArrayList<>();
final Map<Integer, Category> categories = loadCategories(connection); final Map<Integer, Category> categories = loadCategories(connection);
final String sql = Util.getResourceAsString("getRegexes.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getRegexes.sql") //
"budget"); .replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -380,10 +448,11 @@ public class Schema {
final String tableName = "regex"; final String tableName = "regex";
if (!tableExists(connection, null, tableName)) { if (!tableExists(connection, null, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget").replace("${tableName}", tableName)); "${databaseName}", "budget") //
.replace("${tableName}", tableName));
} }
final String insertSql = Util.getResourceAsString("insertRegex.sql").replace( final String insertSql = Util.getResourceAsString("insertRegex.sql") //
"${databaseName}", "budget"); .replace("${databaseName}", "budget");
for (final CatRegex catRegex : catRegexes) { for (final CatRegex catRegex : catRegexes) {
final Integer categoryId = getCategoryId(connection, catRegex.getCategory()); final Integer categoryId = getCategoryId(connection, catRegex.getCategory());
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql, try (PreparedStatement insertStmt = connection.prepareStatement(insertSql,
@ -442,8 +511,8 @@ public class Schema {
private Set<String> getYears(final Connection connection) throws IOException, SQLException { private Set<String> getYears(final Connection connection) throws IOException, SQLException {
final Set<String> years = new HashSet<>(); final Set<String> years = new HashSet<>();
final String sql = Util.getResourceAsString("getYears.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getYears.sql") //
"budget"); .replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -459,20 +528,22 @@ public class Schema {
try (Connection connection = pool.getConnection(8, false, true)) { try (Connection connection = pool.getConnection(8, false, true)) {
final Set<String> years = getYears(connection); final Set<String> years = getYears(connection);
for (final String year : years) { for (final String year : years) {
dbExecute(connection, "drop table if exists budget_" + year + ".transaction"); final String databaseName = "budget_${year}".replace("${year}", year);
dbExecute(connection, dbExecute(connection, Util.getResourceAsString("dropTransactionTable.sql") //
Util.getResourceAsString("createTransactionTable.sql").replace( .replace("${databaseName}", databaseName));
"${databaseName}", "budget_" + year)); dbExecute(connection, Util.getResourceAsString("createTransactionTable.sql") //
ingestAmazonDigitalOrders(connection, year, "sandy"); .replace("${databaseName}", databaseName));
ingestAmazonDigitalOrders(connection, year, "steve"); ingestAmazonDigitalOrders(connection, year, "Sandy");
ingestAmazonDigitalReturns(connection, year, "sandy"); ingestAmazonDigitalOrders(connection, year, "Steve");
ingestAmazonDigitalReturns(connection, year, "steve"); ingestAmazonDigitalReturns(connection, year, "Sandy");
ingestAmazonRetailOrders(connection, year, "sandy"); ingestAmazonDigitalReturns(connection, year, "Steve");
ingestAmazonRetailOrders(connection, year, "steve"); ingestAmazonRetailOrders(connection, year, "Sandy");
ingestAmazonRetailOrders(connection, year, "Steve");
ingestChase(connection, year); ingestChase(connection, year);
ingestDiscover(connection, year); ingestDiscover(connection, year);
ingestCiti(connection, year); ingestCiti(connection, year);
ingestFirstBank(connection, year); ingestFirstBank(connection, year);
ingestElevations(connection, year);
ingestPaypal(connection, year); ingestPaypal(connection, year);
} }
} }
@ -481,36 +552,58 @@ public class Schema {
private void ingestAmazonDigitalOrders(final Connection connection, final String year, private void ingestAmazonDigitalOrders(final Connection connection, final String year,
final String person) throws IOException, SQLException { final String person) throws IOException, SQLException {
System.out.println("process amazon digital orders budget_" + year + " " + person); System.out.println("process amazon digital orders budget_" + year + " " + person);
final String tableName = "amz_${person}_dig_ord_dig_items".replace("${person}", person); String triliumName = "Amazon.${person}.Digital-Ordering.Digital Items" //
.replace("${person}", person);
final String tableName = getTableName(triliumName);
if (tableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
triliumName = "Amazon.${person}.Digital-Ordering.Digital Orders" //
.replace("${person}", person);
final String digOrdersTableName = getTableName(triliumName);
if (digOrdersTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return; return;
} }
final String digOrdersTableName = "amz_${person}_dig_ord_dig_orders".replace("${person}",
person);
if (!tableExists(connection, year, digOrdersTableName)) { if (!tableExists(connection, year, digOrdersTableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", digOrdersTableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdersTableName));
return;
}
triliumName = "Amazon.${person}.Digital-Ordering.Digital Orders Monetary" //
.replace("${person}", person);
// "amz_${person}_dig_ord_dig_orders_monetary"
final String digOrdersMonetaryTableName = getTableName(triliumName);
if (digOrdersMonetaryTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return; return;
} }
final String digOrdersMonetaryTableName = "amz_${person}_dig_ord_dig_orders_monetary".replace(
"${person}", person);
if (!tableExists(connection, year, digOrdersMonetaryTableName)) { if (!tableExists(connection, year, digOrdersMonetaryTableName)) {
System.out.println( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
" table ${databaseName}.${tableName} does not exist".replace("${databaseName}", "${databaseName}", "budget_" + year) //
"budget_" + year).replace("${tableName}", digOrdersMonetaryTableName)); .replace("${tableName}", digOrdersMonetaryTableName));
return; return;
} }
final String sql = Util.getResourceAsString("getDigitalOrders.sql").replace( final String databaseName = "budget_${year}".replace("${year}", year);
"${databaseName}", "budget_" + year).replace("${person}", person).replace( final String sql = Util.getResourceAsString("getDigitalOrders.sql") //
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( .replace("${databaseName}", databaseName) //
"${digOrdItems}", tableName).replace("${digOrders}", .replace("${person}", person) //
digOrdersTableName).replace("${digOrdersMonetary}", .replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
digOrdersMonetaryTableName); .replace("${digOrdItems}", tableName) //
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( .replace("${digOrders}", digOrdersTableName) //
"${databaseName}", "budget_" + year).replace("${person}", person); .replace("${digOrdersMonetary}", digOrdersMonetaryTableName);
final String source = "amz.${person}.dig.ord".replace("${person}", person); final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", databaseName) //
.replace("${person}", person);
final String source = "amz.${person}.dig.ord".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -541,19 +634,27 @@ public class Schema {
private void ingestAmazonRetailOrders(final Connection connection, final String year, private void ingestAmazonRetailOrders(final Connection connection, final String year,
final String person) throws IOException, SQLException { final String person) throws IOException, SQLException {
System.out.println("process amazon retail orders " + year + " " + person); System.out.println("process amazon retail orders " + year + " " + person);
final String tableName = "amz_${person}_retail_order_history".replace("${person}", person); final String triliumName = "Amazon.${person}.Retail.OrderHistory" //
if (!tableExists(connection, year, tableName)) { .replace("${person}", person);
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( final String tableName = getTableName(triliumName);
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); if (tableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return; return;
} }
final String sql = Util.getResourceAsString("getRetailOrders.sql").replace( if (!tableExists(connection, year, tableName)) {
"${databaseName}", "budget_" + year).replace("${person}", person).replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( "${databaseName}", "budget_" + year) //
"${retailOrders}", tableName); .replace("${tableName}", tableName));
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( return;
"${databaseName}", "budget_" + year); }
final String source = "amz.${person}.retail.ord".replace("${person}", person); final String sql = Util.getResourceAsString("getRetailOrders.sql") //
.replace("${databaseName}", "budget_" + year) //
.replace("${person}", person) //
.replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
.replace("${retailOrders}", tableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "amz.${person}.retail.ord".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -587,13 +688,14 @@ public class Schema {
final String tableName = "discover"; final String tableName = "discover";
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return; return;
} }
final String sql = Util.getResourceAsString("getDiscover.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getDiscover.sql") //
"budget_" + year); .replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
final String source = "discover"; final String source = "discover";
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
@ -628,13 +730,14 @@ public class Schema {
final String tableName = "citi"; final String tableName = "citi";
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return; return;
} }
final String sql = Util.getResourceAsString("getCiti.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getCiti.sql") //
"budget_" + year); .replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
final String source = "citi"; final String source = "citi";
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
@ -669,13 +772,14 @@ public class Schema {
final String tableName = "first_bank"; final String tableName = "first_bank";
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return; return;
} }
final String sql = Util.getResourceAsString("getFirstBank.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getFirstBank.sql") //
"budget_" + year); .replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
final String source = "firstbank"; final String source = "firstbank";
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
@ -703,19 +807,79 @@ public class Schema {
} }
} }
private void ingestElevations(final Connection connection, final String year)
throws IOException, SQLException {
System.out.println("process elevations " + year);
final String tableName = "elevations";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getElevations.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "elevations";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
int i = 0;
final String transactionId = resultSet.getString(++i);
final java.sql.Date postingDate = resultSet.getDate(++i);
final java.sql.Date effectiveDate = resultSet.getDate(++i);
final String transactionType = resultSet.getString(++i);
final BigDecimal amount = resultSet.getBigDecimal(++i);
final String checkNumber = resultSet.getString(++i);
final int referenceNumber = resultSet.getInt(++i);
final String description = resultSet.getString(++i);
final String transactionCategory = resultSet.getString(++i);
final String type = resultSet.getString(++i);
final BigDecimal balance = resultSet.getBigDecimal(++i);
final String memo = resultSet.getString(++i);
final Calendar cal = new GregorianCalendar();
cal.setTime(postingDate);
if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) {
try (PreparedStatement insertStmt = connection.prepareStatement(
insertSql)) {
System.out.println("Inserting elevations");
System.out.println("transactionId = " + transactionId);
System.out.println("effectiveDate = " + effectiveDate);
System.out.println("transactionType = " + transactionType);
System.out.println("checkNumber = " + checkNumber);
System.out.println("referenceNumber = " + referenceNumber);
System.out.println("transactionCategory = " + transactionCategory);
System.out.println("balance = " + balance);
System.out.println("memo = " + memo);
insertStmt.setString(1, source);
insertStmt.setString(2, ""); // unique_id
insertStmt.setString(3, type); // type
insertStmt.setString(4, description);
insertStmt.setDate(5, postingDate);
insertStmt.setBigDecimal(6, amount.negate());
insertStmt.execute();
}
}
}
}
}
}
private void ingestChase(final Connection connection, final String year) private void ingestChase(final Connection connection, final String year)
throws IOException, SQLException { throws IOException, SQLException {
System.out.println("process chase " + year); System.out.println("process chase " + year);
final String tableName = "chase"; final String tableName = "chase";
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return; return;
} }
final String sql = Util.getResourceAsString("getChase.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getChase.sql") //
"budget_" + year); .replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
final String source = "chase"; final String source = "chase";
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
@ -750,13 +914,14 @@ public class Schema {
final String tableName = "paypal"; final String tableName = "paypal";
if (!tableExists(connection, year, tableName)) { if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return; return;
} }
final String sql = Util.getResourceAsString("getPaypal.sql").replace("${databaseName}", final String sql = Util.getResourceAsString("getPaypal.sql") //
"budget_" + year); .replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
final String source = "paypal"; final String source = "paypal";
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
@ -821,29 +986,42 @@ public class Schema {
private void ingestAmazonDigitalReturns(final Connection connection, final String year, private void ingestAmazonDigitalReturns(final Connection connection, final String year,
final String person) throws IOException, SQLException { final String person) throws IOException, SQLException {
System.out.println("process amazon digital returns budget_" + year + " " + person); System.out.println("process amazon digital returns budget_" + year + " " + person);
final String digOrdItemsTableName = "amz_${person}_dig_ord_dig_items".replace("${person}", String triliumName = "Amazon.${person}.Digital-Ordering.Digital Items" //
person); .replace("${person}", person);
final String digOrdReturnsMonetaryTableName = "amz_${person}_dig_orders_returns_dig_orders_returns_monetary_1".replace( final String digOrdItemsTableName = getTableName(triliumName);
"${person}", person); if (digOrdItemsTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
if (!tableExists(connection, year, digOrdItemsTableName)) { if (!tableExists(connection, year, digOrdItemsTableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", digOrdItemsTableName)); "${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdItemsTableName));
return;
}
triliumName = "Amazon.${person}.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1" //
.replace("${person}", person);
final String digOrdReturnsMonetaryTableName = getTableName(triliumName);
if (digOrdReturnsMonetaryTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return; return;
} }
if (!tableExists(connection, year, digOrdReturnsMonetaryTableName)) { if (!tableExists(connection, year, digOrdReturnsMonetaryTableName)) {
System.out.println( System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
" table ${databaseName}.${tableName} does not exist".replace("${databaseName}", "${databaseName}", "budget_" + year) //
"budget_" + year).replace("${tableName}", digOrdReturnsMonetaryTableName)); .replace("${tableName}", digOrdReturnsMonetaryTableName));
return; return;
} }
final String sql = Util.getResourceAsString("getDigitalReturns.sql").replace( final String sql = Util.getResourceAsString("getDigitalReturns.sql") //
"${databaseName}", "budget_" + year).replace("${person}", person).replace( .replace("${databaseName}", "budget_" + year) //
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( .replace("${person}", person) //
"${digOrdItems}", digOrdItemsTableName).replace("${digOrdReturnsMonetary}", .replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
digOrdReturnsMonetaryTableName); .replace("${digOrdItems}", digOrdItemsTableName) //
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( .replace("${digOrdReturnsMonetary}", digOrdReturnsMonetaryTableName);
"${databaseName}", "budget_" + year).replace("${person}", person); final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
final String source = "amz.${person}.dig.ret".replace("${person}", person); .replace("${databaseName}", "budget_" + year) //
.replace("${person}", person);
final String source = "amz.${person}.dig.ret".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) { try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) { while (resultSet.next()) {
@ -873,8 +1051,8 @@ public class Schema {
private void linkRegex(final Connection connection, final String year, final Integer regexId, private void linkRegex(final Connection connection, final String year, final Integer regexId,
final int transactionId) throws IOException, SQLException { final int transactionId) throws IOException, SQLException {
final String sql = Util.getResourceAsString("updateRegexLink.sql").replace( final String sql = Util.getResourceAsString("updateRegexLink.sql") //
"${databaseName}", "budget_" + year); .replace("${databaseName}", "budget_" + year);
try (PreparedStatement stmt = connection.prepareStatement(sql)) { try (PreparedStatement stmt = connection.prepareStatement(sql)) {
stmt.setInt(1, regexId); stmt.setInt(1, regexId);
stmt.setInt(2, transactionId); stmt.setInt(2, transactionId);
@ -894,10 +1072,10 @@ public class Schema {
private Integer getCategoryPartId(final Connection connection, final Integer parentCategoryId, private Integer getCategoryPartId(final Connection connection, final Integer parentCategoryId,
final String categoryPart) throws SQLException, IOException { final String categoryPart) throws SQLException, IOException {
final String getRootSql = Util.getResourceAsString("getRootCategoryId.sql").replace( final String getRootSql = Util.getResourceAsString("getRootCategoryId.sql") //
"${databaseName}", "budget"); .replace("${databaseName}", "budget");
final String getChildSql = Util.getResourceAsString("getChildCategoryId.sql").replace( final String getChildSql = Util.getResourceAsString("getChildCategoryId.sql") //
"${databaseName}", "budget"); .replace("${databaseName}", "budget");
Integer categoryId = null; Integer categoryId = null;
final String getSql = parentCategoryId == null ? getRootSql : getChildSql; final String getSql = parentCategoryId == null ? getRootSql : getChildSql;
try (PreparedStatement stmt = connection.prepareStatement(getSql)) { try (PreparedStatement stmt = connection.prepareStatement(getSql)) {
@ -919,10 +1097,10 @@ public class Schema {
} }
if (categoryId == null) { if (categoryId == null) {
final String insertSql = parentCategoryId == null // final String insertSql = parentCategoryId == null //
? Util.getResourceAsString("insertRootCategory.sql").replace("${databaseName}", ? Util.getResourceAsString("insertRootCategory.sql") //
"budget") // .replace("${databaseName}", "budget") //
: Util.getResourceAsString("insertChildCategory.sql").replace("${databaseName}", : Util.getResourceAsString("insertChildCategory.sql") //
"budget"); .replace("${databaseName}", "budget");
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql, try (PreparedStatement insertStmt = connection.prepareStatement(insertSql,
Statement.RETURN_GENERATED_KEYS)) { Statement.RETURN_GENERATED_KEYS)) {
insertStmt.setString(1, categoryPart); insertStmt.setString(1, categoryPart);
@ -1142,9 +1320,9 @@ public class Schema {
return result; return result;
} }
private Set<String> generate(final String name, final int siblingCount, private void generate(final String name, final int siblingCount, final String triliumSearch,
final String triliumSearch) throws Exception { final DatabaseInitializer databaseInitializer) throws Exception {
final Set<String> years = new HashSet<>(); new HashSet<>();
final String output = getTriliumContent( final String output = getTriliumContent(
"/notes?search=" + triliumSearch + "&fastSearch=true"); "/notes?search=" + triliumSearch + "&fastSearch=true");
final ObjectMapper mapper = new ObjectMapper(); final ObjectMapper mapper = new ObjectMapper();
@ -1169,19 +1347,22 @@ public class Schema {
if (!rootFound) { if (!rootFound) {
continue; continue;
} }
years.addAll(generate(name, 1, noteNode)); generate(name, 1, null, noteNode, databaseInitializer);
} }
return years; return;
} }
public Set<String> generate(String name, final int siblingCount, final JsonNode noteNode) public void generate(String name, final int siblingCount, final JsonNode parentNoteNode,
final JsonNode noteNode, final DatabaseInitializer databaseInitializer)
throws Exception { throws Exception {
final Set<String> years = new HashSet<>();
final JsonNode titleNode = noteNode.get("title"); final JsonNode titleNode = noteNode.get("title");
if (titleNode == null) { if (titleNode == null) {
return years; return;
} }
String childName = titleNode.textValue(); String childName = titleNode.textValue();
final JsonNode parentTitleNode = parentNoteNode == null ? null
: parentNoteNode.get("title");
final String parentTitle = parentTitleNode == null ? null : parentTitleNode.textValue();
String ext = null; String ext = null;
final int lastIndexOfDot = childName.lastIndexOf("."); final int lastIndexOfDot = childName.lastIndexOf(".");
if (lastIndexOfDot >= 0) { if (lastIndexOfDot >= 0) {
@ -1199,24 +1380,28 @@ public class Schema {
if (noteIdNode != null) { if (noteIdNode != null) {
final String noteId = noteIdNode.textValue(); final String noteId = noteIdNode.textValue();
final List<String[]> records = getTriliumRecords("/notes/" + noteId + "/content"); final List<String[]> records = getTriliumRecords("/notes/" + noteId + "/content");
years.addAll(generate(name, siblingCount, records, generate(name, siblingCount, records, "Firstbank".equals(parentTitle),
childName.startsWith("FirstbankDownload"), "qb.csv".equals(childName))); "qb.csv".equals(childName), databaseInitializer);
} }
} }
final JsonNode childNodeIdsNode = noteNode.get("childNoteIds"); final JsonNode childNodeIdsNode = noteNode.get("childNoteIds");
if (childNodeIdsNode == null) { if (childNodeIdsNode == null) {
return years; return;
} }
for (final JsonNode childNodeIdNode : childNodeIdsNode) { for (final JsonNode childNodeIdNode : childNodeIdsNode) {
final String childNodeId = childNodeIdNode.textValue(); final String childNodeId = childNodeIdNode.textValue();
final JsonNode childNode = getTriliumJsonContent("/notes/" + childNodeId); final JsonNode childNode = getTriliumJsonContent("/notes/" + childNodeId);
years.addAll(generate(name, childNodeIdsNode.size(), childNode)); generate(name, childNodeIdsNode.size(), noteNode, childNode, databaseInitializer);
} }
return years; return;
} }
private Set<String> generate(String name, final int siblingCount, final File file) private boolean isFirstBankCsv(final String name) {
throws Exception { return name.startsWith("FirstbankDownload") || name.equals("2025-checking");
}
private void generate(String name, final int siblingCount, final File file,
final DatabaseInitializer databaseInitializer) throws Exception {
String childName = file.getName(); String childName = file.getName();
String ext = null; String ext = null;
if (file.isFile()) { if (file.isFile()) {
@ -1232,28 +1417,54 @@ public class Schema {
else if (siblingCount > 1) { else if (siblingCount > 1) {
name = name + "." + childName; name = name + "." + childName;
} }
final Set<String> years = new HashSet<>(); new HashSet<>();
if (file.isDirectory()) { if (file.isDirectory()) {
final File[] children = file.listFiles(); final File[] children = file.listFiles();
for (final File child : children) { for (final File child : children) {
years.addAll(generate(name, children.length, child)); generate(name, children.length, child, databaseInitializer);
} }
} }
else if (file.isFile() && "csv".equalsIgnoreCase(ext)) { else if (file.isFile() && "csv".equalsIgnoreCase(ext)) {
final List<String[]> records = readAllLines(Paths.get(file.getPath())); final List<String[]> records = readAllLines(Paths.get(file.getPath()));
years.addAll(generate(name, siblingCount, records, generate(name, siblingCount, records, isFirstBankCsv(file.getName()),
file.getName().startsWith("FirstbankDownload"), "qb.csv".equals(file.getName()))); "qb.csv".equals(file.getName()), databaseInitializer);
} }
return years; return;
} }
private static Pattern NAME_PATTERN = Pattern.compile("^Budget\\.([0-9]+)\\.(.*)$"); private static Pattern NAME_PATTERN = Pattern.compile("^Budget\\.([0-9]+)\\.(.*)$");
private static Map<String, String> TABLE_NAMES = new HashMap<>();
static {
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Items", "amz_steve_dig_ord_items");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Items", "amz_sandy_dig_ord_items");
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Orders", "amz_steve_dig_ord_ords");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Orders", "amz_sandy_dig_ord_ords");
// Amazon.${person}.Digital-Ordering.Digital Orders Monetary
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Orders Monetary",
"amz_steve_dig_ord_ords_mon");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Orders Monetary",
"amz_sandy_dig_ord_ords_mon");
TABLE_NAMES.put("Amazon.Steve.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1",
"amz_steve_dig_ords_rets_mon");
TABLE_NAMES.put("Amazon.Sandy.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1",
"amz_sandy_dig_ords_rets_mon");
TABLE_NAMES.put("Amazon.Steve.Retail.OrderHistory", "amz_steve_ret_ord_hist");
TABLE_NAMES.put("Amazon.Sandy.Retail.OrderHistory", "amz_sandy_ret_ord_hist");
TABLE_NAMES.put("Chase", "chase");
TABLE_NAMES.put("Paypal", "paypal");
TABLE_NAMES.put("Firstbank", "first_bank");
TABLE_NAMES.put("Firstbank.2025-checking", "first_bank");
TABLE_NAMES.put("Citi", "citi");
TABLE_NAMES.put("Discover", "discover");
TABLE_NAMES.put("Elevations", "elevations");
TABLE_NAMES.put("Elevations.2025-checking", "elevations");
}
private Set<String> generate(final String name, final int siblingCount, private void generate(final String name, final int siblingCount, final List<String[]> records,
final List<String[]> records, final boolean isFirstBank, final boolean isQuickBooks) final boolean isFirstBank, final boolean isQuickBooks,
throws SQLException { final DatabaseInitializer databaseInitializer) throws SQLException {
int recordIndex = 0; int recordIndex = 0;
final Set<String> years = new HashSet<>(); new HashSet<>();
List<FieldInfo> fields = null; List<FieldInfo> fields = null;
for (final String[] record : records) { for (final String[] record : records) {
if (recordIndex == 0) { if (recordIndex == 0) {
@ -1398,35 +1609,72 @@ public class Schema {
} }
final Matcher matcher = NAME_PATTERN.matcher(name); final Matcher matcher = NAME_PATTERN.matcher(name);
if (!matcher.matches()) { if (!matcher.matches()) {
throw new RuntimeException("Name does not begin with Budget.[year]"); System.out.println("Skipping " + name + " because name pattern does not match");
return;
} }
final String year = matcher.group(1); final String year = matcher.group(1);
years.add(year); final String triliumName = matcher.group(2);
final String tableName = decamel(matcher.group(2)).replace("budget_", "").replace("amazon_", final String tableName = getTableName(triliumName);
"amz_").replace("_digital_", "_dig_").replace("_information", "_inf").replace(
"_payment_", "_pmt_").replace("_ordering_", "_ord_").replace("_customer_",
"_cust_").replace("_communication_", "_comm_").replace("_preferences_",
"_pref_").replace("_physical_", "_phys_").replace("_whole_foods_", "_wf_");
if (fields != null) { if (fields != null) {
System.out.println("Loading CSV " + name); if (tableName == null) {
try (Connection connection = pool.getConnection(8, false, true)) { System.out.println("Skipping CSV " + name + " because table not found");
final String qualifiedTableName = "budget_" + year + "." + tableName; }
dbExecute(connection, "drop table if exists " + qualifiedTableName); else {
dbExecute(connection, getCreateTableSql(qualifiedTableName, fields)); databaseInitializer.initializeDatabase(year);
doInserts(connection, qualifiedTableName, fields, records, isFirstBank, try (Connection connection = pool.getConnection(8, false, true)) {
isQuickBooks); final String qualifiedTableName = "budget_" + year + "." + tableName;
System.out.println("Loading CSV " + name + " into " + qualifiedTableName);
dbExecute(connection, "drop table if exists " + qualifiedTableName);
dbExecute(connection, getCreateTableSql(qualifiedTableName, fields));
doInserts(connection, qualifiedTableName, fields, records, isFirstBank,
isQuickBooks);
}
} }
} }
return years; return;
}
private static String getTableName(final String triliumName) {
final String tableName = TABLE_NAMES.get(triliumName);
return tableName;
/*
if (tableName != null) {
return tableName;
}
return decamel(triliumName) //
.replace("budget_", "") //
.replace("amazon_", "amz_") //
.replace("_digital_", "_dig_") //
.replace("_information", "_inf") //
.replace("_payment_", "_pmt_") //
.replace("_ordering_", "_ord_") //
.replace("_customer_", "_cust_") //
.replace("_communication_", "_comm_") //
.replace("_preferences_", "_pref_") //
.replace("_physical_", "_phys_") //
.replace("_whole_foods_", "_wf_") //
.replace("_subscriptions_subscriptions_", "_ssubs_") //
.replace("_subscriptions_", "_subs_") //
.replace("_subscription_", "_sub_") //
.replace("_status_", "_stat_") //
.replace("_billing_", "_bill_") //
.replace("_refunds_", "_refs_") //
.replace("_returns_", "_rets_") //
.replace("_orders_", "_ords") //
.replace("_monetary_", "_mon_") //
.replace("_and_", "_") //
.replace("_data", "_dat") //
.replace("_history", "_hist");
*/
} }
private void saveYears(final Set<String> years) throws SQLException, IOException { private void saveYears(final Set<String> years) throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) { try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists budget.years"); dbExecute(connection, "drop table if exists budget.years");
dbExecute(connection, Util.getResourceAsString("createYearsTable.sql").replace( dbExecute(connection, Util.getResourceAsString("createYearsTable.sql") //
"${databaseName}", "budget")); .replace("${databaseName}", "budget"));
final String insertSql = Util.getResourceAsString("insertYear.sql").replace( final String insertSql = Util.getResourceAsString("insertYear.sql") //
"${databaseName}", "budget"); .replace("${databaseName}", "budget");
for (final String year : years) { for (final String year : years) {
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql)) { try (PreparedStatement insertStmt = connection.prepareStatement(insertSql)) {
insertStmt.setInt(1, Integer.parseInt(year)); insertStmt.setInt(1, Integer.parseInt(year));
@ -1454,7 +1702,17 @@ public class Schema {
final String sql = getInsertSql(tableName, record, fields); final String sql = getInsertSql(tableName, record, fields);
try (final PreparedStatement stmt = connection.prepareStatement(sql)) { try (final PreparedStatement stmt = connection.prepareStatement(sql)) {
setValues(stmt, record, fields); setValues(stmt, record, fields);
stmt.execute(); try {
stmt.execute();
}
catch (final SQLException e) {
System.out.println("SQL Exception: " + e);
System.out.println(sql);
for (int i = 0; i < record.length; i++) {
System.out.println(i + " " + record[i]);
}
throw e;
}
} }
} }
recordIndex++; recordIndex++;
@ -1579,13 +1837,14 @@ public class Schema {
} }
} }
private String decamel(final String name) { private static String decamel(final String name) {
if (name == null || name.trim().length() == 0) { if (name == null || name.trim().length() == 0) {
return "unknown"; return "unknown";
} }
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder();
final char[] chars = name.replace("ID", "Id").replace("RMA", "Rma").replace("ASIN", final char[] chars = name.replace("ID", "Id") //
"Asin").toCharArray(); .replace("RMA", "Rma") //
.replace("ASIN", "Asin").toCharArray();
int charIndex = 0; int charIndex = 0;
char prevChar = '_'; char prevChar = '_';
while (charIndex < chars.length) { while (charIndex < chars.length) {

View file

@ -0,0 +1,16 @@
create table ${databaseName}.budget_amount (
category_id int not null primary key,
year_amount decimal(10,2),
jan_amount decimal(10,2),
feb_amount decimal(10,2),
mar_amount decimal(10,2),
apr_amount decimal(10,2),
may_amount decimal(10,2),
jun_amount decimal(10,2),
jul_amount decimal(10,2),
aug_amount decimal(10,2),
sep_amount decimal(10,2),
oct_amount decimal(10,2),
nov_amount decimal(10,2),
dec_amount decimal(10,2)
)

View file

@ -0,0 +1 @@
drop table if exists ${databaseName}.budget_amount

View file

@ -0,0 +1 @@
drop table if exists ${databaseName}.transaction

View file

@ -0,0 +1,15 @@
select
transaction_id,
posting_date,
effective_date,
transaction_type,
amount,
check_number,
reference_number,
description,
transaction_category,
type,
balance,
memo,
extended_description
from ${databaseName}.elevations