steve/com.stephenschafer.budget
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add init-db

Browse source 
Add save-regex
This commit is contained in:
Steve Schafer 2026-01-17 09:50:26 -07:00
parent 72d424a3f8
commit ea3ebbb298
20 changed files with 1125 additions and 754 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -11,6 +11,7 @@
/budget/
/build/
/categorized.lst
/category-regex-*.lst
/chromedriver
/databases.lst
/explore*.sql
@ -27,3 +28,4 @@
/test-urls
/test_files
/upload.sql
*.java_*

2
budget.properties
View file

@ -3,4 +3,4 @@ db-url=jdbc:mysql://localhost:3306/budget_2023?serverTimezone=UTC&useSSL=false
db-username=elephant
db-password=zsKRtBw6gPi0B0hMg1c2
trilium-url=http://nuc1:8080/etapi
trilium-password=scale-daughter-twiddling-educate-afraid
trilium-password=$cale-D@ughter-TWenty5

1174
category-regex.lst
View file

File diff suppressed because it is too large Load diff

7
run
View file

@ -1,3 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
java -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"
if java -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"; then
echo "success"
else
echo "failure"
exit 1
fi

8
run-categorize
View file

@ -1,4 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run categorize budget.properties
if ./run categorize budget.properties > run-categorize.log 2> run-categorize.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

7
run-debug
View file

@ -1,4 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
JVM_ARGS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8004"
java $JVM_ARGS -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"
if java $JVM_ARGS -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@"; then
echo "success"
else
echo "failure"
exit 1
fi

4
run-debug-load
View file

@ -1,4 +0,0 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run-debug load-csv budget.properties trilium Budget

9
run-debug-load-csv Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run-debug load-csv budget.properties trilium Budget > run-debug-load-csv.log 2> run-debug-load-csv.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

9
run-init-db Executable file
View file

@ -0,0 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run init-db budget.properties > run-init-db.log 2> run-init-db.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

8
run-load-csv
View file

@ -1,4 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run load-csv budget.properties trilium Budget
if ./run load-csv budget.properties trilium Budget > run-load-csv.log 2> run-load-csv.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

7
run-load-regex
View file

@ -1,4 +1,9 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run load-regex budget.properties
if ./run load-regex budget.properties > run-load-regex.log 2> run-load-regex.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

8
run-process
View file

@ -1,4 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
./run process budget.properties
if ./run process budget.properties > run-process.log 2> run-process.err.log; then
echo "success"
else
echo "failure"
return 1
fi

8
run-save-regex Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
cd "$(dirname "${BASH_SOURCE[0]}")"
if ./run save-regex budget.properties > run-save-regex.log 2> run-save-regex.err.log; then
echo "success"
else
echo "failure"
exit 1
fi

25
src/main/java/com/stephenschafer/budget/schema/CatRegex.java
View file

@ -82,6 +82,31 @@ public class CatRegex {
this.year = year;
}
public String toLine() {
final StringBuilder sb = new StringBuilder();
sb.append(category);
sb.append(", ");
sb.append(pattern.pattern());
sb.append(", ");
String flags = "";
if ((pattern.flags() & Pattern.CASE_INSENSITIVE) != 0) {
flags += "i";
}
if ((pattern.flags() & Pattern.MULTILINE) != 0) {
flags += "m";
}
sb.append(flags);
sb.append(", ");
sb.append(source == null ? "" : source);
sb.append(", ");
sb.append(priority == 0 ? "" : String.valueOf(priority));
sb.append(", ");
sb.append(extraDescription);
sb.append(", ");
sb.append(year == null ? "" : year.toString());
return sb.toString();
}
public String getSource() {
return source;
}

1
src/main/java/com/stephenschafer/budget/schema/Category.java
View file

@ -18,6 +18,7 @@ class Category {
if (parentId != null) {
final Category parentCategory = categories.get(parentId);
sb.append(parentCategory.getFullName(categories));
sb.append(".");
}
sb.append(name);
return sb.toString();

553
src/main/java/com/stephenschafer/budget/schema/Schema.java
View file

@ -9,6 +9,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
@ -26,9 +27,12 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
@ -115,17 +119,31 @@ public class Schema {
final Schema schema = new Schema(argsHelper.get("properties file"));
schema.loadRegex();
});
PHASES.put("init-db", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file"));
schema.initDb();
});
PHASES.put("save-regex", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file"));
schema.saveRegex();
});
PHASES.put("load-csv", argsHelper -> {
final Schema schema = new Schema(argsHelper.get("properties file"));
final String source = argsHelper.get("source");
final String name = argsHelper.get("name");
final Set<String> years = new HashSet<>();
final DatabaseInitializer databaseInitializer = year -> {
if (!years.contains(year)) {
years.add(year);
schema.initializeDatabase(year);
}
};
if ("file".equals(source)) {
years.addAll(schema.generate(null, 1, new File(name)));
schema.generate(null, 1, new File(name), databaseInitializer);
}
else if ("trilium".equals(source)) {
schema.triliumLogin(schema.config.getTriliumPassword());
years.addAll(schema.generate(null, 1, name));
schema.generate(null, 1, name, databaseInitializer);
}
else {
System.out.println("First argument must be 'file' or 'trilium'");
@ -144,6 +162,10 @@ public class Schema {
});
}
private interface DatabaseInitializer {
void initializeDatabase(String year) throws SQLException;
}
private void categorize()
throws SQLException, UnsupportedEncodingException, FileNotFoundException, IOException {
final Map<String, List<Detail>> categoryMap = new HashMap<>();
@ -175,7 +197,9 @@ public class Schema {
final java.sql.Date date = rs.getDate(2);
final String source = rs.getString(3);
final String description = rs.getString(4);
final BigDecimal amount = rs.getBigDecimal(5);
final BigDecimal tmpAmount = rs.getBigDecimal(5);
final BigDecimal amount = tmpAmount == null ? new BigDecimal(0)
: tmpAmount;
final Detail detail = new Detail();
detail.transactionId = transactionId;
detail.source = source;
@ -258,8 +282,8 @@ public class Schema {
"${catRegex}", catRegex.toString()));
try (PreparedStatement insertStmt = connection.prepareStatement(
insertSql)) {
insertStmt.setInt(1, transactionId);
insertStmt.setInt(2, catRegex.getId());
insertStmt.setInt(1, catRegex.getId());
insertStmt.setInt(2, transactionId);
insertStmt.execute();
}
}
@ -297,14 +321,58 @@ public class Schema {
}
}
private void initializeDatabase(final String year) throws SQLException {
try (Connection connection = pool.getConnection(8, false, true)) {
System.out.println("Initializing database budget_" + year);
dbExecute(connection, "drop database if exists budget_" + year);
dbExecute(connection, "create database budget_" + year);
}
}
private void saveRegex() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
final List<CatRegex> catRegexes = getCatRegexes(connection);
final File catRegexFile = new File("category-regex.lst");
if (catRegexFile.exists()) {
final DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
final String newFilename = "category-regex-${timestamp}.lst" //
.replace("${timestamp}", df.format(new Date()));
catRegexFile.renameTo(new File(newFilename));
}
try (PrintWriter pw = new PrintWriter(new FileOutputStream(catRegexFile))) {
for (final CatRegex catRegex : catRegexes) {
pw.println(catRegex.toLine());
}
}
}
}
private void initDb() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists " + "budget.category");
dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") //
.replace("${databaseName}", "budget"));
final Set<String> years = getYears(connection);
for (final String year : years) {
dbExecute(connection, Util.getResourceAsString("dropBudgetAmountTable.sql") //
.replace("${databaseName}", "budget_" + year));
dbExecute(connection, Util.getResourceAsString("createBudgetAmountTable.sql") //
.replace("${databaseName}", "budget_" + year));
}
dbExecute(connection, "drop table if exists budget.regex");
dbExecute(connection, Util.getResourceAsString("createRegexTable.sql") //
.replace("${databaseName}", "budget"));
}
}
private void loadRegex() throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists " + "budget.category");
dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") //
.replace("${databaseName}", "budget"));
dbExecute(connection, "drop table if exists budget.regex");
dbExecute(connection, Util.getResourceAsString("createRegexTable.sql").replace(
"${databaseName}", "budget"));
dbExecute(connection, Util.getResourceAsString("createRegexTable.sql") //
.replace("${databaseName}", "budget"));
final List<CatRegex> catRegexes = new ArrayList<>();
final File catRegexesFile = new File("category-regex.lst");
try (BufferedReader reader = new BufferedReader(
@ -324,8 +392,8 @@ public class Schema {
Map<Integer, Category> loadCategories(final Connection connection)
throws IOException, SQLException {
final Map<Integer, Category> categories = new HashMap<>();
final String sql = Util.getResourceAsString("getCategories.sql").replace("${databaseName}",
"budget");
final String sql = Util.getResourceAsString("getCategories.sql") //
.replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -346,8 +414,8 @@ public class Schema {
List<CatRegex> getCatRegexes(final Connection connection) throws IOException, SQLException {
final List<CatRegex> catRegexes = new ArrayList<>();
final Map<Integer, Category> categories = loadCategories(connection);
final String sql = Util.getResourceAsString("getRegexes.sql").replace("${databaseName}",
"budget");
final String sql = Util.getResourceAsString("getRegexes.sql") //
.replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -380,10 +448,11 @@ public class Schema {
final String tableName = "regex";
if (!tableExists(connection, null, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget").replace("${tableName}", tableName));
"${databaseName}", "budget") //
.replace("${tableName}", tableName));
}
final String insertSql = Util.getResourceAsString("insertRegex.sql").replace(
"${databaseName}", "budget");
final String insertSql = Util.getResourceAsString("insertRegex.sql") //
.replace("${databaseName}", "budget");
for (final CatRegex catRegex : catRegexes) {
final Integer categoryId = getCategoryId(connection, catRegex.getCategory());
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql,
@ -442,8 +511,8 @@ public class Schema {
private Set<String> getYears(final Connection connection) throws IOException, SQLException {
final Set<String> years = new HashSet<>();
final String sql = Util.getResourceAsString("getYears.sql").replace("${databaseName}",
"budget");
final String sql = Util.getResourceAsString("getYears.sql") //
.replace("${databaseName}", "budget");
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -459,20 +528,22 @@ public class Schema {
try (Connection connection = pool.getConnection(8, false, true)) {
final Set<String> years = getYears(connection);
for (final String year : years) {
dbExecute(connection, "drop table if exists budget_" + year + ".transaction");
dbExecute(connection,
Util.getResourceAsString("createTransactionTable.sql").replace(
"${databaseName}", "budget_" + year));
ingestAmazonDigitalOrders(connection, year, "sandy");
ingestAmazonDigitalOrders(connection, year, "steve");
ingestAmazonDigitalReturns(connection, year, "sandy");
ingestAmazonDigitalReturns(connection, year, "steve");
ingestAmazonRetailOrders(connection, year, "sandy");
ingestAmazonRetailOrders(connection, year, "steve");
final String databaseName = "budget_${year}".replace("${year}", year);
dbExecute(connection, Util.getResourceAsString("dropTransactionTable.sql") //
.replace("${databaseName}", databaseName));
dbExecute(connection, Util.getResourceAsString("createTransactionTable.sql") //
.replace("${databaseName}", databaseName));
ingestAmazonDigitalOrders(connection, year, "Sandy");
ingestAmazonDigitalOrders(connection, year, "Steve");
ingestAmazonDigitalReturns(connection, year, "Sandy");
ingestAmazonDigitalReturns(connection, year, "Steve");
ingestAmazonRetailOrders(connection, year, "Sandy");
ingestAmazonRetailOrders(connection, year, "Steve");
ingestChase(connection, year);
ingestDiscover(connection, year);
ingestCiti(connection, year);
ingestFirstBank(connection, year);
ingestElevations(connection, year);
ingestPaypal(connection, year);
}
}
@ -481,36 +552,58 @@ public class Schema {
private void ingestAmazonDigitalOrders(final Connection connection, final String year,
final String person) throws IOException, SQLException {
System.out.println("process amazon digital orders budget_" + year + " " + person);
final String tableName = "amz_${person}_dig_ord_dig_items".replace("${person}", person);
String triliumName = "Amazon.${person}.Digital-Ordering.Digital Items" //
.replace("${person}", person);
final String tableName = getTableName(triliumName);
if (tableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
triliumName = "Amazon.${person}.Digital-Ordering.Digital Orders" //
.replace("${person}", person);
final String digOrdersTableName = getTableName(triliumName);
if (digOrdersTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
final String digOrdersTableName = "amz_${person}_dig_ord_dig_orders".replace("${person}",
person);
if (!tableExists(connection, year, digOrdersTableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", digOrdersTableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdersTableName));
return;
}
triliumName = "Amazon.${person}.Digital-Ordering.Digital Orders Monetary" //
.replace("${person}", person);
// "amz_${person}_dig_ord_dig_orders_monetary"
final String digOrdersMonetaryTableName = getTableName(triliumName);
if (digOrdersMonetaryTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
final String digOrdersMonetaryTableName = "amz_${person}_dig_ord_dig_orders_monetary".replace(
"${person}", person);
if (!tableExists(connection, year, digOrdersMonetaryTableName)) {
System.out.println(
" table ${databaseName}.${tableName} does not exist".replace("${databaseName}",
"budget_" + year).replace("${tableName}", digOrdersMonetaryTableName));
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdersMonetaryTableName));
return;
}
final String sql = Util.getResourceAsString("getDigitalOrders.sql").replace(
"${databaseName}", "budget_" + year).replace("${person}", person).replace(
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace(
"${digOrdItems}", tableName).replace("${digOrders}",
digOrdersTableName).replace("${digOrdersMonetary}",
digOrdersMonetaryTableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year).replace("${person}", person);
final String source = "amz.${person}.dig.ord".replace("${person}", person);
final String databaseName = "budget_${year}".replace("${year}", year);
final String sql = Util.getResourceAsString("getDigitalOrders.sql") //
.replace("${databaseName}", databaseName) //
.replace("${person}", person) //
.replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
.replace("${digOrdItems}", tableName) //
.replace("${digOrders}", digOrdersTableName) //
.replace("${digOrdersMonetary}", digOrdersMonetaryTableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", databaseName) //
.replace("${person}", person);
final String source = "amz.${person}.dig.ord".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -541,19 +634,27 @@ public class Schema {
private void ingestAmazonRetailOrders(final Connection connection, final String year,
final String person) throws IOException, SQLException {
System.out.println("process amazon retail orders " + year + " " + person);
final String tableName = "amz_${person}_retail_order_history".replace("${person}", person);
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
final String triliumName = "Amazon.${person}.Retail.OrderHistory" //
.replace("${person}", person);
final String tableName = getTableName(triliumName);
if (tableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
final String sql = Util.getResourceAsString("getRetailOrders.sql").replace(
"${databaseName}", "budget_" + year).replace("${person}", person).replace(
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace(
"${retailOrders}", tableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String source = "amz.${person}.retail.ord".replace("${person}", person);
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getRetailOrders.sql") //
.replace("${databaseName}", "budget_" + year) //
.replace("${person}", person) //
.replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
.replace("${retailOrders}", tableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "amz.${person}.retail.ord".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -587,13 +688,14 @@ public class Schema {
final String tableName = "discover";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getDiscover.sql").replace("${databaseName}",
"budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("getDiscover.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "discover";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
@ -628,13 +730,14 @@ public class Schema {
final String tableName = "citi";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getCiti.sql").replace("${databaseName}",
"budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("getCiti.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "citi";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
@ -669,13 +772,14 @@ public class Schema {
final String tableName = "first_bank";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getFirstBank.sql").replace("${databaseName}",
"budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("getFirstBank.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "firstbank";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
@ -703,19 +807,79 @@ public class Schema {
}
}
private void ingestElevations(final Connection connection, final String year)
throws IOException, SQLException {
System.out.println("process elevations " + year);
final String tableName = "elevations";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getElevations.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "elevations";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
int i = 0;
final String transactionId = resultSet.getString(++i);
final java.sql.Date postingDate = resultSet.getDate(++i);
final java.sql.Date effectiveDate = resultSet.getDate(++i);
final String transactionType = resultSet.getString(++i);
final BigDecimal amount = resultSet.getBigDecimal(++i);
final String checkNumber = resultSet.getString(++i);
final int referenceNumber = resultSet.getInt(++i);
final String description = resultSet.getString(++i);
final String transactionCategory = resultSet.getString(++i);
final String type = resultSet.getString(++i);
final BigDecimal balance = resultSet.getBigDecimal(++i);
final String memo = resultSet.getString(++i);
final Calendar cal = new GregorianCalendar();
cal.setTime(postingDate);
if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) {
try (PreparedStatement insertStmt = connection.prepareStatement(
insertSql)) {
System.out.println("Inserting elevations");
System.out.println("transactionId = " + transactionId);
System.out.println("effectiveDate = " + effectiveDate);
System.out.println("transactionType = " + transactionType);
System.out.println("checkNumber = " + checkNumber);
System.out.println("referenceNumber = " + referenceNumber);
System.out.println("transactionCategory = " + transactionCategory);
System.out.println("balance = " + balance);
System.out.println("memo = " + memo);
insertStmt.setString(1, source);
insertStmt.setString(2, ""); // unique_id
insertStmt.setString(3, type); // type
insertStmt.setString(4, description);
insertStmt.setDate(5, postingDate);
insertStmt.setBigDecimal(6, amount.negate());
insertStmt.execute();
}
}
}
}
}
}
private void ingestChase(final Connection connection, final String year)
throws IOException, SQLException {
System.out.println("process chase " + year);
final String tableName = "chase";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getChase.sql").replace("${databaseName}",
"budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("getChase.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "chase";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
@ -750,13 +914,14 @@ public class Schema {
final String tableName = "paypal";
if (!tableExists(connection, year, tableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", tableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", tableName));
return;
}
final String sql = Util.getResourceAsString("getPaypal.sql").replace("${databaseName}",
"budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("getPaypal.sql") //
.replace("${databaseName}", "budget_" + year);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year);
final String source = "paypal";
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
@ -821,29 +986,42 @@ public class Schema {
private void ingestAmazonDigitalReturns(final Connection connection, final String year,
final String person) throws IOException, SQLException {
System.out.println("process amazon digital returns budget_" + year + " " + person);
final String digOrdItemsTableName = "amz_${person}_dig_ord_dig_items".replace("${person}",
person);
final String digOrdReturnsMonetaryTableName = "amz_${person}_dig_orders_returns_dig_orders_returns_monetary_1".replace(
"${person}", person);
String triliumName = "Amazon.${person}.Digital-Ordering.Digital Items" //
.replace("${person}", person);
final String digOrdItemsTableName = getTableName(triliumName);
if (digOrdItemsTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
if (!tableExists(connection, year, digOrdItemsTableName)) {
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year).replace("${tableName}", digOrdItemsTableName));
"${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdItemsTableName));
return;
}
triliumName = "Amazon.${person}.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1" //
.replace("${person}", person);
final String digOrdReturnsMonetaryTableName = getTableName(triliumName);
if (digOrdReturnsMonetaryTableName == null) {
System.out.println("Table corresponding to " + triliumName + " not found");
return;
}
if (!tableExists(connection, year, digOrdReturnsMonetaryTableName)) {
System.out.println(
" table ${databaseName}.${tableName} does not exist".replace("${databaseName}",
"budget_" + year).replace("${tableName}", digOrdReturnsMonetaryTableName));
System.out.println(" table ${databaseName}.${tableName} does not exist".replace(
"${databaseName}", "budget_" + year) //
.replace("${tableName}", digOrdReturnsMonetaryTableName));
return;
}
final String sql = Util.getResourceAsString("getDigitalReturns.sql").replace(
"${databaseName}", "budget_" + year).replace("${person}", person).replace(
"${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace(
"${digOrdItems}", digOrdItemsTableName).replace("${digOrdReturnsMonetary}",
digOrdReturnsMonetaryTableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace(
"${databaseName}", "budget_" + year).replace("${person}", person);
final String source = "amz.${person}.dig.ret".replace("${person}", person);
final String sql = Util.getResourceAsString("getDigitalReturns.sql") //
.replace("${databaseName}", "budget_" + year) //
.replace("${person}", person) //
.replace("${productNameCol}", "2023".equals(year) ? "title" : "product_name") //
.replace("${digOrdItems}", digOrdItemsTableName) //
.replace("${digOrdReturnsMonetary}", digOrdReturnsMonetaryTableName);
final String insertSql = Util.getResourceAsString("insertTransaction.sql") //
.replace("${databaseName}", "budget_" + year) //
.replace("${person}", person);
final String source = "amz.${person}.dig.ret".replace("${person}", person.toLowerCase());
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
try (ResultSet resultSet = stmt.executeQuery()) {
while (resultSet.next()) {
@ -873,8 +1051,8 @@ public class Schema {
private void linkRegex(final Connection connection, final String year, final Integer regexId,
final int transactionId) throws IOException, SQLException {
final String sql = Util.getResourceAsString("updateRegexLink.sql").replace(
"${databaseName}", "budget_" + year);
final String sql = Util.getResourceAsString("updateRegexLink.sql") //
.replace("${databaseName}", "budget_" + year);
try (PreparedStatement stmt = connection.prepareStatement(sql)) {
stmt.setInt(1, regexId);
stmt.setInt(2, transactionId);
@ -894,10 +1072,10 @@ public class Schema {
private Integer getCategoryPartId(final Connection connection, final Integer parentCategoryId,
final String categoryPart) throws SQLException, IOException {
final String getRootSql = Util.getResourceAsString("getRootCategoryId.sql").replace(
"${databaseName}", "budget");
final String getChildSql = Util.getResourceAsString("getChildCategoryId.sql").replace(
"${databaseName}", "budget");
final String getRootSql = Util.getResourceAsString("getRootCategoryId.sql") //
.replace("${databaseName}", "budget");
final String getChildSql = Util.getResourceAsString("getChildCategoryId.sql") //
.replace("${databaseName}", "budget");
Integer categoryId = null;
final String getSql = parentCategoryId == null ? getRootSql : getChildSql;
try (PreparedStatement stmt = connection.prepareStatement(getSql)) {
@ -919,10 +1097,10 @@ public class Schema {
}
if (categoryId == null) {
final String insertSql = parentCategoryId == null //
? Util.getResourceAsString("insertRootCategory.sql").replace("${databaseName}",
"budget") //
: Util.getResourceAsString("insertChildCategory.sql").replace("${databaseName}",
"budget");
? Util.getResourceAsString("insertRootCategory.sql") //
.replace("${databaseName}", "budget") //
: Util.getResourceAsString("insertChildCategory.sql") //
.replace("${databaseName}", "budget");
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql,
Statement.RETURN_GENERATED_KEYS)) {
insertStmt.setString(1, categoryPart);
@ -1142,9 +1320,9 @@ public class Schema {
return result;
}
private Set<String> generate(final String name, final int siblingCount,
final String triliumSearch) throws Exception {
final Set<String> years = new HashSet<>();
private void generate(final String name, final int siblingCount, final String triliumSearch,
final DatabaseInitializer databaseInitializer) throws Exception {
new HashSet<>();
final String output = getTriliumContent(
"/notes?search=" + triliumSearch + "&fastSearch=true");
final ObjectMapper mapper = new ObjectMapper();
@ -1169,19 +1347,22 @@ public class Schema {
if (!rootFound) {
continue;
}
years.addAll(generate(name, 1, noteNode));
generate(name, 1, null, noteNode, databaseInitializer);
}
return years;
return;
}
public Set<String> generate(String name, final int siblingCount, final JsonNode noteNode)
public void generate(String name, final int siblingCount, final JsonNode parentNoteNode,
final JsonNode noteNode, final DatabaseInitializer databaseInitializer)
throws Exception {
final Set<String> years = new HashSet<>();
final JsonNode titleNode = noteNode.get("title");
if (titleNode == null) {
return years;
return;
}
String childName = titleNode.textValue();
final JsonNode parentTitleNode = parentNoteNode == null ? null
: parentNoteNode.get("title");
final String parentTitle = parentTitleNode == null ? null : parentTitleNode.textValue();
String ext = null;
final int lastIndexOfDot = childName.lastIndexOf(".");
if (lastIndexOfDot >= 0) {
@ -1199,24 +1380,28 @@ public class Schema {
if (noteIdNode != null) {
final String noteId = noteIdNode.textValue();
final List<String[]> records = getTriliumRecords("/notes/" + noteId + "/content");
years.addAll(generate(name, siblingCount, records,
childName.startsWith("FirstbankDownload"), "qb.csv".equals(childName)));
generate(name, siblingCount, records, "Firstbank".equals(parentTitle),
"qb.csv".equals(childName), databaseInitializer);
}
}
final JsonNode childNodeIdsNode = noteNode.get("childNoteIds");
if (childNodeIdsNode == null) {
return years;
return;
}
for (final JsonNode childNodeIdNode : childNodeIdsNode) {
final String childNodeId = childNodeIdNode.textValue();
final JsonNode childNode = getTriliumJsonContent("/notes/" + childNodeId);
years.addAll(generate(name, childNodeIdsNode.size(), childNode));
generate(name, childNodeIdsNode.size(), noteNode, childNode, databaseInitializer);
}
return years;
return;
}
private Set<String> generate(String name, final int siblingCount, final File file)
throws Exception {
private boolean isFirstBankCsv(final String name) {
return name.startsWith("FirstbankDownload") || name.equals("2025-checking");
}
private void generate(String name, final int siblingCount, final File file,
final DatabaseInitializer databaseInitializer) throws Exception {
String childName = file.getName();
String ext = null;
if (file.isFile()) {
@ -1232,28 +1417,54 @@ public class Schema {
else if (siblingCount > 1) {
name = name + "." + childName;
}
final Set<String> years = new HashSet<>();
new HashSet<>();
if (file.isDirectory()) {
final File[] children = file.listFiles();
for (final File child : children) {
years.addAll(generate(name, children.length, child));
generate(name, children.length, child, databaseInitializer);
}
}
else if (file.isFile() && "csv".equalsIgnoreCase(ext)) {
final List<String[]> records = readAllLines(Paths.get(file.getPath()));
years.addAll(generate(name, siblingCount, records,
file.getName().startsWith("FirstbankDownload"), "qb.csv".equals(file.getName())));
generate(name, siblingCount, records, isFirstBankCsv(file.getName()),
"qb.csv".equals(file.getName()), databaseInitializer);
}
return years;
return;
}
private static Pattern NAME_PATTERN = Pattern.compile("^Budget\\.([0-9]+)\\.(.*)$");
private static Map<String, String> TABLE_NAMES = new HashMap<>();
static {
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Items", "amz_steve_dig_ord_items");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Items", "amz_sandy_dig_ord_items");
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Orders", "amz_steve_dig_ord_ords");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Orders", "amz_sandy_dig_ord_ords");
// Amazon.${person}.Digital-Ordering.Digital Orders Monetary
TABLE_NAMES.put("Amazon.Steve.Digital-Ordering.Digital Orders Monetary",
"amz_steve_dig_ord_ords_mon");
TABLE_NAMES.put("Amazon.Sandy.Digital-Ordering.Digital Orders Monetary",
"amz_sandy_dig_ord_ords_mon");
TABLE_NAMES.put("Amazon.Steve.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1",
"amz_steve_dig_ords_rets_mon");
TABLE_NAMES.put("Amazon.Sandy.Digital.Orders.Returns.Digital.Orders.Returns.Monetary.1",
"amz_sandy_dig_ords_rets_mon");
TABLE_NAMES.put("Amazon.Steve.Retail.OrderHistory", "amz_steve_ret_ord_hist");
TABLE_NAMES.put("Amazon.Sandy.Retail.OrderHistory", "amz_sandy_ret_ord_hist");
TABLE_NAMES.put("Chase", "chase");
TABLE_NAMES.put("Paypal", "paypal");
TABLE_NAMES.put("Firstbank", "first_bank");
TABLE_NAMES.put("Firstbank.2025-checking", "first_bank");
TABLE_NAMES.put("Citi", "citi");
TABLE_NAMES.put("Discover", "discover");
TABLE_NAMES.put("Elevations", "elevations");
TABLE_NAMES.put("Elevations.2025-checking", "elevations");
}
private Set<String> generate(final String name, final int siblingCount,
final List<String[]> records, final boolean isFirstBank, final boolean isQuickBooks)
throws SQLException {
private void generate(final String name, final int siblingCount, final List<String[]> records,
final boolean isFirstBank, final boolean isQuickBooks,
final DatabaseInitializer databaseInitializer) throws SQLException {
int recordIndex = 0;
final Set<String> years = new HashSet<>();
new HashSet<>();
List<FieldInfo> fields = null;
for (final String[] record : records) {
if (recordIndex == 0) {
@ -1398,35 +1609,72 @@ public class Schema {
}
final Matcher matcher = NAME_PATTERN.matcher(name);
if (!matcher.matches()) {
throw new RuntimeException("Name does not begin with Budget.[year]");
System.out.println("Skipping " + name + " because name pattern does not match");
return;
}
final String year = matcher.group(1);
years.add(year);
final String tableName = decamel(matcher.group(2)).replace("budget_", "").replace("amazon_",
"amz_").replace("_digital_", "_dig_").replace("_information", "_inf").replace(
"_payment_", "_pmt_").replace("_ordering_", "_ord_").replace("_customer_",
"_cust_").replace("_communication_", "_comm_").replace("_preferences_",
"_pref_").replace("_physical_", "_phys_").replace("_whole_foods_", "_wf_");
final String triliumName = matcher.group(2);
final String tableName = getTableName(triliumName);
if (fields != null) {
System.out.println("Loading CSV " + name);
if (tableName == null) {
System.out.println("Skipping CSV " + name + " because table not found");
}
else {
databaseInitializer.initializeDatabase(year);
try (Connection connection = pool.getConnection(8, false, true)) {
final String qualifiedTableName = "budget_" + year + "." + tableName;
System.out.println("Loading CSV " + name + " into " + qualifiedTableName);
dbExecute(connection, "drop table if exists " + qualifiedTableName);
dbExecute(connection, getCreateTableSql(qualifiedTableName, fields));
doInserts(connection, qualifiedTableName, fields, records, isFirstBank,
isQuickBooks);
}
}
return years;
}
return;
}
private static String getTableName(final String triliumName) {
final String tableName = TABLE_NAMES.get(triliumName);
return tableName;
/*
if (tableName != null) {
return tableName;
}
return decamel(triliumName) //
.replace("budget_", "") //
.replace("amazon_", "amz_") //
.replace("_digital_", "_dig_") //
.replace("_information", "_inf") //
.replace("_payment_", "_pmt_") //
.replace("_ordering_", "_ord_") //
.replace("_customer_", "_cust_") //
.replace("_communication_", "_comm_") //
.replace("_preferences_", "_pref_") //
.replace("_physical_", "_phys_") //
.replace("_whole_foods_", "_wf_") //
.replace("_subscriptions_subscriptions_", "_ssubs_") //
.replace("_subscriptions_", "_subs_") //
.replace("_subscription_", "_sub_") //
.replace("_status_", "_stat_") //
.replace("_billing_", "_bill_") //
.replace("_refunds_", "_refs_") //
.replace("_returns_", "_rets_") //
.replace("_orders_", "_ords") //
.replace("_monetary_", "_mon_") //
.replace("_and_", "_") //
.replace("_data", "_dat") //
.replace("_history", "_hist");
*/
}
private void saveYears(final Set<String> years) throws SQLException, IOException {
try (Connection connection = pool.getConnection(8, false, true)) {
dbExecute(connection, "drop table if exists budget.years");
dbExecute(connection, Util.getResourceAsString("createYearsTable.sql").replace(
"${databaseName}", "budget"));
final String insertSql = Util.getResourceAsString("insertYear.sql").replace(
"${databaseName}", "budget");
dbExecute(connection, Util.getResourceAsString("createYearsTable.sql") //
.replace("${databaseName}", "budget"));
final String insertSql = Util.getResourceAsString("insertYear.sql") //
.replace("${databaseName}", "budget");
for (final String year : years) {
try (PreparedStatement insertStmt = connection.prepareStatement(insertSql)) {
insertStmt.setInt(1, Integer.parseInt(year));
@ -1454,8 +1702,18 @@ public class Schema {
final String sql = getInsertSql(tableName, record, fields);
try (final PreparedStatement stmt = connection.prepareStatement(sql)) {
setValues(stmt, record, fields);
try {
stmt.execute();
}
catch (final SQLException e) {
System.out.println("SQL Exception: " + e);
System.out.println(sql);
for (int i = 0; i < record.length; i++) {
System.out.println(i + " " + record[i]);
}
throw e;
}
}
}
recordIndex++;
}
@ -1579,13 +1837,14 @@ public class Schema {
}
}
private String decamel(final String name) {
private static String decamel(final String name) {
if (name == null || name.trim().length() == 0) {
return "unknown";
}
final StringBuilder sb = new StringBuilder();
final char[] chars = name.replace("ID", "Id").replace("RMA", "Rma").replace("ASIN",
"Asin").toCharArray();
final char[] chars = name.replace("ID", "Id") //
.replace("RMA", "Rma") //
.replace("ASIN", "Asin").toCharArray();
int charIndex = 0;
char prevChar = '_';
while (charIndex < chars.length) {

16
src/main/resources/createBudgetAmountTable.sql Normal file
View file

@ -0,0 +1,16 @@
create table ${databaseName}.budget_amount (
category_id int not null primary key,
year_amount decimal(10,2),
jan_amount decimal(10,2),
feb_amount decimal(10,2),
mar_amount decimal(10,2),
apr_amount decimal(10,2),
may_amount decimal(10,2),
jun_amount decimal(10,2),
jul_amount decimal(10,2),
aug_amount decimal(10,2),
sep_amount decimal(10,2),
oct_amount decimal(10,2),
nov_amount decimal(10,2),
dec_amount decimal(10,2)
)

1
src/main/resources/dropBudgetAmountTable.sql Normal file
View file

@ -0,0 +1 @@
drop table if exists ${databaseName}.budget_amount

1
src/main/resources/dropTransactionTable.sql Normal file
View file

@ -0,0 +1 @@
drop table if exists ${databaseName}.transaction

15
src/main/resources/getElevations.sql Normal file
View file

@ -0,0 +1,15 @@
select
transaction_id,
posting_date,
effective_date,
transaction_type,
amount,
check_number,
reference_number,
description,
transaction_category,
type,
balance,
memo,
extended_description
from ${databaseName}.elevations