diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..49f7be5 --- /dev/null +++ b/.classpath @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.factorypath b/.factorypath new file mode 100644 index 0000000..4791d72 --- /dev/null +++ b/.factorypath @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..567d65d --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +/.attach_pid* +/*.html +/pages +/*.xls +/*.pdf +/sandy/ +/amazon/ +/amazon-steve* +*.log +/budget.sql +/budget/ +/build/ +/categorized.lst +/chromedriver +/databases.lst +/explore*.sql +/list-descriptions.sql +/load.sql +/export* +/gnucash* +/hrefs +/*.out +/*-urls +/tables.sql +/target +/temp.sql +/test-urls +/test_files +/upload.sql diff --git a/.project b/.project new file mode 100644 index 0000000..438369c --- /dev/null +++ b/.project @@ -0,0 +1,37 @@ + + + com.stephenschafer.budget + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.wst.common.project.facet.core.builder + + + + + org.eclipse.wst.validation.validationbuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + org.eclipse.jem.workbench.JavaEMFNature + org.eclipse.wst.common.modulecore.ModuleCoreNature + org.eclipse.wst.common.project.facet.core.nature + org.eclipse.jdt.core.javanature + org.eclipse.wst.jsdt.core.jsNature + + diff --git a/.settings/.jsdtscope b/.settings/.jsdtscope new file mode 100644 index 0000000..92e666d --- /dev/null +++ b/.settings/.jsdtscope @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 0000000..dfa4f3a --- /dev/null +++ b/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=true +org.eclipse.jdt.apt.genSrcDir=target/generated-sources/annotations +org.eclipse.jdt.apt.genTestSrcDir=target/generated-test-sources/test-annotations diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..26aef1a --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,12 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.processAnnotations=enabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component new file mode 100644 index 0000000..c84696c --- /dev/null +++ b/.settings/org.eclipse.wst.common.component @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.settings/org.eclipse.wst.common.project.facet.core.xml b/.settings/org.eclipse.wst.common.project.facet.core.xml new file mode 100644 index 0000000..f270c78 --- /dev/null +++ b/.settings/org.eclipse.wst.common.project.facet.core.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/.settings/org.eclipse.wst.jsdt.ui.superType.container b/.settings/org.eclipse.wst.jsdt.ui.superType.container new file mode 100644 index 0000000..3bd5d0a --- /dev/null +++ b/.settings/org.eclipse.wst.jsdt.ui.superType.container @@ -0,0 +1 @@ +org.eclipse.wst.jsdt.launching.baseBrowserLibrary \ No newline at end of file diff --git a/.settings/org.eclipse.wst.jsdt.ui.superType.name b/.settings/org.eclipse.wst.jsdt.ui.superType.name new file mode 100644 index 0000000..05bd71b --- /dev/null +++ b/.settings/org.eclipse.wst.jsdt.ui.superType.name @@ -0,0 +1 @@ +Window \ No newline at end of file diff --git a/.settings/org.eclipse.wst.validation.prefs b/.settings/org.eclipse.wst.validation.prefs new file mode 100644 index 0000000..04cad8c --- /dev/null +++ b/.settings/org.eclipse.wst.validation.prefs @@ -0,0 +1,2 @@ +disabled=06target +eclipse.preferences.version=1 diff --git a/README.md b/README.md index 2cf6ff8..e0a40b7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # com.stephenschafer.budget -Budget java app \ No newline at end of file +Create budget from downloaded CSV files \ No newline at end of file diff --git a/WebContent/META-INF/MANIFEST.MF b/WebContent/META-INF/MANIFEST.MF new file mode 100644 index 0000000..5e94951 --- /dev/null +++ b/WebContent/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Class-Path: + diff --git a/WebContent/WEB-INF/web.xml b/WebContent/WEB-INF/web.xml new file mode 100644 index 0000000..b0534bf --- /dev/null +++ b/WebContent/WEB-INF/web.xml @@ -0,0 +1,87 @@ + + + upload + com.stephenschafer.budget.UploadServlet + 1 + + + upload + /upload + + + get-transaction + com.stephenschafer.budget.GetTransactionServlet + + + get-transaction + /get-transaction + + + get-transactions + com.stephenschafer.budget.GetTransactionsServlet + + + get-transactions + /get-transactions + + + get-recipient + com.stephenschafer.budget.GetRecipientServlet + + + get-recipient + /get-recipient + + + get-accounts + com.stephenschafer.budget.GetAccountsServlet + + + get-accounts + /get-accounts + + + get-budgets + com.stephenschafer.budget.GetBudgetsServlet + + + get-budgets + /get-budgets + + + add-account + com.stephenschafer.budget.AddAccountServlet + + + add-account + /add-account + + + save-distributions + com.stephenschafer.budget.SaveDistributionsServlet + + + save-distributions + /save-distributions + + + save-recipient-distributions + com.stephenschafer.budget.SaveRecipientDistributionsServlet + + + save-recipient-distributions + /save-recipient-distributions + + + delete-account + com.stephenschafer.budget.DeleteAccountServlet + + + delete-account + /delete-account + + \ No newline at end of file diff --git a/budget.properties b/budget.properties new file mode 100644 index 0000000..9e99106 --- /dev/null +++ b/budget.properties @@ -0,0 +1,6 @@ +log-filename=budget.log +db-url=jdbc:mysql://localhost:3306/budget_2023?serverTimezone=UTC&useSSL=false +db-username=elephant +db-password=zsKRtBw6gPi0B0hMg1c2 +trilium-url=http://nuc1:8080/etapi +trilium-password=scale-daughter-twiddling-educate-afraid \ No newline at end of file diff --git a/build-jar b/build-jar new file mode 100755 index 0000000..89da5c5 --- /dev/null +++ b/build-jar @@ -0,0 +1,7 @@ +#!/bin/sh +if mvn -f pom-jar.xml clean install > build-jar.log 2> build-jar.err.log; then + echo "success" +else + echo "failure" + exit 1 +fi \ No newline at end of file diff --git a/category-regex.lst b/category-regex.lst new file mode 100644 index 0000000..6a27e50 --- /dev/null +++ b/category-regex.lst @@ -0,0 +1,587 @@ +accounting, CHECK # 4046, , firstbank, , peakview +auto.equipment, .*garmin.*, i, +auto.gasoline, .*CONOCO.*, , chase +auto.gasoline, .*SHELL OIL.* +auto.gasoline, CONOCO .*, citi +auto.license, AIR CARE COLORADO.*, , chase +auto.services, .*PEDERSEN TOYOTA.* +auto.subscriptions, .*AARP ROADSIDE.*, , chase +auto.subscriptions, ROADSIDE ASSISTANCE.*, , chase +bank-fees, .* FEE, , firstbank +cellphone, .* VZ WIRELESS VW, , firstbank +clothing, .* barefoot shoes .*, i +clothing, .* cotton .* leggings .*, i +clothing, .* Grip Socks .* +clothing, .* House Shoes .* +clothing, .* Orthopeic Flip Flops .* +clothing, .* Underwear .* +clothing, .*columbia.*, i, +clothing, .*dress belt.*, i, +clothing, .*Flip Flop Socks.*, , amz.sandy.retail.ord +clothing, .*LLBEANINC.* +clothing, .*Nightgown.* +clothing, .*Wireless Bra.*, , amz.sandy.retail.ord +clothing, Cotton Tops.* +clothing, Gaiam.* +clothing, HAFLINGER.* +clothing, HiGropcore.* +clothing, Hugh Ugoli.* +clothing, Island Genius.* +clothing, LeIsfIt.* +clothing, MACYS.*, , chase +clothing, PAYPAL \*ZAPPOS.*, , chase +clothing, vanity fair .* nylon brief .*, i +clothing, Weintee .* +communication, .*DISCORD.* +computer.equipment, .* Charger Cable .* +computer.equipment, .* Charging Cable.* +computer.equipment, .* Computer Power Supply .* +computer.equipment, .* ipad .*, i +computer.equipment, .* Lightning Cable .* +computer.equipment, .* Magnetic Fast Charger .* +computer.equipment, .* mechanical keyboard .*, i, +computer.equipment, .* Power Adapter .* +computer.equipment, .* Power Over Ethernet .* +computer.equipment, .*3\.5mm jack.*, i, +computer.equipment, .*APPLE\.COM/US.* +computer.equipment, .*av cable.*, i, +computer.equipment, .*BESTBUY.*, , chase +computer.equipment, .*CANAKIT.*, , chase +computer.equipment, .*cyberpower.*, i, +computer.equipment, .*data charging cable.*, i, +computer.equipment, .*deskstar.*, i, +computer.equipment, .*DVD Drive.* +computer.equipment, .*flash memory.*, i, +computer.equipment, .*led display.*, i, +computer.equipment, .*mid tower case.*, i, +computer.equipment, .*mini plug.*, i, +computer.equipment, .*monitor cable.*, i, +computer.equipment, .*NEWEGG.* +computer.equipment, .*oem drive.*, i, +computer.equipment, .*scanner.*, i, +computer.equipment, .*videosecu.*, i, +computer.equipment, .*Wireless Vertical Ergonomic Optical Mouse.*, , amz.sandy.retail.ord +computer.equipment, Arducam .* +computer.equipment, Corsair .* +computer.equipment, HGST .* +computer.equipment, Micro HDMI to HDMI Adapter.* +computer.equipment, Mohu Leaf .* +computer.equipment, NETGEAR .* +computer.equipment, Raspberry Pi .* +computer.equipment, SiliconDust HDHomeRun .* +computer.equipment, U6 PRO .* +computer.equipment, WWZMDiB .* +computer.software, DejaOffice, , amz.sandy.dig.ord, 10 +computer.software, File Manager, , amz.sandy.dig.ord, 10 +computer.subscriptions, .*APPLE\.COM/BILL.* +computer.supplies, .* Compressed Air Duster .* +computer.supplies, .* Compressed Gas Duster.* +dental.services, NORTHERN COLORADO ENDO.*, , discover +dental.services, OWENS DENTAL.* +dental.services, RECLAIM DENTISTRY .* +dental.services, THE THERMOGRAM CENTER .*, , discover +dining, .*AUSTINS AMERICAN .*, , chase +dining, .*MOOT HOUSE.* +dining, .*Simmer ECOM.*, , chase +dining, .*SIMMER.*Food.*Drink.* +dining, .*YOUNG.*CAFE.* +education, PAYPAL \*DAILYOM .*, , chase +education, PAYPAL \*WHOLE WOMAN .*, , chase +entertainment.books, .*BN PAPERSRC .*, , chase, , Barnes and Noble +entertainment.books, Barnes \& Noble Inc .*, , paypal, 10 +entertainment.games, Humble Bundle.*, , paypal, 10 +entertainment.games, Valve Corp.*, , paypal, 10 +entertainment.subscriptions.spotify, Spotify USA .*, , paypal, 10 +entertainment.subscriptions.gaia, .*GAIA.* +entertainment.subscriptions.netflix, .*NETFLIX.* +entertainment.subscriptions.patreon, .*PATREON *MEMBER.* +entertainment.subscriptions.samsung, .*SAMSUNG.*, , chase +entertainment.subscriptions.spotify, .*SPOTIFY.* +entertainment.subscriptions.youtube, .*YOUTUBE SUBSC.* +entertainment.subscriptions.apple, Apple Services .*-10\.79 .*, , paypal +computer.support, Apple Services .*7\.99 .*, , paypal +computer.support, Apple Services .*2\.99 .*, , paypal +computer.support, Apple Services .*3\.99 .*, , paypal +entertainment.subscriptions.comcast, CABLE COMCAST.*, , firstbank +entertainment.subscriptions.antenna, CHANNELS .* +entertainment.subscriptions.newspaper, Ft Coll Coloradoan.* +entertainment.subscriptions.google, Google .* .2\.17 USD.*, , paypal, 10 +entertainment.subscriptions.hallmark, Hallmark Movies Now, , amz.sandy.dig.ord, 10 +entertainment.subscriptions.hbo, HBO, , amz.sandy.dig.ord, 10 +entertainment.subscriptions.hulu, Hulu .*, , paypal, 10 +entertainment.subscriptions.netflix, Netflix, , amz.sandy.dig.ord, 10 +entertainment.subscriptions.netflix, Netflix\.com .*, , paypal, 10 +entertainment.subscriptions.paramount, Paramount\+ with SHOWTIME, , amz.sandy.dig.ord, 10 +entertainment.subscriptions.patreon, Patreon .*, , paypal, 10 +entertainment.subscriptions.hulu, PAYPAL \*HULU.*, , chase +entertainment.subscriptions.samsung, PAYPAL \*SAMSUNGELEC .*, , chase +entertainment.tvshows/cds, Prime Video.* +entertainment.subscriptions.samsung, Samsung Electronics America.*, , paypal, 10 +entertainment.subscriptions.starz, STARZ, , amz.sandy.dig.ord, 10 +entertainment.subscriptions.tivo, TIVO PLATFORM TECH.* +entertainment.subscriptions.tivo, TIVOPLATTECH.*, , chase +entertainment.books, .*, , amz.sandy.dig.ord +entertainment.books, .*, , amz.steve.dig.ord +entertainment.books, .*transmetropolitan.*, i, +entertainment.books, Alternative Cures.* +entertainment.books, BOOKSHOP\.ORG .* +entertainment.books, this is not a game, i, +entertainment.books, uncanny x.men.*, i, +entertainment.books, zero history, i, +entertainment.equipment, .*amplifier.*, i, +entertainment.equipment, .*speakers.*, i, +entertainment.equipment, wii.*, i, +entertainment.games, .* Jigsaw Puzzle .* +entertainment.games, .*ARENANETLLC.* +entertainment.games, .*HUMBLEBUNDL.* +entertainment.games, .*STEAM GAMES.* +entertainment.tvshows/cds, .* Avengers: Endgame, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Battle Los Angeles, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Battleship, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Black Panther .*, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Captain Marvel, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Fantastic Fungi Remastered, , , 10 +entertainment.tvshows/cds, Jesse Stone .*, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Jupiter Ascending, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Justice League, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Knives Out, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Looper .*, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Lungs: The B.Sides, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, people of the wind, i, +entertainment.tvshows/cds, Predestination, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Spider.Man: .*, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Star Wars: .*, , amz.sandy.dig.ord, 10 +entertainment.tvshows/cds, Wild, , amz.sandy.dig.ord, 10 +home.yard-garden.equipment, .*Soil Moisture Meter.* +home.yard-garden.equipment, WayinTop .* +home.yard-garden.supplies, .*BATH NURSERY.*, , chase +home.yard-garden.supplies, .*GULLEY GREENHOUSE.*, , chase +home.yard-garden.supplies, .*HIGHMOWINGS.* +home.yard-garden.supplies, .*URBAN FARMER.* +home.yard-garden.supplies, .*URBANFARMER.* +gifts, .*ETSY GIFTCARD.*, , chase +donations, .*GREENAMERIC .* +donations, .*Rocky Mountain PBS.* +donations, .*THUNDERBIRD.* +donations, .*WIKIPEDIA.* +misc, Ancestry\.com .*, , paypal +gifts, CHECK # 4044, , firstbank, , leah +dues-subscriptions-memberships, CHECK # 4077, , firstbank, , aarp +donations, CPT12.* +donations, Green America .*, , paypal, 10 +groceries, .* Almond Flour.* +groceries, .* Microgreens Growing Kit.* +groceries, .* Nutritional Yeast .* +groceries, .* Protein Bar.* +groceries, .* SuperFood Starter Culture .* +groceries, .* Yogurt Starter Culture .* +groceries, .*\(panda01\), , amz.sandy.retail.ord +groceries, .*Bee Pollen.*, , amz.sandy.retail.ord +groceries, .*Breath Mints.*, , amz.sandy.retail.ord +groceries, .*Collagen Peptides.* +groceries, .*FAIRCHILDS VINEGAR.*, , chase +groceries, .*Gelatin Powder.* +groceries, .*INSTACART.* +groceries, .*KING SOOPERS.*, , chase +groceries, .*Macadamia Nuts.* +groceries, .*Manuka Honey.*, , amz.sandy.retail.ord +groceries, .*NATIVE HILL.* +groceries, .*Organic.*, i, , -1 +groceries, .*Plantain Chips.*, , amz.sandy.retail.ord +groceries, .*Pork Rinds.* +groceries, .*RED ROBIN.* +groceries, .*ribeye.*, i, +groceries, .*SWEETMARIAS.* +groceries, .*THRIVEMARKE.* +groceries, .*Top Sirloin Steak.*, , amz.sandy.retail.ord +groceries, .*Yogurt Coconut.* +groceries, BochaSweet.*, , amz.sandy.retail.ord +groceries, Bulletproof XCT MCT Oil.* +groceries, Califia Farms.* +groceries, CELTIC OCEAN .*, , chase +groceries, COSTCO WHSE .*, , citi +groceries, Essenzefruits.* +groceries, Instacart .*, , paypal, 10 +groceries, Kerrygold.* +groceries, Large paper bag fee.* +groceries, LILY'S.*, i +groceries, LUCKY.*MARKET.* +groceries, MT. CAPRA.* +groceries, NATURAL GROCERS FC.* +groceries, PALEOVALLEY .*, , chase +groceries, PAYPAL .IHERB LLC .* +groceries, PAYPAL \*WILDERNESSP .*, , chase +groceries, Pecan Shop .* +groceries, Pork Loin Chop .* +groceries, PUR Gum .* +groceries, SPROUTS FARMERS MAR .*, , chase +groceries, Sweet Maria.*, , paypal, 10 +groceries, Vital Farms.*, i, amz.sandy.retail.ord +groceries, Wild Planet .* +groceries, Wilderness Poets .*, , paypal, 10 +groceries, Wilderness Poets.*, , amz.sandy.retail.ord +health.equipment, .* Pulse Oximeter .* +personal.care, .* sleep mask .*, i +health.equipment, .*APOLLONEURO .* +health.equipment, .*balance disc.*, i, +health.equipment, .*heart rate sensor.*, i, +health.equipment, .*INAP SLEEP THERAPY.*, , chase +health.equipment, .*inversion therapy chair.*, i, +health.equipment, .*NOVAALAB.* +health.equipment, .*RINGCONN.* +health.equipment, .*SOMNICSHEAL.* +health.equipment, Apollo Neuroscience.*, , paypal +health.equipment, INAP SLEEP .* +health.equipment, LifePro.* +health.equipment, Omron .* +health.equipment, Optimal Circadian Health.*, , paypal +health.equipment, Oxygen Research Institute .*, , paypal +health.equipment, PAYPAL \*HEALTHYLINE .*, , chase +health.supplies, .* Hearing Aid Batteries.* +health.supplies, .*Breathable Strips.*, i, +health.supplies, .*Hypoallergenic Tape.*, i, +health.supplies, Dynarex .* +health.supplies, Nasacort .* +health.supplies, Polident .* +personal.care, Vibrant Blue Oils.*, , paypal +personal.electronics, Walmart\.com .* -24\.90.*, , paypal +hobby.equipment, uxcell .* +hobby.supplies, .* Acrylic Plastic Cement.* +hobby.supplies, .* rubber roller .*, i +hobby.supplies, .* Wood Screws.* +hobby.supplies, HATCHBOX .* +hobby.supplies, speedball .*, i +home.housewares, .*Bamboo Toilet Stool .*, , amz.sandy.retail.ord +home.appliances, LABIGO.* +home.housewares, .* blueair .* filter .*, i +home.appliances, .* Cordless Stick Vacuum .* +home.appliances, .* Dremel .* +home.appliances, .* Extension Cord .* +home.appliances, .* Extension Cord.* +home.housewares, .* replacement filter for levoit .*, i +home.appliances, .* Lithium Coin Battery .* +home.appliances, .*BLACK\+DECKER dustbuster.*, , amz.sandy.retail.ord +home.appliances, .* BLUEAIR .* Air Purifier.*, , amz.sandy.retail.ord +home.appliances, BLUEAIR Air Purifier.*, , amz.sandy.retail.ord +home.housewares, BLUEAIR Blue Pure .* Replacement Filter.*, , amz.sandy.retail.ord +home.appliances, .*cordless phone.*, i, +home.appliances, .*EMF Meter.* +home.appliances, .*HOME DEPOT.* +home.appliances, .*JOSEPH S HARDWARE.*, , chase +personal.electronics, .*LED Neck Reading Light.* +home.appliances, .*LOWES\.COM.* +home.appliances, .*Phone Headset.* +home.appliances, .*power strip.*, i, +home.appliances, .*PURE WATER PRODUCTS.* +home.appliances, BLACK+DECKER dustbuster.* +home.housewares, BLUEAIR .* Filter +home.appliances, Dremel.* +home.appliances, Kidde .* +home.appliances, levoit .*, i +home.appliances, Portable Charger Power Bank .* +home.appliances, Ryhiac .* +home.appliances, Sunlite .* +home.appliances, SUPRUS .* +home.appliances, Temtop .* +home.housewares, .* Toilet Lid Cover.* +home.furnishings, .*ART\.COM.* +home.furnishings, .*COLORADO BLINDS.*, , chase +home.furnishings, .*FURNITURE ROW.*, , chase +home.housewares, .*Toilet Seat.* +home.furnishings, .*Wall calendar.*, i, +home.furnishings, California Design Den.* +home.furnishings, HOME SMILE.* +home.furnishings, oskas .* +home.furnishings, PAYPAL \*BED BATH .*, , chase +home.furnishings, PUDDING CABIN .* +home.furnishings, CHECK # 4037, , firstbank, , move couch +home.maintenance, CHECK # 4041, , firstbank, , chimney cleaners +home.repair, CHECK # 4072, , firstbank, , freezer repair +home.repair, CHECK # 4073, , firstbank, , freezer repair +home.repair, CHECK # 4076, , firstbank, , fence repair +home.repair, CHECK # 4089, , firstbank, , fix water line in laundry room +home.housewares, .* hand soap.*, i, , 10 +home.housewares, .* Light Bulb.*, , , 10 +home.housewares, Everyone.* +home.housewares, MRS\. MEYER'S.* +home.housewares, PUREBURG .* +home.housewares, Quilted Northern .* +income, INTEREST EARNED, , firstbank +income, INVESTMENT EDWARD JONES, , firstbank +income, PAYROLL TRINET.*, , firstbank +income, XXSOC SEC SSA.*, , firstbank +insurance.auto, .* SAFECO, , firstbank +insurance.auto, Safeco Corporation .* +insurance.life, .*transamerica.*, i, , 10 +insurance.medical, .* HUMANA.*, , firstbank +insurance.product, asurion .*, i, , , breville +internet-services, .*GODADDY\.COM.* +internet-services, .*GOOGLE STORAGE.* +internet-services, .*NAMECHEAP.* +internet-services, .*NETDORM.* +internet-services, Amazon Drive, , amz.sandy.dig.ord, 10 +internet-services, NAME-CHEAP.* +internet-services, FORT COLLINS CONNEXION .*, , discover +internet-services, VISA FORT COLLINS CONNEXION.*, , firstbank +investment, MONEYLINK SCHWAB.*, , firstbank +investment, TRANSFER PAYPAL, , firstbank +home.appliances, .* Coffee Maker .* +home.appliances, .*COMFEE.*Electric Kettle.*, , amz.sandy.retail.ord +home.appliances, .*food processor.*, i, +home.appliances, Instant Pot.* +home.appliances, Vitamix .* +home.housewares, .* Pill Cutter .* +home.housewares, hemp mats .*, i +home.housewares, Melitta .* +home.housewares, Seventh Generation .*, , , 10 +home.housewares, .* coffee mug .*, i +home.housewares, .* cutting board.*, i +home.housewares, .* Dish Drying Mat .* +home.housewares, .* Jar Lids.* +home.housewares, .* Kitchen Utensils .* +home.housewares, .* Knife Edge Guards.* +home.housewares, .* Mason Jar Shaker Lids .* +home.housewares, .* Rubber Spatula.* +home.housewares, .* spatula .*, i +home.housewares, .*Coffee Carafe Tea Pot.*, , amz.sandy.retail.ord +home.housewares, .*Dish Towels.*, , amz.sandy.retail.ord +home.housewares, .*Donut Pan.* +home.housewares, .*Foam Ear Plugs.* +home.housewares, .*Food Storage Containers.* +home.housewares, .*GREENPAN.* +home.housewares, .*kitchen knife.*, i, +home.housewares, Euro Cuisine.* +home.housewares, Fino Pour-Over Coffee Brewing Filter Cone.* +home.housewares, GMISUN Oil Dispenser.* +home.housewares, GreenPan.* +home.housewares, IDEATECH.* +home.housewares, LOVE MOMENT.* +home.housewares, Misen .* +home.housewares, Noble Home .* +home.housewares, OXO .* +home.housewares, Pour Over Coffee Dripper.* +home.housewares, souper cubes .*, i +home.housewares, SPLF .* +home.housewares, TeamFar .* +home.housewares, Wooden Spurtle .* +home.housewares, Zeppoli .* +medical.services, .* UCHEALTH .*, , chase +medical.services, .* UCHEALTH .*, , discover +medical.services, .*Ortho Spine Ctr Rockies.* +medical.services, .*PERFORMANCE PHYSICAL THER.* +medical.services, .*UCHEALTH .*, , discover +medical.services, .*UCHEALTH.*, , chase +medical.services, ACCTVERIFY ATLAS\.MD, , firstbank +medical.services, ADVANCED MEDICAL IMAGING.*, , chase +medical.services, AUDIOLOGY GROUP .* +health.practitioners, CHECK # 4035, , firstbank, , shelby kahl +medical.services, CHECK # 4036, , firstbank, , +health.practitioners, CHECK # 4038, , firstbank, , shelby kahl +health.practitioners, CHECK # 4039, , firstbank, , shelby kahl +health.practitioners, CHECK # 4042, , firstbank, , shelby kahl +health.practitioners, CHECK # 4043, , firstbank, , shelby kahl +health.practitioners, CHECK # 4045, , firstbank, , shelby kahl +health.practitioners, CHECK # 4048, , firstbank, , shelby kahl +health.practitioners, CHECK # 4053, , firstbank, , shelby kahl +medical.services, CHECK # 4071, , firstbank, , copay kidney dr. +health.practitioners, CHECK # 4075, , firstbank, , ashley +health.practitioners, CHECK # 4078, , firstbank, , shelby kahl +health.practitioners, CHECK # 4081, , firstbank, , shelby kahl +health.practitioners, CHECK # 4082, , firstbank, , shelby kahl +health.practitioners, CHECK # 4083, , firstbank, , shelby kahl +health.practitioners, CHECK # 4085, , firstbank, , shelby kahl +health.practitioners, CHECK # 4086, , firstbank, , shelby kahl +health.practitioners, CHECK # 4090, , firstbank, , shelby kahl +supplements, CHECK # 4091, , firstbank, , shelby kahl +medical.services, CHIROPRACTIC ASSOCIATES.* +health.practitioners, IV NUTRITION .* +medical.services, NORTH VISTA.* +medical.services, OBGA MY OBGYN .*, , discover +medical.services, UCHEALTH .*, , firstbank +medical.subscriptions, DFC OF NOC ATLAS\.MD, , firstbank +medical.subscriptions, DFC OF NOCO .*, , chase +medical.subscriptions, DFC OF NOCO .*, , discover +medical.supplies, WALGREENS .*, , chase +medicines, .*BELMAR PHARMACY.*, , chase +medicines, .*EBM MEDICAL.*, , chase +medicines, .*SAFEWAY.* +medicines, GOOD DAY PHARMACY.* +medicines, UpSpring .* +dues-subscriptions-memberships, CHECK # 4047, , firstbank, , AARP +dues-subscriptions-memberships, CONSUMERREPORTS.*, , chase +dues-subscriptions-memberships, prime membership fee, i, amz.sandy.dig.ord, 10 +mortgage, CASH PENNYMAC.*, , firstbank +office.supplies, .*Ballpoint Pens.*, , amz.sandy.retail.ord +office.supplies, .*Magnifying Glass.* +office.supplies, .*NOKBOX.*, , chase +office.supplies, .*Spiral Notebook.* +office.supplies, HP Printer Paper .* +office.supplies, HP.*Ink Cartridge.* +office.supplies, Pentel .* +passports, CHECK # 4050, , firstbank, , larimer county clerk +passports, PAYMENT PASSPORTSERVICES, , firstbank, , us dept of state +passports, PAYMENT PASSPORTSERVICES, , firstbank, , us dept of state +payment, .* CHASE CREDIT CRD, , firstbank +payment, .*AMAZON PRIME.*, , chase +payment, .*ANNUAL MEMBERSHIP FEE.*, , chase +payment, .*Payment Thank You.* +payment, AMAZON MKTPL.*, , chase +payment, Amazon Prime.*, , chase +payment, Amazon\.com.* +payment, AMZN Mktp.* +payment, CASHBACK BONUS REDEMPTION .*, , discover +payment, INST XFER PAYPAL, , firstbank +payment, Kindle Svcs.*, , chase, 10 +payment, PAYMENT CITI.*, , firstbank +payment, PAYPAL \*.*, , chase, 10, , 2025 +payment, PHONE PAY DISCOVER, , firstbank +payment, PHONE PAYMENT .*, , discover +payment, PP\*.*, , chase, 10 +payment, Prime Video Channels .*, , chase, 10 +payment, WHOLEFDS.* +personal.care, .* Scalp Serum .* +personal.care, .* Shaving Soap .* +personal.care, .*dental care.*, i, +personal.care, .*Dental Floss.*, , amz.sandy.retail.ord +personal.care, .*Eye Cream.*, , amz.sandy.retail.ord +personal.care, .*Eye Wrinkle Cream.*, , amz.sandy.retail.ord +personal.care, .*HENSONSHAVI.* +personal.care, .*Nail File.*, i, +personal.care, .*shave soap.*, i, +personal.care, .*Skin Scraping Tool.*, , amz.sandy.retail.ord +personal.care, .*VIBRANTBLUE .* +personal.care, AYR Saline.* +personal.care, Biossance.*, , amz.sandy.retail.ord +personal.care, Elysian Honey.* +personal.care, Kiss My Face.* +personal.care, Life-flo.* +personal.care, MagniLife.* +personal.care, micropore tape .*, i +personal.care, MyChelle Dermaceuticals .* +personal.care, Nioxin .* +personal.care, OraWellness .* +personal.care, PAYPAL \*BIOSSANCE .*, , chase +personal.care, PROFOUND HEALTH LTD .*, , , , eye drops +personal.care, Sondery .* +personal.care, The Naked Bee .* +personal.care, THG Beauty Limited .*, , paypal +personal.care, Zeasorb AF Jock Itch Powder.* +personal.electronics, .* apple watch .*, i +personal.care, .* toothbrush .*, i +personal.electronics, LiCB .* Watch Battery.* +personal.electronics, Moto G .* +personal.electronics, PAYPAL .WALMART COM .*, , , , hearing aid batteries +personal.electronics, Poetic Revolution .* +personal.electronics, RingConn .* +dues-subscriptions-memberships, .*BUFFMUFF.* +dues-subscriptions-memberships, CHECK # 4070, , firstbank, , green america +dues-subscriptions-memberships, GANNETT MEDIA .*, , chase +dues-subscriptions-memberships, Google .* -1\.08 USD.*, , paypal, 10 +dues-subscriptions-memberships, Google .* -3\.26 USD.*, , paypal, 10 +dues-subscriptions-memberships, INSTAPAPER .*, , chase +dues-subscriptions-memberships, PAYPAL \*CONSUMERLAB .*, , chase +supplements, .* Boron Complex .* +supplements, .*Ancestral Supplements.*, , amz.sandy.retail.ord +supplements, .*ANTIAGINGCO.* +supplements, .*BioGaia Gastrus.*, , amz.sandy.retail.ord +supplements, .*Glutamine.* +supplements, .*Gummies.* +supplements, .*LIFEEXTENSI.* +supplements, .*Melatonin.*, , amz.sandy.retail.ord +supplements, .*PUREFORMULA .* +supplements, .*SEEKHEALTH.* +supplements, .*SOMALIFE.* +supplements, .*Veggie Capsules.*, , amz.sandy.retail.ord +supplements, Biosil.*, , amz.sandy.retail.ord +supplements, Biotics.*, , amz.sandy.retail.ord +supplements, Boiron.*, , amz.sandy.retail.ord +supplements, Cardiovascular Research.* +supplements, CHECK # 4040, , firstbank, , shelby kahl +supplements, Designs for Health.* +supplements, Doctor's Best.* +supplements, Dr\. Berg.* +supplements, Dr\. Clark.* +supplements, Dr\. Mercola.* +supplements, DrFormulas.* +supplements, Enzymedica.* +supplements, FULLSCRIPT.*, , chase +supplements, Garden of Life.* +supplements, Health Thru Nutrition.* +supplements, HealthForce.* +supplements, Horbäach.* +supplements, Hyland's.* +supplements, iHerb.*, , paypal, 10 +supplements, Jarrow.* +supplements, keto chow .*, i +supplements, Life Extension.* +supplements, Living Silica Collagen Booster Liquid.* +supplements, Magnesium L Threonate Capsules.* +supplements, MARCO PHARMA.* +supplements, MICROVASCULAR.*, , chase, , endocaylx +supplements, MIRACLES OF HEALTH .*, , chase +supplements, multivitamin .*, i +supplements, Nattokinase .* +supplements, Natural Factors.* +supplements, Nature's Way .* +supplements, NEORA.* +supplements, New Chapter .* +supplements, Nordic Naturals .* +supplements, NOW .* +supplements, Nutricost.* +supplements, PAYPAL \*LIFE ENTH .*, , chase +supplements, PAYPAL \*SWANSONHEAL .*, , chase, , +supplements, PAYPAL \*VITACOSTCOM .*, , chase +supplements, Premier Research Labs .* +supplements, Pure Encapsulations .* +supplements, PureFormulas .*, , paypal, 10 +supplements, Renew Life .* +supplements, RnA ReSet .* +supplements, SeabuckWonders .*, i +supplements, SeekingHealth .*, , paypal, 10 +supplements, SOLARAY .*, i +supplements, Solgar .* +supplements, Source Naturals .* +supplements, Standard Process .* +supplements, SUBSCRIBE .* SAVE .*, , chase, , paleo valley +supplements, Swanson Health Products.*, , paypal +supplements, Teliaoils .* +supplements, THORNE .* +supplements, Trace Minerals Mega-Mag.* +supplements, Vital Earth .* +supplements, Zazzee .* +taxes.refunds, .* DEPT REVENUE, , firstbank +taxes.refunds, TAX REF IRS TREAS.*, , firstbank +taxes, TAXES PEAKVIEW.*, , firstbank +taxes, USATAXPYMT IRS.*, , firstbank +tips, .*Amazon Tips.*, , chase +unknown, .*OCHEALTH .* +supplements, CHECK # 4034, , firstbank, , shelby kahl +unknown, CHECK # 4084, , firstbank, , +unknown, CHECK # 4088, , firstbank, , +unknown, Colorado Interactive.*, , paypal +unknown, Google .* -31\.15.*, , paypal +unknown, MICROSOFT.*, , discover +unknown, PAYPAL \*COLORADOINT .*, , chase +unknown, Spigen .* +utilities.garbage, .* REPUBLICSERVICES, , firstbank +utilities.electricity/water, CITY OF FORT COLLINS .*, , discover +utilities.electricity/water, VISA CITY OF FORT COLLINS.*, , firstbank +utilities.gas, XCELENERGY.*, , firstbank +vision.eyewear, DOOViC.* +vision.eyewear, ThinOptics .* +vision.services, .*EYECARE ASSOCIATES.*, , chase +vision.services, EYECARE ASSOCIATES .*, , discover +home.yard-garden.equipment, .* Drip Irrigation .* +home.yard-garden.equipment, .* Replacement Spool for Black and Decker String Trimmer .* +home.yard-garden.equipment, .*APEX GARDEN Replacement Canopy Top.*, , amz.sandy.retail.ord +home.yard-garden.equipment, .*ETSY INC.* +home.yard-garden.equipment, .*Water Hose Nozzle.* +home.yard-garden.equipment, Bug Zapper.*, , amz.sandy.retail.ord +home.yard-garden.equipment, Drip Irrigation .* +home.yard-garden.equipment, ego power.*, i +home.yard-garden.equipment, Flame King .* +home.yard-garden.equipment, Yzert .* +home.yard-garden.services, .*ROYAL TURF.* +home.yard-garden.services, CHECK # 4052, , firstbank, , chris coopenhaver +home.yard-garden.services, CHECK # 4080, , firstbank, , chris coopenhaver +medical.services, CHECK # 4087, , firstbank, , hospital? diff --git a/deploy b/deploy new file mode 100755 index 0000000..a83f1df --- /dev/null +++ b/deploy @@ -0,0 +1,7 @@ +#!/bin/sh +echo "deploying budget" +PROJECT="$HOME/projects/com.stephenschafer.budget/target" +WAR="com.stephenschafer.budget-0.0.1-SNAPSHOT.war" +cp $PROJECT/$WAR . +mv ./$WAR ./budget.war +mv ./budget.war /disk1/tomcat/webapps diff --git a/pom-jar.xml b/pom-jar.xml new file mode 100644 index 0000000..1597c17 --- /dev/null +++ b/pom-jar.xml @@ -0,0 +1,121 @@ + + 4.0.0 + com.stephenschafer + budget + 0.0.1-SNAPSHOT + jar + + src/main/java + + + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + prepare-package + + copy-dependencies + + + + ${project.build.directory}/libs + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + libs/ + + com.stephenschafer.budget.Amazon + + + + + + + + + + javax.servlet + javax.servlet-api + 3.0.1 + provided + + + mysql + mysql-connector-java + 8.0.11 + + + commons-fileupload + commons-fileupload + 1.3.3 + + + org.apache.commons + commons-csv + 1.5 + + + com.fasterxml.jackson.core + jackson-core + 2.9.5 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.5 + + + jstl + jstl + 1.2 + compile + + + taglibs + standard + 1.1.2 + compile + + + org.seleniumhq.selenium + selenium-java + 4.1.2 + + + org.junit.jupiter + junit-jupiter-engine + 5.8.2 + test + + + io.github.bonigarcia + webdrivermanager + 5.1.0 + + + com.opencsv + opencsv + 5.8 + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..8425898 --- /dev/null +++ b/pom.xml @@ -0,0 +1,114 @@ + + 4.0.0 + com.stephenschafer + com.stephenschafer.budget + 0.0.1-SNAPSHOT + war + + 8 + 8 + + + src/main/java + + + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + maven-war-plugin + 3.3.2 + + WebContent + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + **/*Example.java + + + + + + + + javax.servlet + javax.servlet-api + 3.0.1 + provided + + + mysql + mysql-connector-java + 8.0.11 + + + commons-fileupload + commons-fileupload + 1.3.3 + + + org.apache.commons + commons-csv + 1.5 + + + com.fasterxml.jackson.core + jackson-core + 2.9.5 + + + com.fasterxml.jackson.core + jackson-databind + 2.9.5 + + + jstl + jstl + 1.2 + compile + + + taglibs + standard + 1.1.2 + compile + + + org.seleniumhq.selenium + selenium-java + 4.1.2 + + + org.junit.jupiter + junit-jupiter-engine + 5.8.2 + test + + + io.github.bonigarcia + webdrivermanager + 5.1.0 + + + com.opencsv + opencsv + 5.8 + + + org.apache.httpcomponents.client5 + httpclient5 + 5.4.1 + + + \ No newline at end of file diff --git a/rebuild b/rebuild new file mode 100755 index 0000000..577ed23 --- /dev/null +++ b/rebuild @@ -0,0 +1,5 @@ +#!/bin/sh +cd $HOME/projects/com.stephenschafer.budget +if mvn clean package; then + ./deploy +fi diff --git a/run b/run new file mode 100755 index 0000000..90d7483 --- /dev/null +++ b/run @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +java -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@" diff --git a/run-categorize b/run-categorize new file mode 100755 index 0000000..1cb8ef8 --- /dev/null +++ b/run-categorize @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run categorize budget.properties + diff --git a/run-debug b/run-debug new file mode 100755 index 0000000..5763704 --- /dev/null +++ b/run-debug @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +JVM_ARGS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8004" +java $JVM_ARGS -cp target/*.jar com.stephenschafer.budget.schema.Schema "$@" diff --git a/run-debug-categorize b/run-debug-categorize new file mode 100755 index 0000000..7dd15cf --- /dev/null +++ b/run-debug-categorize @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run-debug categorize budget.properties + diff --git a/run-debug-load b/run-debug-load new file mode 100755 index 0000000..28e1336 --- /dev/null +++ b/run-debug-load @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run-debug load-csv budget.properties trilium Budget + diff --git a/run-load-csv b/run-load-csv new file mode 100755 index 0000000..5da0f8f --- /dev/null +++ b/run-load-csv @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run load-csv budget.properties trilium Budget + diff --git a/run-load-regex b/run-load-regex new file mode 100755 index 0000000..e2254f5 --- /dev/null +++ b/run-load-regex @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run load-regex budget.properties + diff --git a/run-process b/run-process new file mode 100755 index 0000000..cfd7841 --- /dev/null +++ b/run-process @@ -0,0 +1,4 @@ +#!/bin/bash +cd "$(dirname "${BASH_SOURCE[0]}")" +./run process budget.properties + diff --git a/send b/send new file mode 100755 index 0000000..8c4be65 --- /dev/null +++ b/send @@ -0,0 +1,6 @@ +#!/bin/sh +echo "sending budget to stephenschafer.com" +PROJECT="$HOME/projects/com.stephenschafer.budget/target" +WAR="com.stephenschafer.budget-0.0.1-SNAPSHOT.war" +scp $PROJECT/$WAR steve@stephenschafer.com:~ +ssh steve@stephenschafer.com "bin/deploy-budget" diff --git a/show-tables.sql b/show-tables.sql new file mode 100644 index 0000000..658a418 --- /dev/null +++ b/show-tables.sql @@ -0,0 +1,2 @@ +select table_schema, table_name +from information_schema.tables where table_schema = 'budget_2024' and table_name like '%sandy%retail%return%'; \ No newline at end of file diff --git a/src/main/java/com/stephenschafer/budget/schema/CatRegex.java b/src/main/java/com/stephenschafer/budget/schema/CatRegex.java new file mode 100644 index 0000000..d97add8 --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/CatRegex.java @@ -0,0 +1,150 @@ +package com.stephenschafer.budget.schema; + +import java.util.regex.Pattern; + +public class CatRegex { + private final Pattern pattern; + private final String category; + private final String source; + private final int priority; + private final String extraDescription; + private final Integer year; + private int id; + + public CatRegex(final int id, final Pattern pattern, final String category, final String source, + final int priority, final String extraDescription, final Integer year) { + this.id = id; + this.pattern = pattern; + this.category = category; + this.source = source; + this.priority = priority; + this.extraDescription = extraDescription; + this.year = year; + } + + public CatRegex(final String line) { + final String[] parts = line.split(","); + final String category = parts[0].trim(); + final String regex = parts[1].trim(); + String flagsString = ""; + String source = ""; + int priority = 0; + String extraDescription = ""; + Integer year = null; + if (parts.length > 2) { + flagsString = parts[2].trim(); + if (parts.length > 3) { + source = parts[3].trim(); + if (parts.length > 4) { + final String priorityString = parts[4].trim(); + if (priorityString.length() > 0) { + try { + priority = Integer.parseInt(priorityString); + } + catch (final NumberFormatException e) { + System.out.println( + "Invalid priority in reegex line: " + line + "
"); + } + } + if (parts.length > 5) { + extraDescription = parts[5].trim(); + if (parts.length > 6) { + final String yearString = parts[6].trim(); + if (yearString.length() > 0) { + try { + year = Integer.valueOf(yearString); + } + catch (final NumberFormatException e) { + System.out.println( + "Invalid year in reegex line: " + line + "
"); + } + } + if (parts.length > 7) { + System.out.println("Too many parts in " + line); + } + } + } + } + } + } + int flags = 0; + if (flagsString.indexOf("i") >= 0) { + flags |= Pattern.CASE_INSENSITIVE; + } + if (flagsString.indexOf("m") >= 0) { + flags |= Pattern.MULTILINE; + } + pattern = Pattern.compile(regex, flags); + this.category = category; + this.source = source.length() == 0 ? null : source; + this.priority = priority; + this.extraDescription = extraDescription; + this.year = year; + } + + public String getSource() { + return source; + } + + public Pattern getPattern() { + return pattern; + } + + public String getCategory() { + return category; + } + + public int getPriority() { + return priority; + } + + @Override + public int hashCode() { + int hashCode = 0; + if (source != null) { + hashCode += source.hashCode(); + } + return hashCode + pattern.hashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (obj instanceof CatRegex) { + final CatRegex that = (CatRegex) obj; + if (!source.equals(that.source) || !pattern.equals(that.pattern)) { + return false; + } + } + return true; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(category); + sb.append(": "); + sb.append(pattern.toString()); + if (source != null && source.trim().length() > 0) { + sb.append(" (" + source + ")"); + } + sb.append(", "); + sb.append(priority); + return sb.toString(); + } + + public String getExtraDescription() { + return extraDescription; + } + + public int getId() { + return id; + } + + public void setId(final int id) { + this.id = id; + } + + public Integer getYear() { + return year; + } +} \ No newline at end of file diff --git a/src/main/java/com/stephenschafer/budget/schema/Category.java b/src/main/java/com/stephenschafer/budget/schema/Category.java new file mode 100644 index 0000000..b85d3da --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/Category.java @@ -0,0 +1,37 @@ +package com.stephenschafer.budget.schema; + +import java.util.Map; + +class Category { + final int id; + final Integer parentId; + final String name; + + public Category(final int id, final Integer parentId, final String name) { + this.id = id; + this.parentId = parentId; + this.name = name; + } + + String getFullName(final Map categories) { + final StringBuilder sb = new StringBuilder(); + if (parentId != null) { + final Category parentCategory = categories.get(parentId); + sb.append(parentCategory.getFullName(categories)); + } + sb.append(name); + return sb.toString(); + } + + public int getId() { + return id; + } + + public Integer getParentId() { + return parentId; + } + + public String getName() { + return name; + } +} \ No newline at end of file diff --git a/src/main/java/com/stephenschafer/budget/schema/Config.java b/src/main/java/com/stephenschafer/budget/schema/Config.java new file mode 100644 index 0000000..a97deb8 --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/Config.java @@ -0,0 +1,75 @@ +package com.stephenschafer.budget.schema; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class Config { + private final Properties systemProperties; + private final Properties properties; + private String triliumUrl; + private String triliumPassword; + private String logFilename; + private String dbUrl; + private String dbUsername; + private String dbPassword; + + public Config() { + systemProperties = System.getProperties(); + properties = new Properties(); + } + + public void load(final String fileName) throws FileNotFoundException, IOException { + String propertiesFileName = fileName; + if (propertiesFileName == null) { + propertiesFileName = systemProperties.getProperty("properties-filename"); + } + if (propertiesFileName != null) { + final File propertiesFile = new File(propertiesFileName); + try (InputStream inStream = new FileInputStream(propertiesFile)) { + properties.load(inStream); + } + } + triliumUrl = getProperty("trilium-url"); + triliumPassword = getProperty("trilium-password"); + logFilename = getProperty("log-filename"); + dbUrl = getProperty("db-url"); + dbUsername = getProperty("db-username"); + dbPassword = getProperty("db-password"); + } + + private String getProperty(final String name) { + String triliumUrl = systemProperties.getProperty(name); + if (triliumUrl == null) { + triliumUrl = properties.getProperty(name); + } + return triliumUrl; + } + + public String getTriliumUrl() { + return triliumUrl; + } + + public String getTriliumPassword() { + return triliumPassword; + } + + public String getLogFilename() { + return logFilename; + } + + public String getDbUrl() { + return dbUrl; + } + + public String getDbUsername() { + return dbUsername; + } + + public String getDbPassword() { + return dbPassword; + } +} diff --git a/src/main/java/com/stephenschafer/budget/schema/DbConnectionPool.java b/src/main/java/com/stephenschafer/budget/schema/DbConnectionPool.java new file mode 100644 index 0000000..c9aa8ee --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/DbConnectionPool.java @@ -0,0 +1,690 @@ +package com.stephenschafer.budget.schema; + +import java.io.Serializable; +import java.net.SocketException; +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.NClob; +import java.sql.PreparedStatement; +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Savepoint; +import java.sql.Statement; +import java.sql.Struct; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Queue; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class DbConnectionPool implements Serializable { + private static final long serialVersionUID = 1L; + private static final Logger LOGGER; + private final String uRL; + private final String username; + private final String password; + private final String driver; + private final Queue connections; + private long openConnectionIndex; + private final Map openConnections; + private long timeout; + static { + LOGGER = Logger.getLogger(DbConnectionPool.class.getName()); + } + + public DbConnectionPool(final String dbDriver, final String dbName, final String dbUsername, + final String dbPassword) throws ClassNotFoundException, SQLException { + connections = new LinkedList<>(); + openConnectionIndex = 0L; + openConnections = new HashMap<>(); + timeout = 900000L; + driver = dbDriver; + uRL = dbName; + username = dbUsername; + password = dbPassword; + validateConnection(); + } + + public final void validateConnection() throws ClassNotFoundException, SQLException { + DbConnectionPool.LOGGER.log(Level.FINEST, "Testing connection pool"); + DbConnectionPool.LOGGER.log(Level.FINEST, "Instantiating " + driver + "\n"); + Class.forName(driver); + DbConnectionPool.LOGGER.log(Level.FINEST, "Connecting"); + final Connection connection = DriverManager.getConnection(uRL, username, password); + try { + final DatabaseMetaData dbmd = connection.getMetaData(); + DbConnectionPool.LOGGER.log(Level.FINEST, + "Connection to " + dbmd.getDatabaseProductName() + " " + + dbmd.getDatabaseProductVersion() + " successful."); + } + finally { + connection.close(); + } + connection.close(); + } + + public final void clear() throws SQLException { + synchronized (this) { + while (!connections.isEmpty()) { + final PooledConnection connection = connections.remove(); + connection.reallyClose(); + } + } + } + + public final PooledConnection getConnection(final int transactionIsolation, + final boolean readOnly, final boolean autoCommit) throws SQLException { + while (true) { + PooledConnection oldConnection; + final long timeout; + synchronized (this) { + if (connections.isEmpty()) { + oldConnection = null; + } + else { + oldConnection = connections.remove(); + } + timeout = this.timeout; + } + if (oldConnection == null) { + DbConnectionPool.LOGGER.log(Level.FINEST, "Connecting"); + SQLException exception = null; + int retryCount = 0; + while (retryCount < 10) { + Connection newConnection; + try { + newConnection = DriverManager.getConnection(uRL, username, password); + } + catch (final SQLException e) { + if (!(e.getCause() instanceof SocketException)) { + throw e; + } + exception = e; + DbConnectionPool.LOGGER.log(Level.FINEST, "Retrying", e); + try { + Thread.sleep(1000L); + } + catch (final InterruptedException ex) { + } + ++retryCount; + newConnection = null; + } + if (newConnection != null) { + newConnection.setAutoCommit(true); + newConnection.setTransactionIsolation(transactionIsolation); + newConnection.setReadOnly(readOnly); + newConnection.setAutoCommit(autoCommit); + final long index = incrementOpenConnectionCount(); + return new PooledConnection(newConnection, index); + } + } + if (exception != null) { + throw exception; + } + } + if (oldConnection != null) { + if (oldConnection.isClosed()) { + DbConnectionPool.LOGGER.log(Level.WARNING, + "Pooled connection was already closed"); + } + else { + Label_0430: { + Label_0418: { + if ((timeout != 0L) && (System.currentTimeMillis() + - oldConnection.getLastAccess() >= timeout)) { + break Label_0418; + } + try { + oldConnection.setAutoCommit(true); + oldConnection.setTransactionIsolation(transactionIsolation); + oldConnection.setReadOnly(readOnly); + oldConnection.setAutoCommit(autoCommit); + synchronized (this) { + final Long key = Long.valueOf(oldConnection.getIndex()); + final OpenConnectionInfo info = openConnections.get(key); + if (info != null) { + DbConnectionPool.LOGGER.log(Level.WARNING, + "Overwriting open connection info: " + key + " " + + info); + } + openConnections.put(key, new OpenConnectionInfo()); + } + return oldConnection; + } + catch (final Exception e2) { + DbConnectionPool.LOGGER.log(Level.SEVERE, + "Unable to reuse DB connection", e2); + break Label_0430; + } + } + DbConnectionPool.LOGGER.log(Level.FINEST, "DB connection timed out"); + try { + oldConnection.reallyClose(); + } + catch (final Exception e2) { + DbConnectionPool.LOGGER.log(Level.SEVERE, + "Unable to really close DB connection", e2); + } + } + } + } + } + } + + private long incrementOpenConnectionCount() { + synchronized (this) { + final long index = openConnectionIndex++; + final Long key = Long.valueOf(index); + final OpenConnectionInfo info = openConnections.get(key); + if (info != null) { + DbConnectionPool.LOGGER.log(Level.WARNING, + "Overwriting open connection info: " + key + " " + info); + } + openConnections.put(key, new OpenConnectionInfo()); + return index; + } + } + + protected final void add(final PooledConnection connection) { + synchronized (this) { + final OpenConnectionInfo info = openConnections.remove( + Long.valueOf(connection.getIndex())); + if (info == null) { + DbConnectionPool.LOGGER.log(Level.WARNING, + "adding orphaned connection: " + connection.getIndex()); + } + else { + connections.offer(connection); + } + } + } + + public final long getOpenConnectionIndex() { + synchronized (this) { + return openConnectionIndex; + } + } + + public final OpenConnectionInfo getOpenConnectionInfo(final long index) { + synchronized (this) { + return openConnections.get(Long.valueOf(index)); + } + } + + public final int getOpenConnectionCount() { + synchronized (this) { + return openConnections.size(); + } + } + + public final int getPooledConnectionCount() { + synchronized (this) { + return connections.size(); + } + } + + @Override + public final String toString() { + final StringBuilder buf = new StringBuilder(); + buf.append("DB "); + buf.append(uRL); + buf.append(" "); + synchronized (this) { + buf.append("open: "); + buf.append(openConnections.size()); + buf.append(", pooled: "); + buf.append(connections.size()); + buf.append(", next: "); + buf.append(openConnectionIndex); + } + return buf.toString(); + } + + public final long getTimeout() { + synchronized (this) { + return timeout; + } + } + + public final void setTimeout(final long timeout) { + synchronized (this) { + this.timeout = timeout; + } + } + + public final String getDriver() { + return driver; + } + + public final String getURL() { + return uRL; + } + + public final String getPassword() { + return password; + } + + public final String getUsername() { + return username; + } + + public synchronized Map getOpenConnections() { + final Map map = new HashMap<>(); + for (final Long index : openConnections.keySet()) { + final OpenConnectionInfo info = openConnections.get(index); + map.put(index, Long.valueOf(info.timestamp)); + } + return map; + } + + public static final class OpenConnectionInfo { + public final long timestamp; + private final List stackTrace; + + public OpenConnectionInfo() { + timestamp = System.currentTimeMillis(); + final StackTraceElement[] steArray = Thread.currentThread().getStackTrace(); + final List stackTrace = new ArrayList<>( + steArray.length); + StackTraceElement[] array; + for (int length = (array = steArray).length, i = 0; i < length; ++i) { + final StackTraceElement element = array[i]; + stackTrace.add(element); + } + this.stackTrace = Collections.unmodifiableList(stackTrace); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append("Timestamp: "); + sb.append(new Date(timestamp)); + sb.append("\n"); + for (final StackTraceElement element : stackTrace) { + sb.append(element); + sb.append("\n"); + } + return sb.toString(); + } + } + + public class PooledConnection implements Connection { + private final Connection connection; + private long lastAccess; + private boolean autoCommit; + private boolean readOnly; + private final long index; + + public PooledConnection(final Connection connection, final long index) { + lastAccess = System.currentTimeMillis(); + autoCommit = false; + readOnly = false; + this.index = index; + this.connection = connection; + try { + autoCommit = connection.getAutoCommit(); + } + catch (final SQLException e) { + DbConnectionPool.LOGGER.log(Level.WARNING, "Unable to get auto commit", e); + } + try { + readOnly = connection.isReadOnly(); + } + catch (final SQLException e) { + DbConnectionPool.LOGGER.log(Level.WARNING, "Unable to get read only", e); + } + } + + public void clearPool() throws SQLException { + clear(); + } + + @Override + public String nativeSQL(final String sql) throws SQLException { + return connection.nativeSQL(sql); + } + + @Override + public int hashCode() { + return connection.hashCode(); + } + + @Override + public Map> getTypeMap() throws SQLException { + return connection.getTypeMap(); + } + + @Override + public PreparedStatement prepareStatement(final String sql) throws SQLException { + final PreparedStatement stmt = connection.prepareStatement(sql); + if (connection.getAutoCommit() != autoCommit || connection.isReadOnly() != readOnly) { + final StringBuilder sb = new StringBuilder(); + String sep = ""; + if (connection.getAutoCommit() != autoCommit) { + sb.append(sep); + sep = " and "; + sb.append("autoCommit has changed from " + autoCommit); + } + if (connection.isReadOnly() != readOnly) { + sb.append(sep); + sep = " and "; + sb.append("readOnly has changed from " + readOnly); + } + sb.append(" in "); + sb.append(stmt); + DbConnectionPool.LOGGER.log(Level.WARNING, sb.toString()); + } + return stmt; + } + + @Override + public void setTransactionIsolation(final int level) throws SQLException { + connection.setTransactionIsolation(level); + } + + @Override + public String getCatalog() throws SQLException { + return connection.getCatalog(); + } + + @Override + public int getTransactionIsolation() throws SQLException { + return connection.getTransactionIsolation(); + } + + @Override + public void releaseSavepoint(final Savepoint savepoint) throws SQLException { + connection.releaseSavepoint(savepoint); + } + + @Override + public int getHoldability() throws SQLException { + return connection.getHoldability(); + } + + @Override + public CallableStatement prepareCall(final String sql, final int resultSetType, + final int resultSetConcurrency, final int resultSetHoldability) + throws SQLException { + return connection.prepareCall(sql, resultSetType, resultSetConcurrency, + resultSetHoldability); + } + + @Override + public boolean getAutoCommit() throws SQLException { + return connection.getAutoCommit(); + } + + @Override + public Statement createStatement() throws SQLException { + return connection.createStatement(); + } + + @Override + public CallableStatement prepareCall(final String sql) throws SQLException { + return connection.prepareCall(sql); + } + + @Override + public void setAutoCommit(final boolean autoCommit) throws SQLException { + this.autoCommit = autoCommit; + connection.setAutoCommit(autoCommit); + } + + @Override + public PreparedStatement prepareStatement(final String sql, final int autoGeneratedKeys) + throws SQLException { + return connection.prepareStatement(sql, autoGeneratedKeys); + } + + @Override + public void setReadOnly(final boolean readOnly) throws SQLException { + this.readOnly = readOnly; + connection.setReadOnly(readOnly); + } + + @Override + public CallableStatement prepareCall(final String sql, final int resultSetType, + final int resultSetConcurrency) throws SQLException { + return connection.prepareCall(sql, resultSetType, resultSetConcurrency); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return connection.getWarnings(); + } + + @Override + public PreparedStatement prepareStatement(final String sql, final int resultSetType, + final int resultSetConcurrency) throws SQLException { + return connection.prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + @Override + public boolean equals(final Object obj) { + return connection.equals(obj); + } + + @Override + public PreparedStatement prepareStatement(final String sql, final int[] columnIndexes) + throws SQLException { + return connection.prepareStatement(sql, columnIndexes); + } + + @Override + public boolean isClosed() throws SQLException { + return connection.isClosed(); + } + + @Override + public PreparedStatement prepareStatement(final String sql, final int resultSetType, + final int resultSetConcurrency, final int resultSetHoldability) + throws SQLException { + return connection.prepareStatement(sql, resultSetType, resultSetConcurrency, + resultSetHoldability); + } + + @Override + public void commit() throws SQLException { + connection.commit(); + } + + @Override + public void clearWarnings() throws SQLException { + connection.clearWarnings(); + } + + @Override + public void setCatalog(final String catalog) throws SQLException { + connection.setCatalog(catalog); + } + + @Override + public void close() { + add(this); + lastAccess = System.currentTimeMillis(); + } + + public void reallyClose() throws SQLException { + connection.close(); + } + + @Override + public String toString() { + return connection.toString(); + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return connection.getMetaData(); + } + + @Override + public void rollback() throws SQLException { + connection.rollback(); + } + + @Override + public Savepoint setSavepoint(final String name) throws SQLException { + return connection.setSavepoint(name); + } + + @Override + public boolean isReadOnly() throws SQLException { + return connection.isReadOnly(); + } + + @Override + public Statement createStatement(final int resultSetType, final int resultSetConcurrency) + throws SQLException { + return connection.createStatement(resultSetType, resultSetConcurrency); + } + + @Override + public void rollback(final Savepoint savepoint) throws SQLException { + connection.rollback(savepoint); + } + + @Override + public PreparedStatement prepareStatement(final String sql, final String[] columnNames) + throws SQLException { + return connection.prepareStatement(sql, columnNames); + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return connection.setSavepoint(); + } + + @Override + public Statement createStatement(final int resultSetType, final int resultSetConcurrency, + final int resultSetHoldability) throws SQLException { + return connection.createStatement(resultSetType, resultSetConcurrency, + resultSetHoldability); + } + + @Override + public void setTypeMap(final Map> map) throws SQLException { + connection.setTypeMap(map); + } + + @Override + public void setHoldability(final int holdability) throws SQLException { + connection.setHoldability(holdability); + } + + public long getLastAccess() { + return lastAccess; + } + + @Override + public Array createArrayOf(final String arg0, final Object[] arg1) throws SQLException { + return connection.createArrayOf(arg0, arg1); + } + + @Override + public Blob createBlob() throws SQLException { + return connection.createBlob(); + } + + @Override + public Clob createClob() throws SQLException { + return connection.createClob(); + } + + @Override + public NClob createNClob() throws SQLException { + return connection.createNClob(); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + return connection.createSQLXML(); + } + + @Override + public Struct createStruct(final String arg0, final Object[] arg1) throws SQLException { + return connection.createStruct(arg0, arg1); + } + + @Override + public Properties getClientInfo() throws SQLException { + return connection.getClientInfo(); + } + + @Override + public String getClientInfo(final String arg0) throws SQLException { + return connection.getClientInfo(arg0); + } + + @Override + public boolean isValid(final int arg0) throws SQLException { + return connection.isValid(arg0); + } + + @Override + public void setClientInfo(final Properties arg0) throws SQLClientInfoException { + connection.setClientInfo(arg0); + } + + @Override + public void setClientInfo(final String arg0, final String arg1) + throws SQLClientInfoException { + connection.setClientInfo(arg0, arg1); + } + + @Override + public boolean isWrapperFor(final Class arg0) throws SQLException { + return connection.isWrapperFor(arg0); + } + + @Override + public T unwrap(final Class arg0) throws SQLException { + return connection.unwrap(arg0); + } + + public long getIndex() { + return index; + } + + @Override + public void setSchema(final String schema) throws SQLException { + connection.setSchema(schema); + } + + @Override + public String getSchema() throws SQLException { + return connection.getSchema(); + } + + @Override + public void abort(final Executor executor) throws SQLException { + connection.abort(executor); + } + + @Override + public void setNetworkTimeout(final Executor executor, final int milliseconds) + throws SQLException { + connection.setNetworkTimeout(executor, milliseconds); + } + + @Override + public int getNetworkTimeout() throws SQLException { + return connection.getNetworkTimeout(); + } + } +} diff --git a/src/main/java/com/stephenschafer/budget/schema/Detail.java b/src/main/java/com/stephenschafer/budget/schema/Detail.java new file mode 100644 index 0000000..40db0e6 --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/Detail.java @@ -0,0 +1,12 @@ +package com.stephenschafer.budget.schema; + +import java.math.BigDecimal; +import java.util.Date; + +class Detail { + int transactionId; + String source; + String description; + Date date; + BigDecimal amount; +} \ No newline at end of file diff --git a/src/main/java/com/stephenschafer/budget/schema/Schema.java b/src/main/java/com/stephenschafer/budget/schema/Schema.java new file mode 100644 index 0000000..29899ea --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/Schema.java @@ -0,0 +1,1623 @@ +package com.stephenschafer.budget.schema; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.MathContext; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVReader; +import com.opencsv.exceptions.CsvException; + +public class Schema { + private final Config config; + private String triliumAuthToken; + private final DbConnectionPool pool; + private static final Pattern INT_PATTERN = Pattern.compile("^-{0,1}([0-9]+)$"); + private static final Pattern DECIMAL_PATTERN = Pattern.compile( + "^(-|\\+){0,1}([0-9]+)\\.([0-9]+)"); + private static final Pattern DATE_PATTERN_YYYYMMDD = Pattern.compile( + "^([0-9]{4})-([0-9]{2})-([0-9]{2})$"); + private static final Pattern DATE_PATTERN_MMDDYYYY = Pattern.compile( + "^([0-9]{1,2})/([0-9]{1,2})/([0-9]{4})$"); + private static final Pattern DATE_PATTERN_MMDDYY = Pattern.compile( + "^([0-9]{1,2})/([0-9]{1,2})/([0-9]{2})$"); + // 2023-12-19T18:48:46.251Z + private static final Pattern DATETIME_PATTERN = Pattern.compile( + "^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$"); + private static final Pattern DATETIME_PATTERN_SECS = Pattern.compile( + "^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}\\.[0-9]{3})Z$"); + + public Schema(final String propertiesFileName) + throws FileNotFoundException, IOException, ClassNotFoundException, SQLException { + config = new Config(); + config.load(propertiesFileName); + pool = new DbConnectionPool("com.mysql.cj.jdbc.Driver", config.getDbUrl(), + config.getDbUsername(), config.getDbPassword()); + } + + public static void main(final String[] args) throws Exception { + final ArgsHelper argsHelper = new ArgsHelper(args); + final String phaseName = argsHelper.get("phase name"); + if (phaseName == null) { + System.out.println("Please supply a phase name"); + System.exit(1); + return; + } + final Phase phase = PHASES.get(phaseName); + if (phase == null) { + System.out.println("Invalid phase name"); + System.exit(1); + return; + } + phase.execute(argsHelper); + } + + static class ArgsHelper { + final String[] args; + int i = 0; + + ArgsHelper(final String[] args) { + this.args = args; + } + + String get(final String name) { + if (this.args.length <= i) { + System.out.println("Please supply argument " + (i + 1) + " = " + name); + System.exit(1); + return null; + } + return this.args[i++]; + } + } + + interface Phase { + void execute(ArgsHelper argsHelper) throws Exception; + } + + static Map PHASES = new HashMap<>(); + static { + PHASES.put("load-regex", argsHelper -> { + final Schema schema = new Schema(argsHelper.get("properties file")); + schema.loadRegex(); + }); + PHASES.put("load-csv", argsHelper -> { + final Schema schema = new Schema(argsHelper.get("properties file")); + final String source = argsHelper.get("source"); + final String name = argsHelper.get("name"); + final Set years = new HashSet<>(); + if ("file".equals(source)) { + years.addAll(schema.generate(null, 1, new File(name))); + } + else if ("trilium".equals(source)) { + schema.triliumLogin(schema.config.getTriliumPassword()); + years.addAll(schema.generate(null, 1, name)); + } + else { + System.out.println("First argument must be 'file' or 'trilium'"); + System.exit(1); + return; + } + schema.saveYears(years); + }); + PHASES.put("process", argsHelper -> { + final Schema schema = new Schema(argsHelper.get("properties file")); + schema.process(); + }); + PHASES.put("categorize", argsHelper -> { + final Schema schema = new Schema(argsHelper.get("properties file")); + schema.categorize(); + }); + } + + private void categorize() + throws SQLException, UnsupportedEncodingException, FileNotFoundException, IOException { + final Map> categoryMap = new HashMap<>(); + final File htmlFile = new File("categorize.html"); + try (final PrintStream ps = new PrintStream(new FileOutputStream(htmlFile))) { + try (Connection connection = pool.getConnection(8, false, true)) { + final Set years = getYears(connection); + final List catRegexes = getCatRegexes(connection); + final Set unresolvedSet = new HashSet<>(); + for (final String year : years) { + ps.println("${year}
".replace("${year}", year)); + dbExecute(connection, Util.getResourceAsString("clearExtraDescriptions.sql") // + .replace("${databaseName}", "budget_" + year)); + dbExecute(connection, + "drop table if exists ${databaseName}.transaction_regex_mtm" // + .replace("${databaseName}", "budget_" + year)); + dbExecute(connection, + Util.getResourceAsString("createTransactionRegexTable.sql") // + .replace("${databaseName}", "budget_" + year)); + final String sql = Util.getResourceAsString("getTransactionDescriptions.sql") // + .replace("${databaseName}", "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransactionRegex.sql") // + .replace("${databaseName}", "budget_" + year); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet rs = stmt.executeQuery()) { + ps.println(""); + while (rs.next()) { + final int transactionId = rs.getInt(1); + final java.sql.Date date = rs.getDate(2); + final String source = rs.getString(3); + final String description = rs.getString(4); + final BigDecimal amount = rs.getBigDecimal(5); + final Detail detail = new Detail(); + detail.transactionId = transactionId; + detail.source = source; + detail.description = description; + detail.date = date; + detail.amount = amount; + if (description != null) { + int maxPriority = Integer.MIN_VALUE; + final Map> matchesFound = new HashMap<>(); + for (final CatRegex catRegex : catRegexes) { + final String requiredSource = catRegex.getSource(); + if (requiredSource == null + || requiredSource.equals(source)) { + final Integer requiredYear = catRegex.getYear(); + if (requiredYear == null + || requiredYear.equals(Integer.valueOf(year))) { + final Pattern pattern = catRegex.getPattern(); + final Matcher matcher = pattern.matcher( + description); + final String category = catRegex.getCategory(); + if (matcher.matches()) { + final int priority = catRegex.getPriority(); + if (priority > maxPriority) { + maxPriority = priority; + final int regexId = catRegex.getId(); + linkRegex(connection, year, regexId, + transactionId); + } + Set set = matchesFound.get(category); + if (set == null) { + set = new HashSet<>(); + matchesFound.put(category, set); + } + set.add(catRegex); + } + } + } + } + if (matchesFound.isEmpty()) { + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) > 2023) { + final UnresolvedItem unresolvedItem = new UnresolvedItem( + Integer.parseInt(year), source, description, + date, amount); + unresolvedSet.add(unresolvedItem); + } + } + else if (matchesFound.size() > 1) { + // more than one category matches this description + // find the one(s) with the highest priority + final Map> actualMatchesFound = new HashMap<>(); + for (final String category : matchesFound.keySet()) { + final Set set = matchesFound.get(category); + for (final CatRegex catRegex : set) { + if (catRegex.getPriority() == maxPriority) { + Set newSet = actualMatchesFound.get( + category); + if (newSet == null) { + newSet = new HashSet<>(); + actualMatchesFound.put(category, newSet); + } + newSet.add(catRegex); + } + } + } + if (actualMatchesFound.size() > 1) { + // more than one category matches this description and also has the max priority + // print to HTML so user can make the regex more precise + final StringBuilder sb = new StringBuilder(); + sb.append( + ""); + ps.println(sb.toString()); + } + else { + addTransactionToCategory(year, connection, + actualMatchesFound, categoryMap, detail); + } + } + else { + addTransactionToCategory(year, connection, matchesFound, + categoryMap, detail); + } + } + } + ps.println("
${description}".replace( + "${description}", description)); + for (final String category : actualMatchesFound.keySet()) { + final Set newSet = actualMatchesFound.get( + category); + for (final CatRegex catRegex : newSet) { + sb.append( + "".replace( + "${catRegex}", catRegex.toString())); + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setInt(1, transactionId); + insertStmt.setInt(2, catRegex.getId()); + insertStmt.execute(); + } + } + } + sb.append("
${catRegex}
"); + } + } + } + final List unresolvedList = new ArrayList<>(unresolvedSet); + Collections.sort(unresolvedList); + ps.println("

Unresolved

"); + ps.println(""); + ps.println( + ""); + for (final UnresolvedItem unresolvedItem : unresolvedList) { + ps.println(unresolvedItem.replace( + "")); + } + ps.println("
yearsourcedescriptiondateamount
${year}${source}${description}${date}${amount}
"); + } + } + } + + private void loadRegex() throws SQLException, IOException { + try (Connection connection = pool.getConnection(8, false, true)) { + dbExecute(connection, "drop table if exists " + "budget.category"); + dbExecute(connection, Util.getResourceAsString("createCategoryTable.sql") // + .replace("${databaseName}", "budget")); + dbExecute(connection, "drop table if exists budget.regex"); + dbExecute(connection, Util.getResourceAsString("createRegexTable.sql").replace( + "${databaseName}", "budget")); + final List catRegexes = new ArrayList<>(); + final File catRegexesFile = new File("category-regex.lst"); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(catRegexesFile), "utf-8"))) { + String line = reader.readLine(); + while (line != null) { + if (line.trim().length() > 0) { + catRegexes.add(new CatRegex(line)); + } + line = reader.readLine(); + } + } + populateRegexTable(connection, catRegexes); + } + } + + Map loadCategories(final Connection connection) + throws IOException, SQLException { + final Map categories = new HashMap<>(); + final String sql = Util.getResourceAsString("getCategories.sql").replace("${databaseName}", + "budget"); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + int i = 0; + final int id = resultSet.getInt(++i); + final int parentIdInt = resultSet.getInt(++i); + final Integer parentId = resultSet.wasNull() ? null + : Integer.valueOf(parentIdInt); + final String name = resultSet.getString(++i); + final Category category = new Category(id, parentId, name); + categories.put(Integer.valueOf(category.id), category); + } + } + } + return categories; + } + + List getCatRegexes(final Connection connection) throws IOException, SQLException { + final List catRegexes = new ArrayList<>(); + final Map categories = loadCategories(connection); + final String sql = Util.getResourceAsString("getRegexes.sql").replace("${databaseName}", + "budget"); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + int i = 0; + final int id = resultSet.getInt(++i); + final int categoryId = resultSet.getInt(++i); + final String regex = resultSet.getString(++i); + final int flags = resultSet.getInt(++i); + final String source = resultSet.getString(++i); + final int priority = resultSet.getInt(++i); + final String description = resultSet.getString(++i); + final int yearInt = resultSet.getInt(++i); + final Integer year = resultSet.wasNull() ? null : Integer.valueOf(yearInt); + final Pattern pattern = Pattern.compile(regex, flags); + final Category category = categories.get(categoryId); + final String categoryName = category == null ? null + : category.getFullName(categories); + final CatRegex catRegex = new CatRegex(id, pattern, categoryName, source, + priority, description, year); + catRegexes.add(catRegex); + } + } + } + return catRegexes; + } + + private void populateRegexTable(final Connection connection, final List catRegexes) + throws IOException, SQLException { + System.out.println("populate regex table"); + final String tableName = "regex"; + if (!tableExists(connection, null, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget").replace("${tableName}", tableName)); + } + final String insertSql = Util.getResourceAsString("insertRegex.sql").replace( + "${databaseName}", "budget"); + for (final CatRegex catRegex : catRegexes) { + final Integer categoryId = getCategoryId(connection, catRegex.getCategory()); + try (PreparedStatement insertStmt = connection.prepareStatement(insertSql, + Statement.RETURN_GENERATED_KEYS)) { + insertStmt.setInt(1, categoryId); + insertStmt.setString(2, catRegex.getPattern().pattern()); + insertStmt.setInt(3, catRegex.getPattern().flags()); + insertStmt.setString(4, catRegex.getSource()); + insertStmt.setInt(5, catRegex.getPriority()); + insertStmt.setString(6, catRegex.getExtraDescription()); + final Integer year = catRegex.getYear(); + if (year == null) { + insertStmt.setNull(7, Types.INTEGER); + } + else { + insertStmt.setInt(7, catRegex.getYear()); + } + insertStmt.execute(); + final ResultSet rs = insertStmt.getGeneratedKeys(); + while (rs.next()) { + final int regexId = rs.getInt(1); + catRegex.setId(regexId); + } + } + } + } + + void addTransactionToCategory(final String year, final Connection connection, + final Map> matchesFound, + final Map> categoryMap, final Detail detail) + throws SQLException, IOException { + final StringBuilder sb = new StringBuilder(); + String sep = ""; + for (final String category : matchesFound.keySet()) { + final Set regexSet = matchesFound.get(category); + List details = categoryMap.get(category); + if (details == null) { + details = new ArrayList<>(); + categoryMap.put(category, details); + } + details.add(detail); + for (final CatRegex regex : regexSet) { + sb.append(sep); + sep = ", "; + sb.append(regex.getExtraDescription()); + } + } + final String setSql = Util.getResourceAsString("setExtraDescription.sql") // + .replace("${databaseName}", "budget_" + year); + try (PreparedStatement setStmt = connection.prepareStatement(setSql)) { + setStmt.setString(1, sb.toString()); + setStmt.setInt(2, detail.transactionId); + setStmt.execute(); + } + } + + private Set getYears(final Connection connection) throws IOException, SQLException { + final Set years = new HashSet<>(); + final String sql = Util.getResourceAsString("getYears.sql").replace("${databaseName}", + "budget"); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final int year = resultSet.getInt(1); + years.add(String.valueOf(year)); + } + } + } + return years; + } + + private void process() throws SQLException, IOException { + try (Connection connection = pool.getConnection(8, false, true)) { + final Set years = getYears(connection); + for (final String year : years) { + dbExecute(connection, "drop table if exists budget_" + year + ".transaction"); + dbExecute(connection, + Util.getResourceAsString("createTransactionTable.sql").replace( + "${databaseName}", "budget_" + year)); + ingestAmazonDigitalOrders(connection, year, "sandy"); + ingestAmazonDigitalOrders(connection, year, "steve"); + ingestAmazonDigitalReturns(connection, year, "sandy"); + ingestAmazonDigitalReturns(connection, year, "steve"); + ingestAmazonRetailOrders(connection, year, "sandy"); + ingestAmazonRetailOrders(connection, year, "steve"); + ingestChase(connection, year); + ingestDiscover(connection, year); + ingestCiti(connection, year); + ingestFirstBank(connection, year); + ingestPaypal(connection, year); + } + } + } + + private void ingestAmazonDigitalOrders(final Connection connection, final String year, + final String person) throws IOException, SQLException { + System.out.println("process amazon digital orders budget_" + year + " " + person); + final String tableName = "amz_${person}_dig_ord_dig_items".replace("${person}", person); + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String digOrdersTableName = "amz_${person}_dig_ord_dig_orders".replace("${person}", + person); + if (!tableExists(connection, year, digOrdersTableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", digOrdersTableName)); + return; + } + final String digOrdersMonetaryTableName = "amz_${person}_dig_ord_dig_orders_monetary".replace( + "${person}", person); + if (!tableExists(connection, year, digOrdersMonetaryTableName)) { + System.out.println( + " table ${databaseName}.${tableName} does not exist".replace("${databaseName}", + "budget_" + year).replace("${tableName}", digOrdersMonetaryTableName)); + return; + } + final String sql = Util.getResourceAsString("getDigitalOrders.sql").replace( + "${databaseName}", "budget_" + year).replace("${person}", person).replace( + "${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( + "${digOrdItems}", tableName).replace("${digOrders}", + digOrdersTableName).replace("${digOrdersMonetary}", + digOrdersMonetaryTableName); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year).replace("${person}", person); + final String source = "amz.${person}.dig.ord".replace("${person}", person); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final String type = resultSet.getString(1); + final String productName = resultSet.getString(2); + final String uniqueId = resultSet.getString(3); + final java.sql.Date date = resultSet.getDate(4); + final BigDecimal amount = resultSet.getBigDecimal(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, uniqueId); + insertStmt.setString(3, "Price Amount".equals(type) ? "Sale" : type); + insertStmt.setString(4, productName); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestAmazonRetailOrders(final Connection connection, final String year, + final String person) throws IOException, SQLException { + System.out.println("process amazon retail orders " + year + " " + person); + final String tableName = "amz_${person}_retail_order_history".replace("${person}", person); + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getRetailOrders.sql").replace( + "${databaseName}", "budget_" + year).replace("${person}", person).replace( + "${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( + "${retailOrders}", tableName); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "amz.${person}.retail.ord".replace("${person}", person); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final String website = resultSet.getString(1); + final String productName = resultSet.getString(2); + final String uniqueId = resultSet.getString(3); + final java.sql.Date date = resultSet.getDate(4); + final BigDecimal amount = resultSet.getBigDecimal(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, uniqueId); + insertStmt.setString(3, "Sale"); + insertStmt.setString(4, productName + " (" + website + ")"); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestDiscover(final Connection connection, final String year) + throws IOException, SQLException { + System.out.println("process discover " + year); + final String tableName = "discover"; + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getDiscover.sql").replace("${databaseName}", + "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "discover"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final java.sql.Date transDate = resultSet.getDate(1); + resultSet.getDate(2); + final String description = resultSet.getString(3); + final BigDecimal amount = resultSet.getBigDecimal(4); + final String category = resultSet.getString(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(transDate); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, ""); // unique_id + insertStmt.setString(3, ""); // type + insertStmt.setString(4, description + " (" + category + ")"); + insertStmt.setDate(5, transDate); + insertStmt.setBigDecimal(6, amount); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestCiti(final Connection connection, final String year) + throws IOException, SQLException { + System.out.println("process citi " + year); + final String tableName = "citi"; + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getCiti.sql").replace("${databaseName}", + "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "citi"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + resultSet.getString(1); + final java.sql.Date date = resultSet.getDate(2); + final String description = resultSet.getString(3); + final BigDecimal debit = resultSet.getBigDecimal(4); + resultSet.getString(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, ""); // unique_id + insertStmt.setString(3, ""); // type + insertStmt.setString(4, description); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, debit); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestFirstBank(final Connection connection, final String year) + throws IOException, SQLException { + System.out.println("process firstbank " + year); + final String tableName = "first_bank"; + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getFirstBank.sql").replace("${databaseName}", + "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "firstbank"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final java.sql.Date date = resultSet.getDate(1); + final String description = resultSet.getString(2); + final String type = resultSet.getString(3); + final BigDecimal amount = resultSet.getBigDecimal(4); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, ""); // unique_id + insertStmt.setString(3, type); // type + insertStmt.setString(4, description); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount.negate()); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestChase(final Connection connection, final String year) + throws IOException, SQLException { + System.out.println("process chase " + year); + final String tableName = "chase"; + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getChase.sql").replace("${databaseName}", + "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "chase"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final java.sql.Date date = resultSet.getDate(1); + final String description = resultSet.getString(2); + final String category = resultSet.getString(3); + final String type = resultSet.getString(4); + final BigDecimal amount = resultSet.getBigDecimal(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, ""); // unique_id + insertStmt.setString(3, type); // type + insertStmt.setString(4, description + " (" + category + ")"); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount.negate(MathContext.DECIMAL64)); + insertStmt.execute(); + } + } + } + } + } + } + + private void ingestPaypal(final Connection connection, final String year) + throws IOException, SQLException { + System.out.println("process paypal " + year); + final String tableName = "paypal"; + if (!tableExists(connection, year, tableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", tableName)); + return; + } + final String sql = Util.getResourceAsString("getPaypal.sql").replace("${databaseName}", + "budget_" + year); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year); + final String source = "paypal"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + int i = 0; + final java.sql.Date date = resultSet.getDate(++i); + final String time = resultSet.getString(++i); + final String timeZone = resultSet.getString(++i); + final String name = resultSet.getString(++i); + final String type = resultSet.getString(++i).trim(); + final String status = resultSet.getString(++i); + final String currency = resultSet.getString(++i); + final BigDecimal amount = resultSet.getBigDecimal(++i); + final String receiptId = resultSet.getString(++i); + final BigDecimal balance = resultSet.getBigDecimal(++i); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if ((cal.get(Calendar.YEAR) == Integer.parseInt(year)) + && !"General Card Deposit".equals(type) + && !"General Card Withdrawal".equals(type) + && !"General Authorization".equals(type) + && !"Bank Deposit to PP Account".equals(type) && amount != null) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, ""); // unique_id + insertStmt.setString(3, type); // type + insertStmt.setString(4, + name + " (" + time + " " + timeZone + ", " + status + ", " + balance + + " " + currency + (receiptId == null ? "" : (", " + receiptId)) + + ")"); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount.negate(MathContext.DECIMAL64)); + insertStmt.execute(); + } + } + } + } + } + } + + private boolean tableExists(final Connection connection, final String year, + final String tableName) throws IOException, SQLException { + final String sql = Util.getResourceAsString("findTable.sql"); + int tableRows = 0; + boolean tableExists = false; + final String dbname = year == null ? "budget" : ("budget_" + year); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + stmt.setString(1, dbname); + stmt.setString(2, tableName); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + tableRows += rs.getInt(1); + tableExists = true; + } + } + } + System.out.println("Table " + dbname + "." + tableName + " rows = " + tableRows); + return tableExists; + } + + private void ingestAmazonDigitalReturns(final Connection connection, final String year, + final String person) throws IOException, SQLException { + System.out.println("process amazon digital returns budget_" + year + " " + person); + final String digOrdItemsTableName = "amz_${person}_dig_ord_dig_items".replace("${person}", + person); + final String digOrdReturnsMonetaryTableName = "amz_${person}_dig_orders_returns_dig_orders_returns_monetary_1".replace( + "${person}", person); + if (!tableExists(connection, year, digOrdItemsTableName)) { + System.out.println(" table ${databaseName}.${tableName} does not exist".replace( + "${databaseName}", "budget_" + year).replace("${tableName}", digOrdItemsTableName)); + return; + } + if (!tableExists(connection, year, digOrdReturnsMonetaryTableName)) { + System.out.println( + " table ${databaseName}.${tableName} does not exist".replace("${databaseName}", + "budget_" + year).replace("${tableName}", digOrdReturnsMonetaryTableName)); + return; + } + final String sql = Util.getResourceAsString("getDigitalReturns.sql").replace( + "${databaseName}", "budget_" + year).replace("${person}", person).replace( + "${productNameCol}", "2023".equals(year) ? "title" : "product_name").replace( + "${digOrdItems}", digOrdItemsTableName).replace("${digOrdReturnsMonetary}", + digOrdReturnsMonetaryTableName); + final String insertSql = Util.getResourceAsString("insertTransaction.sql").replace( + "${databaseName}", "budget_" + year).replace("${person}", person); + final String source = "amz.${person}.dig.ret".replace("${person}", person); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + try (ResultSet resultSet = stmt.executeQuery()) { + while (resultSet.next()) { + final String type = resultSet.getString(1); + final String productName = resultSet.getString(2); + final String uniqueId = resultSet.getString(3); + final java.sql.Date date = resultSet.getDate(4); + final BigDecimal amount = resultSet.getBigDecimal(5); + final Calendar cal = new GregorianCalendar(); + cal.setTime(date); + if (cal.get(Calendar.YEAR) == Integer.parseInt(year)) { + try (PreparedStatement insertStmt = connection.prepareStatement( + insertSql)) { + insertStmt.setString(1, source); + insertStmt.setString(2, uniqueId); + insertStmt.setString(3, type); + insertStmt.setString(4, productName); + insertStmt.setDate(5, date); + insertStmt.setBigDecimal(6, amount); + insertStmt.execute(); + } + } + } + } + } + } + + private void linkRegex(final Connection connection, final String year, final Integer regexId, + final int transactionId) throws IOException, SQLException { + final String sql = Util.getResourceAsString("updateRegexLink.sql").replace( + "${databaseName}", "budget_" + year); + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + stmt.setInt(1, regexId); + stmt.setInt(2, transactionId); + stmt.execute(); + } + } + + private Integer getCategoryId(final Connection connection, final String category) + throws SQLException, IOException { + final String[] categoryParts = category.split("\\."); + Integer categoryPartId = null; + for (final String categoryPart : categoryParts) { + categoryPartId = getCategoryPartId(connection, categoryPartId, categoryPart); + } + return categoryPartId; + } + + private Integer getCategoryPartId(final Connection connection, final Integer parentCategoryId, + final String categoryPart) throws SQLException, IOException { + final String getRootSql = Util.getResourceAsString("getRootCategoryId.sql").replace( + "${databaseName}", "budget"); + final String getChildSql = Util.getResourceAsString("getChildCategoryId.sql").replace( + "${databaseName}", "budget"); + Integer categoryId = null; + final String getSql = parentCategoryId == null ? getRootSql : getChildSql; + try (PreparedStatement stmt = connection.prepareStatement(getSql)) { + stmt.setString(1, categoryPart); + if (parentCategoryId != null) { + stmt.setInt(2, parentCategoryId); + } + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + if (categoryId != null) { + System.out.println("Category " + parentCategoryId + " " + categoryPart + + " is defined more than once in the category table"); + } + else { + categoryId = Integer.valueOf(rs.getInt(1)); + } + } + } + } + if (categoryId == null) { + final String insertSql = parentCategoryId == null // + ? Util.getResourceAsString("insertRootCategory.sql").replace("${databaseName}", + "budget") // + : Util.getResourceAsString("insertChildCategory.sql").replace("${databaseName}", + "budget"); + try (PreparedStatement insertStmt = connection.prepareStatement(insertSql, + Statement.RETURN_GENERATED_KEYS)) { + insertStmt.setString(1, categoryPart); + if (parentCategoryId != null) { + insertStmt.setInt(2, parentCategoryId); + } + insertStmt.execute(); + final ResultSet rs = insertStmt.getGeneratedKeys(); + while (rs.next()) { + categoryId = Integer.valueOf(rs.getInt(1)); + } + } + } + return categoryId; + } + + private enum FieldType { + NUMBER, DATE, DATETIME, TEXT, UNKNOWN + } + + private enum IntType { + BIGINT(), INT, MEDIUMINT, SMALLINT, TINYINT + } + + private enum DateType { + YYYYMMDD, MMDDYYYY, MMDDYY, ISO; + + java.sql.Date toDate(final String fieldValue) { + Matcher matcher; + Calendar cal; + switch (this) { + case YYYYMMDD: + matcher = DATE_PATTERN_YYYYMMDD.matcher(fieldValue); + if (matcher.matches()) { + cal = new GregorianCalendar(); + cal.set(Calendar.YEAR, Integer.parseInt(matcher.group(1))); + cal.set(Calendar.MONTH, Integer.parseInt(matcher.group(2)) - 1); + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(matcher.group(3))); + return new java.sql.Date(cal.getTimeInMillis()); + } + case MMDDYYYY: + matcher = DATE_PATTERN_MMDDYYYY.matcher(fieldValue); + if (matcher.matches()) { + cal = new GregorianCalendar(); + cal.set(Calendar.YEAR, Integer.parseInt(matcher.group(3))); + cal.set(Calendar.MONTH, Integer.parseInt(matcher.group(1)) - 1); + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(matcher.group(2))); + return new java.sql.Date(cal.getTimeInMillis()); + } + case MMDDYY: + matcher = DATE_PATTERN_MMDDYY.matcher(fieldValue); + if (matcher.matches()) { + cal = new GregorianCalendar(); + cal.set(Calendar.YEAR, Integer.parseInt(matcher.group(3)) + 2000); + cal.set(Calendar.MONTH, Integer.parseInt(matcher.group(1)) - 1); + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(matcher.group(2))); + return new java.sql.Date(cal.getTimeInMillis()); + } + return null; + case ISO: + matcher = DATETIME_PATTERN_SECS.matcher(fieldValue); + if (matcher.matches()) { + cal = new GregorianCalendar(); + try { + cal.set(Calendar.YEAR, Integer.parseInt(matcher.group(1))); + cal.set(Calendar.MONTH, Integer.parseInt(matcher.group(2)) - 1); + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(matcher.group(3))); + cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(matcher.group(4))); + cal.set(Calendar.MINUTE, Integer.parseInt(matcher.group(5))); + cal.set(Calendar.SECOND, Double.valueOf(matcher.group(6)).intValue()); + } + catch (final NumberFormatException e) { + System.out.println("Number format exception on " + fieldValue); + for (int i = 0; i < 6; i++) { + System.out.println(matcher.group(i + 1)); + } + } + return new java.sql.Date(cal.getTimeInMillis()); + } + matcher = DATETIME_PATTERN.matcher(fieldValue); + if (matcher.matches()) { + cal = new GregorianCalendar(); + cal.set(Calendar.YEAR, Integer.parseInt(matcher.group(1))); + cal.set(Calendar.MONTH, Integer.parseInt(matcher.group(2)) - 1); + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(matcher.group(3))); + cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(matcher.group(4))); + cal.set(Calendar.MINUTE, Integer.parseInt(matcher.group(5))); + cal.set(Calendar.SECOND, Integer.parseInt(matcher.group(6))); + return new java.sql.Date(cal.getTimeInMillis()); + } + return null; + } + return null; + } + } + + private static class FieldInfo { + String colName = null; + int fractionPlaces = 0; + int maxLength = 0; + int wholePlaces; + FieldType type = FieldType.UNKNOWN; + IntType intType = IntType.INT; + DateType dateType = DateType.YYYYMMDD; + } + + private void triliumLogin(final String password) throws URISyntaxException, IOException { + final String triliumUrl = config.getTriliumUrl(); + if (triliumUrl == null) { + throw new IllegalArgumentException("Trillium URL not provided"); + } + final URI uri = new URI(triliumUrl + "/auth/login"); + final URL url = uri.toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Accept", "application/json"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setDoOutput(true); + final String request = "{\"password\":\"" + password + "\"}"; + try (final OutputStream os = conn.getOutputStream()) { + final byte[] input = request.getBytes("utf-8"); + os.write(input, 0, input.length); + } + try (BufferedReader br = new BufferedReader( + new InputStreamReader(conn.getInputStream(), "utf-8"))) { + final StringBuilder sb = new StringBuilder(); + String responseLine = null; + while ((responseLine = br.readLine()) != null) { + sb.append(responseLine.trim()); + } + final String responseString = sb.toString(); + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode node = mapper.readTree(responseString); + final JsonNode authTokenNode = node.get("authToken"); + if (authTokenNode == null) { + throw new RuntimeException("Trilium login failed"); + } + triliumAuthToken = authTokenNode.textValue(); + } + if (conn.getResponseCode() != 201) { + throw new RuntimeException("Failed: HTTP error code = " + conn.getResponseCode()); + } + } + + private String getTriliumContent(final String urlPath) throws URISyntaxException, IOException { + final String triliumUrl = config.getTriliumUrl(); + if (triliumUrl == null) { + throw new IllegalArgumentException("Trillium URL not provided"); + } + final URI uri = new URI(triliumUrl + urlPath); + final URL url = uri.toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("Accept", "application/json"); + conn.setRequestProperty("Authorization", triliumAuthToken); + if (conn.getResponseCode() != 200) { + throw new RuntimeException("Failed: HTTP error code = " + conn.getResponseCode()); + } + final BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); + final StringBuilder sb = new StringBuilder(); + String output; + while ((output = br.readLine()) != null) { + sb.append(output); + } + conn.disconnect(); + return sb.toString(); + } + + private JsonNode getTriliumJsonContent(final String urlPath) + throws URISyntaxException, IOException { + final String triliumUrl = config.getTriliumUrl(); + if (triliumUrl == null) { + throw new IllegalArgumentException("Trillium URL not provided"); + } + final URI uri = new URI(triliumUrl + urlPath); + final URL url = uri.toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("Accept", "application/json"); + conn.setRequestProperty("Authorization", triliumAuthToken); + if (conn.getResponseCode() != 200) { + throw new RuntimeException("Failed: HTTP error code = " + conn.getResponseCode()); + } + final BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); + final StringBuilder sb = new StringBuilder(); + String output; + while ((output = br.readLine()) != null) { + sb.append(output); + } + conn.disconnect(); + final ObjectMapper mapper = new ObjectMapper(); + return mapper.readTree(sb.toString()); + } + + private List getTriliumRecords(final String urlPath) + throws URISyntaxException, IOException, CsvException { + final String triliumUrl = config.getTriliumUrl(); + if (triliumUrl == null) { + throw new IllegalArgumentException("Trillium URL not provided"); + } + final URI uri = new URI(triliumUrl + urlPath); + final URL url = uri.toURL(); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("Accept", "application/json"); + conn.setRequestProperty("Authorization", triliumAuthToken); + if (conn.getResponseCode() != 200) { + throw new RuntimeException("Failed: HTTP error code = " + conn.getResponseCode()); + } + List result; + try (Reader reader = new InputStreamReader(conn.getInputStream())) { + try (CSVReader csvReader = new CSVReader(reader)) { + result = csvReader.readAll(); + } + } + conn.disconnect(); + return result; + } + + private Set generate(final String name, final int siblingCount, + final String triliumSearch) throws Exception { + final Set years = new HashSet<>(); + final String output = getTriliumContent( + "/notes?search=" + triliumSearch + "&fastSearch=true"); + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode node = mapper.readTree(output); + final JsonNode resultsNode = node.get("results"); + for (final JsonNode noteNode : resultsNode) { + final JsonNode titleNode = noteNode.get("title"); + if ((titleNode == null) || !"Budget".equals(titleNode.textValue())) { + continue; + } + final JsonNode parentNodeIdsNode = noteNode.get("parentNoteIds"); + if (parentNodeIdsNode == null) { + continue; + } + boolean rootFound = false; + for (final JsonNode parentNodeIdNode : parentNodeIdsNode) { + if ("root".equals(parentNodeIdNode.textValue())) { + rootFound = true; + break; + } + } + if (!rootFound) { + continue; + } + years.addAll(generate(name, 1, noteNode)); + } + return years; + } + + public Set generate(String name, final int siblingCount, final JsonNode noteNode) + throws Exception { + final Set years = new HashSet<>(); + final JsonNode titleNode = noteNode.get("title"); + if (titleNode == null) { + return years; + } + String childName = titleNode.textValue(); + String ext = null; + final int lastIndexOfDot = childName.lastIndexOf("."); + if (lastIndexOfDot >= 0) { + ext = childName.substring(lastIndexOfDot + 1); + childName = childName.substring(0, lastIndexOfDot); + } + if (name == null) { + name = childName; + } + else if (siblingCount > 1) { + name = name + "." + childName; + } + if ("csv".equalsIgnoreCase(ext)) { + final JsonNode noteIdNode = noteNode.get("noteId"); + if (noteIdNode != null) { + final String noteId = noteIdNode.textValue(); + final List records = getTriliumRecords("/notes/" + noteId + "/content"); + years.addAll(generate(name, siblingCount, records, + childName.startsWith("FirstbankDownload"), "qb.csv".equals(childName))); + } + } + final JsonNode childNodeIdsNode = noteNode.get("childNoteIds"); + if (childNodeIdsNode == null) { + return years; + } + for (final JsonNode childNodeIdNode : childNodeIdsNode) { + final String childNodeId = childNodeIdNode.textValue(); + final JsonNode childNode = getTriliumJsonContent("/notes/" + childNodeId); + years.addAll(generate(name, childNodeIdsNode.size(), childNode)); + } + return years; + } + + private Set generate(String name, final int siblingCount, final File file) + throws Exception { + String childName = file.getName(); + String ext = null; + if (file.isFile()) { + final int lastIndexOfDot = childName.lastIndexOf("."); + if (lastIndexOfDot >= 0) { + ext = childName.substring(lastIndexOfDot + 1); + childName = childName.substring(0, lastIndexOfDot); + } + } + if (name == null) { + name = childName; + } + else if (siblingCount > 1) { + name = name + "." + childName; + } + final Set years = new HashSet<>(); + if (file.isDirectory()) { + final File[] children = file.listFiles(); + for (final File child : children) { + years.addAll(generate(name, children.length, child)); + } + } + else if (file.isFile() && "csv".equalsIgnoreCase(ext)) { + final List records = readAllLines(Paths.get(file.getPath())); + years.addAll(generate(name, siblingCount, records, + file.getName().startsWith("FirstbankDownload"), "qb.csv".equals(file.getName()))); + } + return years; + } + + private static Pattern NAME_PATTERN = Pattern.compile("^Budget\\.([0-9]+)\\.(.*)$"); + + private Set generate(final String name, final int siblingCount, + final List records, final boolean isFirstBank, final boolean isQuickBooks) + throws SQLException { + int recordIndex = 0; + final Set years = new HashSet<>(); + List fields = null; + for (final String[] record : records) { + if (recordIndex == 0) { + fields = new ArrayList<>(); + if (isFirstBank) { + FieldInfo field = new FieldInfo(); + field.colName = "date"; + fields.add(field); + field = new FieldInfo(); + field.colName = "description"; + fields.add(field); + field = new FieldInfo(); + field.colName = "type"; + fields.add(field); + field = new FieldInfo(); + field.colName = "amount"; + fields.add(field); + } + else { + for (final String fieldName : record) { + if ("".equals(fieldName)) { + fields.add(null); + } + else { + final FieldInfo field = new FieldInfo(); + field.colName = decamel(fieldName); + fields.add(field); + } + } + } + } + else { + boolean skipRecord = false; + if (isQuickBooks && (recordIndex == 1 || recordIndex == records.size() - 1)) { + skipRecord = true; + } + if (!skipRecord) { + int fieldIndex = 0; + for (final String fieldValue : record) { + final int fieldLength = fieldValue.length(); + final FieldInfo field = fieldIndex < fields.size() + ? fields.get(fieldIndex++) + : null; + if (field == null) { + continue; + } + if (field.maxLength < fieldLength) { + field.maxLength = fieldLength; + } + if (FieldType.TEXT.equals(field.type)) { + // already determined to be TEXT so it can't be anything else + continue; + } + Matcher matcher = INT_PATTERN.matcher(fieldValue.replace(",", "")); + if (matcher.matches()) { + final String numString = matcher.group(1); + field.type = FieldType.NUMBER; + final int wholePlaces = numString.length(); + if (field.wholePlaces < wholePlaces) { + field.wholePlaces = wholePlaces; + } + final long value = Long.parseLong(fieldValue); + // https://dev.mysql.com/doc/refman/8.0/en/integer-types.html + if (value > Long.MAX_VALUE || value < Long.MIN_VALUE) { + field.type = FieldType.TEXT; + } + else if (value > 2147483647 || value < -2147483648) { + field.intType = IntType.BIGINT; + } + else if (value > 8388607 || value < -8388608) { + field.intType = IntType.INT; + } + else if (value > 32767 || value < -32768) { + field.intType = IntType.MEDIUMINT; + } + else if (value > 127 || value < -128) { + field.intType = IntType.SMALLINT; + } + else { + field.intType = IntType.TINYINT; + } + continue; + } + matcher = DECIMAL_PATTERN.matcher(fieldValue.replace(",", "")); + if (matcher.matches()) { + final String wholeString = matcher.group(2); + final String fracString = matcher.group(3); + field.type = FieldType.NUMBER; + final int scalePlaces = fracString.length(); + if (field.fractionPlaces < scalePlaces) { + field.fractionPlaces = scalePlaces; + } + final int wholePlaces = wholeString.length(); + if (field.wholePlaces < wholePlaces) { + field.wholePlaces = wholePlaces; + } + continue; + } + matcher = DATE_PATTERN_YYYYMMDD.matcher(fieldValue); + if (matcher.matches()) { + field.type = FieldType.DATE; + continue; + } + matcher = DATE_PATTERN_MMDDYYYY.matcher(fieldValue); + if (matcher.matches()) { + field.type = FieldType.DATE; + field.dateType = DateType.MMDDYYYY; + continue; + } + matcher = DATE_PATTERN_MMDDYY.matcher(fieldValue); + if (matcher.matches()) { + field.type = FieldType.DATE; + field.dateType = DateType.MMDDYY; + continue; + } + matcher = DATETIME_PATTERN_SECS.matcher(fieldValue); + if (matcher.matches()) { + field.type = FieldType.DATETIME; + field.dateType = DateType.ISO; + continue; + } + matcher = DATETIME_PATTERN.matcher(fieldValue); + if (matcher.matches()) { + field.type = FieldType.DATETIME; + field.dateType = DateType.ISO; + continue; + } + if (!"N/A".equals(fieldValue) + && !"Not Applicable".equalsIgnoreCase(fieldValue) + && !"Not Available".equalsIgnoreCase(fieldValue) + && (!"No Refund".equalsIgnoreCase(fieldValue) + || !"return_amount".equals(field.colName)) + && !fieldValue.startsWith("Due to technical limitations") + && (!"".equals(fieldValue) || FieldType.TEXT.equals(field.type))) { + field.type = FieldType.TEXT; + } + continue; + } + } + } + recordIndex++; + } + final Matcher matcher = NAME_PATTERN.matcher(name); + if (!matcher.matches()) { + throw new RuntimeException("Name does not begin with Budget.[year]"); + } + final String year = matcher.group(1); + years.add(year); + final String tableName = decamel(matcher.group(2)).replace("budget_", "").replace("amazon_", + "amz_").replace("_digital_", "_dig_").replace("_information", "_inf").replace( + "_payment_", "_pmt_").replace("_ordering_", "_ord_").replace("_customer_", + "_cust_").replace("_communication_", "_comm_").replace("_preferences_", + "_pref_").replace("_physical_", "_phys_").replace("_whole_foods_", "_wf_"); + if (fields != null) { + System.out.println("Loading CSV " + name); + try (Connection connection = pool.getConnection(8, false, true)) { + final String qualifiedTableName = "budget_" + year + "." + tableName; + dbExecute(connection, "drop table if exists " + qualifiedTableName); + dbExecute(connection, getCreateTableSql(qualifiedTableName, fields)); + doInserts(connection, qualifiedTableName, fields, records, isFirstBank, + isQuickBooks); + } + } + return years; + } + + private void saveYears(final Set years) throws SQLException, IOException { + try (Connection connection = pool.getConnection(8, false, true)) { + dbExecute(connection, "drop table if exists budget.years"); + dbExecute(connection, Util.getResourceAsString("createYearsTable.sql").replace( + "${databaseName}", "budget")); + final String insertSql = Util.getResourceAsString("insertYear.sql").replace( + "${databaseName}", "budget"); + for (final String year : years) { + try (PreparedStatement insertStmt = connection.prepareStatement(insertSql)) { + insertStmt.setInt(1, Integer.parseInt(year)); + insertStmt.execute(); + } + } + } + } + + private void doInserts(final Connection connection, final String tableName, + final List fields, final List records, final boolean isFirstBank, + final boolean isQuickBooks) throws SQLException { + int recordIndex = 0; + for (final String[] record : records) { + boolean skipRecord = false; + if (isQuickBooks) { + if (recordIndex <= 1 || recordIndex == records.size() - 1) { + skipRecord = true; + } + } + else if (!isFirstBank && (recordIndex == 0)) { + skipRecord = true; + } + if (!skipRecord) { + final String sql = getInsertSql(tableName, record, fields); + try (final PreparedStatement stmt = connection.prepareStatement(sql)) { + setValues(stmt, record, fields); + stmt.execute(); + } + } + recordIndex++; + } + } + + private void setValues(final PreparedStatement stmt, final String[] record, + final List fields) throws NumberFormatException, SQLException { + int index = 1; + for (int fieldIndex = 0; fieldIndex < record.length; fieldIndex++) { + final String fieldValue = record[fieldIndex]; + final FieldInfo field = fieldIndex < fields.size() ? fields.get(fieldIndex) : null; + if (field == null) { + continue; + } + if ("N/A".equals(fieldValue) || "Not Applicable".equalsIgnoreCase(fieldValue) + || "Not Available".equalsIgnoreCase(fieldValue) + || ("No Refund".equalsIgnoreCase(fieldValue) + && "return_amount".equals(field.colName)) + || fieldValue.startsWith("Due to technical limitations") + || ("".equals(fieldValue) && !FieldType.TEXT.equals(field.type))) { + stmt.setNull(index, java.sql.Types.CHAR); + } + else { + switch (field.type) { + case TEXT: + stmt.setString(index, fieldValue); + break; + case DATE: + stmt.setDate(index, field.dateType.toDate(fieldValue)); + break; + case DATETIME: + stmt.setDate(index, field.dateType.toDate(fieldValue)); + break; + case NUMBER: + try { + stmt.setDouble(index, Double.parseDouble(fieldValue.replace(",", ""))); + } + catch (final NumberFormatException e) { + System.out.println("Number format exception in " + fieldValue); + } + break; + case UNKNOWN: + throw new RuntimeException("UNKNOWN: " + fieldValue + " in " + field.colName); + default: + stmt.setString(index, fieldValue); + } + } + index++; + } + } + + private String getInsertSql(final String tableName, final String[] record, + final List fields) { + final StringBuilder sb = new StringBuilder(); + sb.append("insert into " + tableName + " ("); + String sep = ""; + for (int fieldIndex = 0; fieldIndex < record.length; fieldIndex++) { + final FieldInfo field = fieldIndex < fields.size() ? fields.get(fieldIndex) : null; + if (field != null) { + sb.append(sep); + sep = ", "; + sb.append(field.colName); + } + } + sb.append(") values ("); + sep = ""; + for (int fieldIndex = 0; fieldIndex < record.length; fieldIndex++) { + final FieldInfo field = fieldIndex < fields.size() ? fields.get(fieldIndex) : null; + if (field == null) { + continue; + } + sb.append(sep); + sep = ", "; + sb.append("?"); + } + sb.append(")"); + return sb.toString(); + } + + private String getCreateTableSql(final String tableName, final List fields) { + final StringBuilder sb = new StringBuilder(); + sb.append("create table " + tableName + "("); + String sep = "\n"; + for (final FieldInfo field : fields) { + if (field == null) { + continue; + } + final String colName = field.colName; + sb.append(sep); + sep = ",\n"; + String colType = "text"; + if (FieldType.NUMBER.equals(field.type)) { + if (field.fractionPlaces > 0) { + colType = "decimal(" + (field.wholePlaces + field.fractionPlaces) + "," + + field.fractionPlaces + ")"; + } + else { + colType = field.intType.name().toLowerCase() + "(" + field.wholePlaces + ")"; + } + } + else if (FieldType.DATE.equals(field.type)) { + colType = "date"; + } + else if (FieldType.DATETIME.equals(field.type)) { + colType = "datetime"; + } + else if (field.maxLength <= 32) { + colType = "varchar(" + field.maxLength + ")"; + } + else { + } + sb.append(colName + " " + colType); + } + sb.append(")"); + return sb.toString(); + } + + private void dbExecute(final Connection connection, final String string) throws SQLException { + try (final PreparedStatement stmt = connection.prepareStatement(string)) { + stmt.execute(); + } + } + + private String decamel(final String name) { + if (name == null || name.trim().length() == 0) { + return "unknown"; + } + final StringBuilder sb = new StringBuilder(); + final char[] chars = name.replace("ID", "Id").replace("RMA", "Rma").replace("ASIN", + "Asin").toCharArray(); + int charIndex = 0; + char prevChar = '_'; + while (charIndex < chars.length) { + final char curChar = chars[charIndex++]; + if (curChar == '.' || curChar == '-' || curChar == ' ') { + if (prevChar != '_') { + sb.append('_'); + } + prevChar = '_'; + continue; + } + if (Character.isUpperCase(curChar)) { + if (prevChar != '_') { + sb.append('_'); + } + prevChar = Character.toLowerCase(curChar); + sb.append(prevChar); + continue; + } + if (Character.isAlphabetic(curChar) || Character.isDigit(curChar)) { + prevChar = curChar; + sb.append(prevChar); + } + } + return sb.toString(); + } + + private List readAllLines(final Path filePath) throws Exception { + try (Reader reader = Files.newBufferedReader(filePath)) { + try (CSVReader csvReader = new CSVReader(reader)) { + return csvReader.readAll(); + } + } + } +} diff --git a/src/main/java/com/stephenschafer/budget/schema/UnresolvedItem.java b/src/main/java/com/stephenschafer/budget/schema/UnresolvedItem.java new file mode 100644 index 0000000..e25c72e --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/UnresolvedItem.java @@ -0,0 +1,61 @@ +package com.stephenschafer.budget.schema; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.text.DateFormat; +import java.text.SimpleDateFormat; + +class UnresolvedItem implements Comparable { + private final int year; + private final String source; + private final String description; + private final java.sql.Date date; + private final BigDecimal amount; + + public UnresolvedItem(final int year, final String source, final String description, + final java.sql.Date date, final BigDecimal amount) { + this.year = year; + this.source = source; + this.description = description; + this.date = date; + this.amount = amount; + } + + @Override + public int compareTo(final UnresolvedItem that) { + if (this.year < that.year) { + return -1; + } + if (this.year > that.year) { + return 1; + } + String thisSource = source; + if (thisSource == null) { + thisSource = ""; + } + String thatSource = that.source; + if (thatSource == null) { + thatSource = ""; + } + final int comparison = thisSource.compareTo(thatSource); + if (comparison != 0) { + return comparison; + } + String thisdescription = description; + if (thisdescription == null) { + thisdescription = ""; + } + String thatdescription = that.description; + if (thatdescription == null) { + thatdescription = ""; + } + return thisdescription.compareTo(thatdescription); + } + + public String replace(final String string) { + final DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + return string.replace("${year}", String.valueOf(year)).replace("${source}", source).replace( + "${description}", description).replace("${date}", df.format(date)).replace("${amount}", + amount.setScale(2, RoundingMode.HALF_UP).toPlainString()); + } +} \ No newline at end of file diff --git a/src/main/java/com/stephenschafer/budget/schema/Util.java b/src/main/java/com/stephenschafer/budget/schema/Util.java new file mode 100644 index 0000000..78a9b5c --- /dev/null +++ b/src/main/java/com/stephenschafer/budget/schema/Util.java @@ -0,0 +1,22 @@ +package com.stephenschafer.budget.schema; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; + +public class Util { + public static String getResourceAsString(final String resourceName) throws IOException { + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final StringBuilder sb = new StringBuilder(); + try (final Reader reader = new InputStreamReader( + classLoader.getResourceAsStream(resourceName))) { + final char[] buffer = new char[0x1000]; + int charsRead = reader.read(buffer); + while (charsRead >= 0) { + sb.append(buffer, 0, charsRead); + charsRead = reader.read(buffer); + } + } + return sb.toString(); + } +} diff --git a/src/main/resources/categories.js b/src/main/resources/categories.js new file mode 100644 index 0000000..4bee54e --- /dev/null +++ b/src/main/resources/categories.js @@ -0,0 +1,11 @@ +function showDetail(h2, categoryId) { + var tr = document.body.querySelector("#cat" + categoryId); + if(tr.style.display == "none") { + tr.style.display = ""; + h2.style.fontWeight = "bold"; + } + else { + tr.style.display = "none"; + h2.style.fontWeight = "normal"; + } +} diff --git a/src/main/resources/category.html b/src/main/resources/category.html new file mode 100644 index 0000000..badf9a2 --- /dev/null +++ b/src/main/resources/category.html @@ -0,0 +1,17 @@ + + +

${name}

+ + +

${amount}

+ + + + + + + + + + + diff --git a/src/main/resources/clearExtraDescriptions.sql b/src/main/resources/clearExtraDescriptions.sql new file mode 100644 index 0000000..d756d27 --- /dev/null +++ b/src/main/resources/clearExtraDescriptions.sql @@ -0,0 +1 @@ +update ${databaseName}.transaction set extra_description = '', regex_id = null \ No newline at end of file diff --git a/src/main/resources/createCategoryTable.sql b/src/main/resources/createCategoryTable.sql new file mode 100644 index 0000000..4c6013f --- /dev/null +++ b/src/main/resources/createCategoryTable.sql @@ -0,0 +1,6 @@ +create table ${databaseName}.category ( + id int not null primary key auto_increment, + parent_category_id int, + name text not null, + constraint unique key (parent_category_id, name) +) diff --git a/src/main/resources/createRegexTable.sql b/src/main/resources/createRegexTable.sql new file mode 100644 index 0000000..afb364a --- /dev/null +++ b/src/main/resources/createRegexTable.sql @@ -0,0 +1,10 @@ +create table ${databaseName}.regex ( + id int not null primary key auto_increment, + category_id int, + regex text not null, + flags int, + source varchar(32), + priority int, + description text, + year int +) diff --git a/src/main/resources/createTransactionRegexTable.sql b/src/main/resources/createTransactionRegexTable.sql new file mode 100644 index 0000000..987e91d --- /dev/null +++ b/src/main/resources/createTransactionRegexTable.sql @@ -0,0 +1,5 @@ +create table ${databaseName}.transaction_regex_mtm ( + regex_id int not null, + transaction_id int not null, + primary key (transaction_id, regex_id) +) \ No newline at end of file diff --git a/src/main/resources/createTransactionTable.sql b/src/main/resources/createTransactionTable.sql new file mode 100644 index 0000000..3e8974e --- /dev/null +++ b/src/main/resources/createTransactionTable.sql @@ -0,0 +1,12 @@ +create table ${databaseName}.transaction ( + id int not null primary key auto_increment, + source varchar(32), + unique_identifier varchar(64), + type varchar(64), + description text, + extra_description text, + date date, + amount decimal(10,2), + optional int default 0, + regex_id int +) diff --git a/src/main/resources/createYearsTable.sql b/src/main/resources/createYearsTable.sql new file mode 100644 index 0000000..d0a14cd --- /dev/null +++ b/src/main/resources/createYearsTable.sql @@ -0,0 +1,3 @@ +create table ${databaseName}.years ( + year int not null primary key +) diff --git a/src/main/resources/findTable.sql b/src/main/resources/findTable.sql new file mode 100644 index 0000000..51345d8 --- /dev/null +++ b/src/main/resources/findTable.sql @@ -0,0 +1 @@ +select table_rows from information_schema.tables where table_schema = ? and table_name = ? \ No newline at end of file diff --git a/src/main/resources/getCategories.sql b/src/main/resources/getCategories.sql new file mode 100644 index 0000000..b584f9e --- /dev/null +++ b/src/main/resources/getCategories.sql @@ -0,0 +1,5 @@ +select +id, +parent_category_id, +name +from ${databaseName}.category \ No newline at end of file diff --git a/src/main/resources/getChase.sql b/src/main/resources/getChase.sql new file mode 100644 index 0000000..a89c2c7 --- /dev/null +++ b/src/main/resources/getChase.sql @@ -0,0 +1,8 @@ +select +transaction_date, +description, +category, +type, +amount +from ${databaseName}.chase + diff --git a/src/main/resources/getChildCategoryId.sql b/src/main/resources/getChildCategoryId.sql new file mode 100644 index 0000000..57b7dd7 --- /dev/null +++ b/src/main/resources/getChildCategoryId.sql @@ -0,0 +1 @@ +select id from ${databaseName}.category where name = ? and parent_category_id = ? \ No newline at end of file diff --git a/src/main/resources/getCiti.sql b/src/main/resources/getCiti.sql new file mode 100644 index 0000000..db1e756 --- /dev/null +++ b/src/main/resources/getCiti.sql @@ -0,0 +1,7 @@ +select +status, +date, +description, +debit, +credit +from ${databaseName}.citi \ No newline at end of file diff --git a/src/main/resources/getDigitalOrders.sql b/src/main/resources/getDigitalOrders.sql new file mode 100644 index 0000000..32a6b1e --- /dev/null +++ b/src/main/resources/getDigitalOrders.sql @@ -0,0 +1,15 @@ +select +m.monetary_component_type_code as type, +i.${productNameCol} as name, +i.digital_order_item_id as order_id, +i.order_date as date, +sum(m.transaction_amount) as amount +from ${databaseName}.${digOrdItems} i +left outer join ${databaseName}.${digOrders} o on o.order_id = i.order_id +left outer join ${databaseName}.${digOrdersMonetary} m on m.digital_order_item_id = i.digital_order_item_id +group by +i.${productNameCol}, +i.digital_order_item_id, +i.order_date, +m.monetary_component_type_code + diff --git a/src/main/resources/getDigitalReturns.sql b/src/main/resources/getDigitalReturns.sql new file mode 100644 index 0000000..49a5bc9 --- /dev/null +++ b/src/main/resources/getDigitalReturns.sql @@ -0,0 +1,14 @@ +select +m.monetary_component_type as type, +i.${productNameCol} as name, +i.digital_order_item_id as order_id, +i.order_date as date, +-sum(m.transaction_amount) as amount +from ${databaseName}.${digOrdItems} i +inner join ${databaseName}.${digOrdReturnsMonetary} m on m.digital_order_item_id = i.digital_order_item_id +group by +i.${productNameCol}, +i.digital_order_item_id, +i.order_date, +m.monetary_component_type + diff --git a/src/main/resources/getDiscover.sql b/src/main/resources/getDiscover.sql new file mode 100644 index 0000000..02c1d94 --- /dev/null +++ b/src/main/resources/getDiscover.sql @@ -0,0 +1,7 @@ +select +trans_date, +post_date, +description, +amount, +category +from ${databaseName}.discover \ No newline at end of file diff --git a/src/main/resources/getFirstBank.sql b/src/main/resources/getFirstBank.sql new file mode 100644 index 0000000..e0dc4da --- /dev/null +++ b/src/main/resources/getFirstBank.sql @@ -0,0 +1,6 @@ +select +date, +description, +type, +amount +from ${databaseName}.first_bank \ No newline at end of file diff --git a/src/main/resources/getPaypal.sql b/src/main/resources/getPaypal.sql new file mode 100644 index 0000000..172ce8f --- /dev/null +++ b/src/main/resources/getPaypal.sql @@ -0,0 +1,13 @@ +select +date, +time, +time_zone, +name, +type, +status, +currency, +amount, +receipt_id, +balance +from ${databaseName}.paypal + diff --git a/src/main/resources/getRegexes.sql b/src/main/resources/getRegexes.sql new file mode 100644 index 0000000..cc2031e --- /dev/null +++ b/src/main/resources/getRegexes.sql @@ -0,0 +1,10 @@ +select +id, +category_id, +regex, +flags, +source, +priority, +description, +year +from ${databaseName}.regex \ No newline at end of file diff --git a/src/main/resources/getRetailOrders.sql b/src/main/resources/getRetailOrders.sql new file mode 100644 index 0000000..e7e1b7b --- /dev/null +++ b/src/main/resources/getRetailOrders.sql @@ -0,0 +1,8 @@ +select +website as website, +product_name as name, +order_id as order_id, +date(order_date) as date, +total_owed as amount +from ${databaseName}.${retailOrders} o + diff --git a/src/main/resources/getRootCategoryId.sql b/src/main/resources/getRootCategoryId.sql new file mode 100644 index 0000000..d7aab50 --- /dev/null +++ b/src/main/resources/getRootCategoryId.sql @@ -0,0 +1 @@ +select id from ${databaseName}.category where name = ? and parent_category_id is null \ No newline at end of file diff --git a/src/main/resources/getTransactionDescriptions.sql b/src/main/resources/getTransactionDescriptions.sql new file mode 100644 index 0000000..5b4fdc0 --- /dev/null +++ b/src/main/resources/getTransactionDescriptions.sql @@ -0,0 +1,3 @@ +select t.id, t.date, t.source, t.description, t.amount +from ${databaseName}.transaction t +where t.regex_id is null \ No newline at end of file diff --git a/src/main/resources/getYears.sql b/src/main/resources/getYears.sql new file mode 100644 index 0000000..4b36ede --- /dev/null +++ b/src/main/resources/getYears.sql @@ -0,0 +1,3 @@ +select +year +from ${databaseName}.years \ No newline at end of file diff --git a/src/main/resources/insertChildCategory.sql b/src/main/resources/insertChildCategory.sql new file mode 100644 index 0000000..e55d482 --- /dev/null +++ b/src/main/resources/insertChildCategory.sql @@ -0,0 +1 @@ +insert into ${databaseName}.category (name, parent_category_id) values (?, ?) \ No newline at end of file diff --git a/src/main/resources/insertRegex.sql b/src/main/resources/insertRegex.sql new file mode 100644 index 0000000..59b3246 --- /dev/null +++ b/src/main/resources/insertRegex.sql @@ -0,0 +1,9 @@ +insert into ${databaseName}.regex ( + category_id, + regex, + flags, + source, + priority, + description, + year +) values (?, ?, ?, ?, ?, ?, ?) \ No newline at end of file diff --git a/src/main/resources/insertRootCategory.sql b/src/main/resources/insertRootCategory.sql new file mode 100644 index 0000000..9801091 --- /dev/null +++ b/src/main/resources/insertRootCategory.sql @@ -0,0 +1 @@ +insert into ${databaseName}.category (name) values (?) \ No newline at end of file diff --git a/src/main/resources/insertTransaction.sql b/src/main/resources/insertTransaction.sql new file mode 100644 index 0000000..2f686c6 --- /dev/null +++ b/src/main/resources/insertTransaction.sql @@ -0,0 +1,8 @@ +insert into ${databaseName}.transaction ( + source, + unique_identifier, + type, + description, + date, + amount +) values (?, ?, ?, ?, ?, ?) \ No newline at end of file diff --git a/src/main/resources/insertTransactionRegex.sql b/src/main/resources/insertTransactionRegex.sql new file mode 100644 index 0000000..a5430b7 --- /dev/null +++ b/src/main/resources/insertTransactionRegex.sql @@ -0,0 +1 @@ +insert into ${databaseName}.transaction_regex_mtm (regex_id, transaction_id) values (?, ?) \ No newline at end of file diff --git a/src/main/resources/insertYear.sql b/src/main/resources/insertYear.sql new file mode 100644 index 0000000..bcc8e43 --- /dev/null +++ b/src/main/resources/insertYear.sql @@ -0,0 +1,3 @@ +insert into ${databaseName}.years ( + year +) values (?) \ No newline at end of file diff --git a/src/main/resources/setExtraDescription.sql b/src/main/resources/setExtraDescription.sql new file mode 100644 index 0000000..3e3337e --- /dev/null +++ b/src/main/resources/setExtraDescription.sql @@ -0,0 +1 @@ +update ${databaseName}.transaction set extra_description = ? where id = ? \ No newline at end of file diff --git a/src/main/resources/updateRegexLink.sql b/src/main/resources/updateRegexLink.sql new file mode 100644 index 0000000..e9e0fbe --- /dev/null +++ b/src/main/resources/updateRegexLink.sql @@ -0,0 +1 @@ +update ${databaseName}.transaction set regex_id = ? where id = ? \ No newline at end of file
SourceDescriptionDateAmount