Add monzo-row parsing testcases

entity/src/transaction.rs

@@ -11,7 +11,7 @@ pub struct Model {
     pub transaction_type: String,
     pub total_amount: Decimal,
     pub timestamp: DateTime,
-    pub title: String,
+    pub title: Option<String>,
     pub emoji: Option<String>,
     pub notes: Option<String>,
     pub receipt: Option<String>,

fixtures/transactions.csv

@@ -0,0 +1,3 @@
+Transaction ID,Date,Time,Type,Name,Emoji,Category,Amount,Currency,Local amount,Local currency,Notes and #tags,Address,Receipt,Description,Category split
+tx_0000AhmcbFKVlkAFbeZuAk,11/05/2024,13:41:47,Pot transfer,Savings Pot,,Savings,-1.4,GBP,-1.4,GBP,,,,Round up,
+tx_0000AhmgdZ0FzkVZou77Ev,11/05/2024,14:27:01,Card payment,D,💊,C,-100.42,GBP,-100.42,GBP,,B,,A,"Groceries:15.00,Personal care:10.00"

fixtures/transactions.json

@@ -0,0 +1,38 @@
+[
+  [
+    "tx_0000AhmcbFKVlkAFbeZuAk",
+    "2024-05-11T00:00:00.000Z",
+    "1899-12-30T13:41:47.000Z",
+    "Pot transfer",
+    "Savings Pot",
+    "",
+    "Savings",
+    -1.4,
+    "GBP",
+    -1.4,
+    "GBP",
+    "",
+    "",
+    "",
+    "Round up",
+    ""
+  ],
+  [
+    "tx_0000AhmgdZ0FzkVZou77Ev",
+    "2024-05-11T00:00:00.000Z",
+    "1899-12-30T14:27:01.000Z",
+    "Card payment",
+    "D",
+    "💊",
+    "C",
+    -100.42,
+    "GBP",
+    -100.42,
+    "GBP",
+    "",
+    "B",
+    "",
+    "A",
+    "Groceries:15.00,Personal care:10.00"
+  ]
+]

justfile

@@ -0,0 +1,12 @@
+#!/usr/bin/env just --justfile
+
+generate-migration:
+     sea-orm-cli migrate generate $1
+
+migrate:
+    sea-orm-cli migrate up
+
+generate-entities:
+    sea-orm-cli generate entity --with-serde both \
+                                -l -o entity/src \
+                                --ignore-tables monzo_ingestion_seaql_migrations

migration/src/lib.rs

@@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*;
 
 pub mod m20230904_141851_create_monzo_tables;
 mod m20240529_195030_add_transaction_identity_hash;
+mod m20240603_162500_make_title_optional;
 
 pub struct Migrator;
 
@@ -15,6 +16,7 @@ impl MigratorTrait for Migrator {
         vec![
             Box::new(m20230904_141851_create_monzo_tables::Migration),
             Box::new(m20240529_195030_add_transaction_identity_hash::Migration),
+            Box::new(m20240603_162500_make_title_optional::Migration),
         ]
     }
 }

migration/src/m20240603_162500_make_title_optional.rs

@@ -0,0 +1,47 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager.alter_table(
+            TableAlterStatement::new()
+                .table(Transaction::Table)
+                .modify_column(ColumnDef::new(Transaction::Title).string().null())
+                .to_owned()
+        ).await?;
+
+        // Set all empty string titles to null
+        manager.get_connection().execute_unprepared(r#"
+            update transaction
+            set title = null
+            where title = ''
+        "#).await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // Set all null titles to empty string when reverting
+        manager.get_connection().execute_unprepared(r#"
+            update transaction
+            set title = ''
+            where title is null
+        "#).await?;
+
+        manager.alter_table(
+            TableAlterStatement::new()
+                .table(Transaction::Table)
+                .modify_column(ColumnDef::new(Transaction::Title).string().not_null())
+                .to_owned()
+        ).await
+    }
+}
+
+#[derive(DeriveIden)]
+enum Transaction {
+    Table,
+    Title,
+}

src/ingestion/ingestion_logic.rs

@@ -40,7 +40,7 @@ pub struct MonzoRow {
     emoji: Option<String>,
     description: Option<String>,
     transaction_type: String,
-    title: String,
+    title: Option<String>,
     timestamp: NaiveDateTime,
     transaction_id: String,
 }
@@ -153,7 +153,7 @@ pub fn from_json_row(row: Vec<Value>) -> anyhow::Result<MonzoRow> {
 
     Ok(MonzoRow {
         transaction_id: json_required_str(&row[headings::TRANSACTION_ID], "Transaction ID")?,
-        title: json_required_str(&row[headings::NAME], "Title")?,
+        title: json_opt(&row[headings::NAME]),
         transaction_type: json_required_str(&row[headings::TYPE], "Transaction type")?,
         description: json_opt(&row[headings::DESCRIPTION]),
         emoji: json_opt(&row[headings::EMOJI]),
@@ -166,6 +166,32 @@ pub fn from_json_row(row: Vec<Value>) -> anyhow::Result<MonzoRow> {
     })
 }
 
+#[test]
+fn test_json() {
+    let json = include_str!("../../fixtures/transactions.json");
+    let csv = include_str!("../../fixtures/transactions.csv");
+
+    let json: Vec<Vec<Value>> = serde_json::from_str(json).unwrap();
+    let mut csv_reader = csv::Reader::from_reader(csv.as_bytes());
+
+    let json_rows = json.iter()
+        .map(|row| from_json_row(row.clone()))
+        .collect::<Result<Vec<_>, anyhow::Error>>()
+        .unwrap();
+
+    let csv_rows = csv_reader.records()
+        .map(|record| from_csv_row(record.unwrap()))
+        .collect::<Result<Vec<_>, anyhow::Error>>()
+        .unwrap();
+
+    assert_eq!(csv_rows.len(), json_rows.len(), "Different number of rows");
+
+    for (i, (json_row, csv_row)) in json_rows.iter().zip(csv_rows.iter()).enumerate() {
+        assert_eq!(json_row, csv_row, "Row {} is different", i);
+        assert_eq!(json_row.compute_hash(), csv_row.compute_hash(), "Row {} hash are different", i);
+    }
+}
+
 fn csv_opt(s: &str) -> Option<String> {
     match s {
         "" => None,
@@ -185,7 +211,7 @@ pub fn from_csv_row(row: StringRecord) -> anyhow::Result<MonzoRow> {
     Ok(MonzoRow {
         timestamp,
         transaction_id: row[headings::TRANSACTION_ID].to_string(),
-        title: row[headings::NAME].to_string(),
+        title: csv_opt(&row[headings::NAME]),
         transaction_type: row[headings::TYPE].to_string(),
         description: csv_opt(&row[headings::DESCRIPTION]),
         emoji: csv_opt(&row[headings::EMOJI]),