entity/src/transaction.rs

@@ -11,7 +11,7 @@ pub struct Model {
     pub transaction_type: String,
     pub total_amount: Decimal,
     pub timestamp: DateTime,
-    pub title: Option<String>,
+    pub title: String,
     pub emoji: Option<String>,
     pub notes: Option<String>,
     pub receipt: Option<String>,

fixtures/transactions.csv

@@ -1,3 +0,0 @@
-Transaction ID,Date,Time,Type,Name,Emoji,Category,Amount,Currency,Local amount,Local currency,Notes and #tags,Address,Receipt,Description,Category split
-tx_0000AhmcbFKVlkAFbeZuAk,11/05/2024,13:41:47,Pot transfer,Savings Pot,,Savings,-1.4,GBP,-1.4,GBP,,,,Round up,
-tx_0000AhmgdZ0FzkVZou77Ev,11/05/2024,14:27:01,Card payment,D,💊,C,-100.42,GBP,-100.42,GBP,,B,,A,"Groceries:15.00,Personal care:10.00"

fixtures/transactions.json

@@ -1,38 +0,0 @@
-[
-  [
-    "tx_0000AhmcbFKVlkAFbeZuAk",
-    "2024-05-11T00:00:00.000Z",
-    "1899-12-30T13:41:47.000Z",
-    "Pot transfer",
-    "Savings Pot",
-    "",
-    "Savings",
-    -1.4,
-    "GBP",
-    -1.4,
-    "GBP",
-    "",
-    "",
-    "",
-    "Round up",
-    ""
-  ],
-  [
-    "tx_0000AhmgdZ0FzkVZou77Ev",
-    "2024-05-11T00:00:00.000Z",
-    "1899-12-30T14:27:01.000Z",
-    "Card payment",
-    "D",
-    "💊",
-    "C",
-    -100.42,
-    "GBP",
-    -100.42,
-    "GBP",
-    "",
-    "B",
-    "",
-    "A",
-    "Groceries:15.00,Personal care:10.00"
-  ]
-]

justfile

@@ -1,12 +0,0 @@
-#!/usr/bin/env just --justfile
-
-generate-migration:
-     sea-orm-cli migrate generate $1
-
-migrate:
-    sea-orm-cli migrate up
-
-generate-entities:
-    sea-orm-cli generate entity --with-serde both \
-                                -l -o entity/src \
-                                --ignore-tables monzo_ingestion_seaql_migrations

migration/src/lib.rs

@@ -2,7 +2,6 @@ pub use sea_orm_migration::prelude::*;
 
 pub mod m20230904_141851_create_monzo_tables;
 mod m20240529_195030_add_transaction_identity_hash;
-mod m20240603_162500_make_title_optional;
 
 pub struct Migrator;
 
@@ -16,7 +15,6 @@ impl MigratorTrait for Migrator {
         vec![
             Box::new(m20230904_141851_create_monzo_tables::Migration),
             Box::new(m20240529_195030_add_transaction_identity_hash::Migration),
-            Box::new(m20240603_162500_make_title_optional::Migration),
         ]
     }
 }

migration/src/m20240603_162500_make_title_optional.rs

@@ -1,47 +0,0 @@
-use sea_orm_migration::prelude::*;
-
-#[derive(DeriveMigrationName)]
-pub struct Migration;
-
-#[async_trait::async_trait]
-impl MigrationTrait for Migration {
-    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
-        manager.alter_table(
-            TableAlterStatement::new()
-                .table(Transaction::Table)
-                .modify_column(ColumnDef::new(Transaction::Title).string().null())
-                .to_owned()
-        ).await?;
-
-        // Set all empty string titles to null
-        manager.get_connection().execute_unprepared(r#"
-            update transaction
-            set title = null
-            where title = ''
-        "#).await?;
-
-        Ok(())
-    }
-
-    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
-        // Set all null titles to empty string when reverting
-        manager.get_connection().execute_unprepared(r#"
-            update transaction
-            set title = ''
-            where title is null
-        "#).await?;
-
-        manager.alter_table(
-            TableAlterStatement::new()
-                .table(Transaction::Table)
-                .modify_column(ColumnDef::new(Transaction::Title).string().not_null())
-                .to_owned()
-        ).await
-    }
-}
-
-#[derive(DeriveIden)]
-enum Transaction {
-    Table,
-    Title,
-}

src/ingestion/ingestion_logic.rs

@@ -40,7 +40,7 @@ pub struct MonzoRow {
     emoji: Option<String>,
     description: Option<String>,
     transaction_type: String,
-    title: Option<String>,
+    title: String,
     timestamp: NaiveDateTime,
     transaction_id: String,
 }
@@ -153,7 +153,7 @@ pub fn from_json_row(row: Vec<Value>) -> anyhow::Result<MonzoRow> {
 
     Ok(MonzoRow {
         transaction_id: json_required_str(&row[headings::TRANSACTION_ID], "Transaction ID")?,
-        title: json_opt(&row[headings::NAME]),
+        title: json_required_str(&row[headings::NAME], "Title")?,
         transaction_type: json_required_str(&row[headings::TYPE], "Transaction type")?,
         description: json_opt(&row[headings::DESCRIPTION]),
         emoji: json_opt(&row[headings::EMOJI]),
@@ -166,32 +166,6 @@ pub fn from_json_row(row: Vec<Value>) -> anyhow::Result<MonzoRow> {
     })
 }
 
-#[test]
-fn test_json() {
-    let json = include_str!("../../fixtures/transactions.json");
-    let csv = include_str!("../../fixtures/transactions.csv");
-
-    let json: Vec<Vec<Value>> = serde_json::from_str(json).unwrap();
-    let mut csv_reader = csv::Reader::from_reader(csv.as_bytes());
-
-    let json_rows = json.iter()
-        .map(|row| from_json_row(row.clone()))
-        .collect::<Result<Vec<_>, anyhow::Error>>()
-        .unwrap();
-
-    let csv_rows = csv_reader.records()
-        .map(|record| from_csv_row(record.unwrap()))
-        .collect::<Result<Vec<_>, anyhow::Error>>()
-        .unwrap();
-
-    assert_eq!(csv_rows.len(), json_rows.len(), "Different number of rows");
-
-    for (i, (json_row, csv_row)) in json_rows.iter().zip(csv_rows.iter()).enumerate() {
-        assert_eq!(json_row, csv_row, "Row {} is different", i);
-        assert_eq!(json_row.compute_hash(), csv_row.compute_hash(), "Row {} hash are different", i);
-    }
-}
-
 fn csv_opt(s: &str) -> Option<String> {
     match s {
         "" => None,
@@ -211,7 +185,7 @@ pub fn from_csv_row(row: StringRecord) -> anyhow::Result<MonzoRow> {
     Ok(MonzoRow {
         timestamp,
         transaction_id: row[headings::TRANSACTION_ID].to_string(),
-        title: csv_opt(&row[headings::NAME]),
+        title: row[headings::NAME].to_string(),
         transaction_type: row[headings::TYPE].to_string(),
         description: csv_opt(&row[headings::DESCRIPTION]),
         emoji: csv_opt(&row[headings::EMOJI]),