Import of old stuff, it might work now

src/error.rs

@@ -0,0 +1,30 @@
+use axum::response::{IntoResponse, Response};
+use http::StatusCode;
+use sea_orm::DbErr;
+use tracing::log::error;
+
+#[derive(thiserror::Error, Debug)]
+pub enum AppError {
+    /// SeaORM error, separated for ease of use allowing us to `?` db operations.
+    #[error("Internal error")]
+    DbError(#[from] DbErr),
+
+    #[error("Invalid request {0}")]
+    BadRequest(anyhow::Error),
+
+    /// Catch all for error we dont care to expose publicly.
+    #[error("Internal error")]
+    Anyhow(#[from] anyhow::Error),
+}
+
+impl IntoResponse for AppError {
+    fn into_response(self) -> Response {
+        error!("Internal server error: {self:?}");
+
+        let status_code = match self {
+            _ => StatusCode::INTERNAL_SERVER_ERROR,
+        };
+
+        (status_code, self.to_string()).into_response()
+    }
+}

src/ingestion/db.rs

@@ -0,0 +1,62 @@
+use entity::{expenditure, transaction};
+use sea_orm::sea_query::OnConflict;
+use sea_orm::QueryFilter;
+use sea_orm::{ColumnTrait, DatabaseConnection, DbErr, EntityTrait, Iterable, TransactionTrait};
+
+pub struct Insertion {
+    pub transaction: transaction::ActiveModel,
+    pub contained_expenditures: Vec<expenditure::ActiveModel>,
+}
+
+// Note while this is more efficient in db calls, it does bind together the entire group.
+// We employ a batching process for now to try balance speed and failure rate but it is worth trying
+// to move failures earlier and improve reporting.
+pub async fn insert(db: &DatabaseConnection, insertions: Vec<Insertion>) -> Result<(), DbErr> {
+    for insertions in insertions.chunks(400) {
+        let tx = db.begin().await?;
+
+        transaction::Entity::insert_many(
+            insertions.iter().map(|i| &i.transaction).cloned(),
+        )
+        .on_conflict(
+            OnConflict::column(transaction::Column::Id)
+                .update_columns(transaction::Column::iter())
+                .to_owned(),
+        )
+        .exec(&tx)
+        .await?;
+
+        // Expenditures can change as we recagegorise them, so we delete all the old ones and insert
+        // an entirely new set to ensure we don't end up leaving old ones around.
+        expenditure::Entity::delete_many()
+            .filter(
+                expenditure::Column::TransactionId.is_in(
+                    insertions
+                        .iter()
+                        .map(|i| i.transaction.id.as_ref()),
+                ),
+            )
+            .exec(&tx).await?;
+
+        expenditure::Entity::insert_many(
+            insertions
+                .iter()
+                .flat_map(|i| &i.contained_expenditures)
+                .cloned(),
+        )
+        .on_conflict(
+            OnConflict::columns(vec![
+                expenditure::Column::TransactionId,
+                expenditure::Column::Category,
+            ])
+            .update_columns(expenditure::Column::iter())
+            .to_owned(),
+        )
+        .exec(&tx)
+        .await?;
+
+        tx.commit().await?;
+    }
+
+    Ok(())
+}

src/ingestion/ingestion.rs

@@ -0,0 +1,67 @@
+use anyhow::anyhow;
+use axum::extract::{Extension, Json, Multipart};
+use sea_orm::DatabaseConnection;
+use serde_json::Value;
+use std::io::Cursor;
+use crate::error::AppError;
+use crate::ingestion::db;
+use crate::ingestion::ingestion_logic::{from_csv_row, from_json_row};
+
+pub async fn monzo_updated(
+    Extension(db): Extension<DatabaseConnection>,
+    Json(row): Json<Vec<Value>>,
+) -> Result<&'static str, AppError> {
+    db::insert(&db, vec![from_json_row(row)?]).await.unwrap();
+
+    Ok("Ok")
+}
+
+pub async fn monzo_batched_json(
+    Extension(db): Extension<DatabaseConnection>,
+    Json(data): Json<Vec<Vec<Value>>>,
+) -> Result<&'static str, AppError> {
+    let insertions = data
+        .into_iter()
+        .skip(1)
+        .map(|row| from_json_row(row))
+        .collect::<Result<_, _>>()?;
+
+    db::insert(&db, insertions).await.unwrap();
+
+    Ok("Ok")
+}
+
+pub async fn monzo_batched_csv(
+    Extension(db): Extension<DatabaseConnection>,
+    mut multipart: Multipart,
+) -> Result<&'static str, AppError> {
+    let csv = loop {
+        match multipart.next_field().await.unwrap() {
+            Some(field) if field.name() == Some("csv") => {
+                break Some(field.bytes().await.unwrap());
+            }
+
+            Some(_) => {}
+            None => break None,
+        }
+    };
+
+    let Some(csv) = csv else {
+        return Err(AppError::BadRequest(anyhow!("No CSV file provided")));
+    };
+
+    let csv = Cursor::new(csv);
+    let mut csv = csv::Reader::from_reader(csv);
+    let data = csv.records();
+
+    db::insert(
+        &db,
+        data.filter_map(|f| f.ok())
+            .map(from_csv_row)
+            .collect::<Result<_, _>>()?,
+    )
+    .await
+    .unwrap();
+
+    Ok("Ok")
+}

src/ingestion/ingestion_logic.rs

@@ -0,0 +1,182 @@
+use anyhow::Context;
+use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime};
+use entity::expenditure::ActiveModel;
+use entity::transaction;
+use num_traits::FromPrimitive;
+use sea_orm::prelude::Decimal;
+use sea_orm::ActiveValue::*;
+use sea_orm::IntoActiveModel;
+use crate::ingestion::db::Insertion;
+use csv::StringRecord;
+
+#[allow(dead_code)]
+mod headings {
+    pub const TRANSACTION_ID: usize = 0;
+    pub const DATE: usize = 1;
+    pub const TIME: usize = 2;
+    pub const TYPE: usize = 3;
+    pub const NAME: usize = 4;
+    pub const EMOJI: usize = 5;
+    pub const CATEGORY: usize = 6;
+    pub const AMOUNT: usize = 7;
+    pub const CURRENCY: usize = 8;
+    pub const LOCAL_AMOUNT: usize = 9;
+    pub const LOCAL_CURRENCY: usize = 10;
+    pub const NOTES_AND_TAGS: usize = 11;
+    pub const ADDRESS: usize = 12;
+    pub const RECEIPT: usize = 13;
+    pub const DESCRIPTION: usize = 14;
+    pub const CATEGORY_SPLIT: usize = 15;
+}
+
+fn parse_section(monzo_transaction_id: &str, section: &str) -> anyhow::Result<ActiveModel> {
+    let mut components = section.split(':');
+    let category: String = components
+        .next()
+        .context("Missing Missing category")?
+        .to_string();
+
+    let amount = components
+        .next()
+        .context("Missing amount")?
+        .parse::<Decimal>()?;
+
+    Ok(entity::expenditure::Model {
+        transaction_id: monzo_transaction_id.to_string(),
+        category,
+        amount,
+    }
+    .into_active_model())
+}
+
+fn json_opt(value: &serde_json::Value) -> Option<String> {
+    match value {
+        serde_json::Value::String(string) if string.is_empty() => None,
+        serde_json::Value::String(string) => Some(string.to_string()),
+        _ => None,
+    }
+}
+
+pub fn from_json_row(row: Vec<serde_json::Value>) -> anyhow::Result<Insertion> {
+    use serde_json::Value;
+    let monzo_transaction_id = row[headings::TRANSACTION_ID]
+        .as_str()
+        .context("No transaction id")?
+        .to_string();
+
+    let date = DateTime::parse_from_rfc3339(row[headings::DATE].as_str().context("No date")?)
+        .context("Failed to parse date")?;
+
+    let time = DateTime::parse_from_rfc3339(row[headings::TIME].as_str().context("No time")?)
+        .context("Failed to parse date")?
+        .time();
+
+    let timestamp = date.date_naive().and_time(time);
+
+    let title = row[headings::NAME]
+        .as_str()
+        .context("No title")?
+        .to_string();
+
+    let monzo_transaction_type = row[headings::TYPE]
+        .as_str()
+        .context("No transaction type")?
+        .to_string();
+
+    let description = json_opt(&row[headings::DESCRIPTION]);
+    let emoji = json_opt(&row[headings::EMOJI]);
+    let notes = json_opt(&row[headings::NOTES_AND_TAGS]);
+    let receipt = json_opt(&row[headings::RECEIPT]);
+    let total_amount = Decimal::from_f64(row[headings::AMOUNT].as_f64().context("No amount")?)
+        .context("Failed to parse date")?;
+
+    let expenditures: Vec<_> = match row.get(headings::CATEGORY_SPLIT) {
+        Some(Value::String(split)) if !split.is_empty() => split
+            .split(',')
+            .map(|section| parse_section(&monzo_transaction_id, section))
+            .collect::<Result<Vec<_>, anyhow::Error>>()?,
+
+        _ => vec![entity::expenditure::Model {
+            category: row[headings::CATEGORY]
+                .as_str()
+                .context("No context")?
+                .to_string(),
+            amount: total_amount,
+            transaction_id: monzo_transaction_id.clone(),
+        }
+        .into_active_model()],
+    };
+
+    Ok(Insertion {
+        transaction: transaction::ActiveModel {
+            id: Set(monzo_transaction_id),
+            transaction_type: Set(monzo_transaction_type),
+            timestamp: Set(timestamp),
+            title: Set(title),
+            emoji: Set(emoji),
+            notes: Set(notes),
+            receipt: Set(receipt),
+            total_amount: Set(total_amount),
+            description: Set(description),
+        },
+
+        contained_expenditures: expenditures,
+    })
+}
+
+fn csv_opt(s: &str) -> Option<String> {
+    match s {
+        "" => None,
+        v => Some(v.to_string()),
+    }
+}
+
+pub fn from_csv_row(row: StringRecord) -> anyhow::Result<Insertion> {
+    let monzo_transaction_id = row[headings::TRANSACTION_ID].to_string();
+
+    let date = NaiveDate::parse_from_str(&row[headings::DATE], "%d/%m/%Y")
+        .context("Failed to parse date from csv")?;
+
+    let time = NaiveTime::parse_from_str(&row[headings::TIME], "%H:%M:%S")
+        .context("Failed to parse time from csv")?;
+
+    let timestamp = NaiveDateTime::new(date, time);
+
+    let title = row[headings::NAME].to_string();
+    let monzo_transaction_type = row[headings::TYPE].to_string();
+
+    let description = csv_opt(&row[headings::DESCRIPTION]);
+    let emoji = csv_opt(&row[headings::EMOJI]);
+    let notes = csv_opt(&row[headings::NOTES_AND_TAGS]);
+    let receipt = csv_opt(&row[headings::RECEIPT]);
+    let total_amount = row[headings::AMOUNT].parse::<Decimal>()?;
+
+    let expenditures: Vec<_> = match row.get(headings::CATEGORY_SPLIT) {
+        Some(split) if !split.is_empty() => split
+            .split(',')
+            .map(|section| parse_section(&monzo_transaction_id, section))
+            .collect::<Result<Vec<_>, anyhow::Error>>()?,
+
+        _ => vec![entity::expenditure::Model {
+            transaction_id: monzo_transaction_id.clone(),
+            category: row[headings::CATEGORY].to_string(),
+            amount: total_amount,
+        }
+        .into_active_model()],
+    };
+
+    Ok(Insertion {
+        transaction: transaction::ActiveModel {
+            id: Set(monzo_transaction_id),
+            transaction_type: Set(monzo_transaction_type),
+            timestamp: Set(timestamp),
+            title: Set(title),
+            emoji: Set(emoji),
+            notes: Set(notes),
+            receipt: Set(receipt),
+            total_amount: Set(total_amount),
+            description: Set(description),
+        },
+        contained_expenditures: expenditures,
+    })
+}

src/ingestion/mod.rs

@@ -0,0 +1,3 @@
+pub mod db;
+pub mod ingestion;
+pub mod ingestion_logic;

src/main.rs

@@ -1,30 +1,39 @@
-use axum::{
-    routing::{get, post},
-    http::StatusCode,
-    response::IntoResponse,
-    Json, Router,
-};
-use serde::{Deserialize, Serialize};
+mod ingestion;
+mod error;
+
+use axum::{Extension, Router};
 use std::net::SocketAddr;
+use axum::routing::post;
+use clap::Parser;
+use migration::{Migrator, MigratorTrait};
+use crate::ingestion::ingestion::{monzo_batched_csv, monzo_batched_json, monzo_updated};
+
+#[derive(Debug, clap::Parser)]
+struct Config {
+    #[clap(short, long, env)]
+    addr: SocketAddr,
+    #[clap(short, long = "db", env)]
+    database_url: String,
+}
 
 #[tokio::main]
-async fn main() {
-    // initialize tracing
+async fn main() -> anyhow::Result<()> {
+    let config: Config = Config::parse();
+    let connection = sea_orm::Database::connect(&config.database_url).await?;
+    Migrator::up(&connection, None).await?;
+
     tracing_subscriber::fmt::init();
-
-    // build our application with a route
     let app = Router::new()
-        // `GET /` goes to `root`
-        .route("/", get(root))
-        // `POST /users` goes to `create_user`
-        .route("/users", post(create_user));
+        .route("/monzo-updated", post(monzo_updated))
+        .route("/monzo-batch-export", post(monzo_batched_json))
+        .route("/monzo-csv-ingestion", post(monzo_batched_csv))
+        .layer(Extension(connection.clone()));
 
-    // run our app with hyper
-    // `axum::Server` is a re-export of `hyper::Server`
-    let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
-    tracing::debug!("listening on {}", addr);
-    axum::Server::bind(&addr)
+    tracing::debug!("listening on {}", &config.addr);
+    axum::Server::bind(&config.addr)
         .serve(app.into_make_service())
         .await
         .unwrap();
-}
+
+    Ok(())
+}