personal/monzo-ingestion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rustfmt

Browse Source
This commit is contained in:
Joshua Coles 2024-04-02 22:00:15 +01:00
parent 89fccc8d85
commit 660b7b8676
7 changed files with 63 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/target /target
/.idea /.idea
client_secret.json

85
migration/src/m20230904_141851_create_monzo_tables.rs
View File

@ -6,54 +6,40 @@ pub struct Migration;
#[async_trait::async_trait] #[async_trait::async_trait]
impl MigrationTrait for Migration { impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager.create_table( manager
Table::create() .create_table(
.table(Transaction::Table) Table::create()
.if_not_exists() .table(Transaction::Table)
.col( .if_not_exists()
ColumnDef::new(Transaction::Id) .col(
.string() ColumnDef::new(Transaction::Id)
.not_null() .string()
.primary_key(), .not_null()
) .primary_key(),
.col( )
ColumnDef::new(Transaction::TransactionType) .col(
.string() ColumnDef::new(Transaction::TransactionType)
.not_null(), .string()
) .not_null(),
.col( )
ColumnDef::new(Transaction::TotalAmount) .col(
.decimal() ColumnDef::new(Transaction::TotalAmount)
.not_null(), .decimal()
) .not_null(),
.col( )
ColumnDef::new(Transaction::Timestamp) .col(
.timestamp() ColumnDef::new(Transaction::Timestamp)
.not_null(), .timestamp()
) .not_null(),
.col( )
ColumnDef::new(Transaction::Title) .col(ColumnDef::new(Transaction::Title).string().not_null())
.string() .col(ColumnDef::new(Transaction::Emoji).string())
.not_null(), .col(ColumnDef::new(Transaction::Notes).string())
) .col(ColumnDef::new(Transaction::Receipt).string())
.col( .col(ColumnDef::new(Transaction::Description).string())
ColumnDef::new(Transaction::Emoji) .to_owned(),
.string(), )
) .await?;
.col(
ColumnDef::new(Transaction::Notes)
.string(),
)
.col(
ColumnDef::new(Transaction::Receipt)
.string(),
)
.col(
ColumnDef::new(Transaction::Description)
.string(),
)
.to_owned()
).await?;
manager manager
.create_table( .create_table(
@ -73,7 +59,8 @@ impl MigrationTrait for Migration {
) )
.col(ColumnDef::new(Expenditure::Amount).decimal().not_null()) .col(ColumnDef::new(Expenditure::Amount).decimal().not_null())
.to_owned(), .to_owned(),
).await?; )
.await?;
Ok(()) Ok(())
} }

28
src/ingestion/db.rs
View File

@ -15,28 +15,24 @@ pub async fn insert(db: &DatabaseConnection, insertions: Vec<Insertion>) -> Resu
for insertions in insertions.chunks(400) { for insertions in insertions.chunks(400) {
let tx = db.begin().await?; let tx = db.begin().await?;
transaction::Entity::insert_many( transaction::Entity::insert_many(insertions.iter().map(|i| &i.transaction).cloned())
insertions.iter().map(|i| &i.transaction).cloned(), .on_conflict(
) OnConflict::column(transaction::Column::Id)
.on_conflict( .update_columns(transaction::Column::iter())
OnConflict::column(transaction::Column::Id) .to_owned(),
.update_columns(transaction::Column::iter()) )
.to_owned(), .exec(&tx)
) .await?;
.exec(&tx)
.await?;
// Expenditures can change as we recagegorise them, so we delete all the old ones and insert // Expenditures can change as we recagegorise them, so we delete all the old ones and insert
// an entirely new set to ensure we don't end up leaving old ones around. // an entirely new set to ensure we don't end up leaving old ones around.
expenditure::Entity::delete_many() expenditure::Entity::delete_many()
.filter( .filter(
expenditure::Column::TransactionId.is_in( expenditure::Column::TransactionId
insertions .is_in(insertions.iter().map(|i| i.transaction.id.as_ref())),
.iter()
.map(|i| i.transaction.id.as_ref()),
),
) )
.exec(&tx).await?; .exec(&tx)
.await?;
expenditure::Entity::insert_many( expenditure::Entity::insert_many(
insertions insertions

4
src/ingestion/ingestion_logic.rs
View File

@ -1,13 +1,13 @@
use crate::ingestion::db::Insertion;
use anyhow::Context; use anyhow::Context;
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime};
use csv::StringRecord;
use entity::expenditure::ActiveModel; use entity::expenditure::ActiveModel;
use entity::transaction; use entity::transaction;
use num_traits::FromPrimitive; use num_traits::FromPrimitive;
use sea_orm::prelude::Decimal; use sea_orm::prelude::Decimal;
use sea_orm::ActiveValue::*; use sea_orm::ActiveValue::*;
use sea_orm::IntoActiveModel; use sea_orm::IntoActiveModel;
use crate::ingestion::db::Insertion;
use csv::StringRecord;
#[allow(dead_code)] #[allow(dead_code)]
mod headings { mod headings {

2
src/ingestion/mod.rs
View File

@ -1,3 +1,3 @@
pub mod db; pub mod db;
pub mod routes;
pub mod ingestion_logic; pub mod ingestion_logic;
pub mod routes;

6
src/ingestion/routes.rs
View File

@ -1,11 +1,11 @@
use crate::error::AppError;
use crate::ingestion::db;
use crate::ingestion::ingestion_logic::{from_csv_row, from_json_row};
use anyhow::anyhow; use anyhow::anyhow;
use axum::extract::{Extension, Json, Multipart}; use axum::extract::{Extension, Json, Multipart};
use sea_orm::DatabaseConnection; use sea_orm::DatabaseConnection;
use serde_json::Value; use serde_json::Value;
use std::io::Cursor; use std::io::Cursor;
use crate::error::AppError;
use crate::ingestion::db;
use crate::ingestion::ingestion_logic::{from_csv_row, from_json_row};
pub async fn monzo_updated( pub async fn monzo_updated(
Extension(db): Extension<DatabaseConnection>, Extension(db): Extension<DatabaseConnection>,

17
src/main.rs
View File

@ -1,14 +1,14 @@
mod ingestion;
mod error; mod error;
mod ingestion;
use axum::{Extension, Router};
use std::net::SocketAddr;
use axum::routing::{get, post};
use clap::Parser;
use sea_orm::{ConnectionTrait, DatabaseConnection};
use migration::{Migrator, MigratorTrait};
use crate::error::AppError; use crate::error::AppError;
use crate::ingestion::routes::{monzo_batched_csv, monzo_batched_json, monzo_updated}; use crate::ingestion::routes::{monzo_batched_csv, monzo_batched_json, monzo_updated};
use axum::routing::{get, post};
use axum::{Extension, Router};
use clap::Parser;
use migration::{Migrator, MigratorTrait};
use sea_orm::{ConnectionTrait, DatabaseConnection};
use std::net::SocketAddr;
#[derive(Debug, clap::Parser)] #[derive(Debug, clap::Parser)]
struct Config { struct Config {
@ -28,8 +28,7 @@ struct Config {
async fn health_check( async fn health_check(
Extension(db): Extension<DatabaseConnection>, Extension(db): Extension<DatabaseConnection>,
) -> Result<&'static str, AppError> { ) -> Result<&'static str, AppError> {
db.execute_unprepared("SELECT 1") db.execute_unprepared("SELECT 1").await?;
.await?;
Ok("Ok") Ok("Ok")
} }