personal/monzo-ingestion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rustfmt

Browse Source
This commit is contained in:
Joshua Coles 2024-04-02 22:00:15 +01:00
parent 89fccc8d85
commit 660b7b8676
7 changed files with 63 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/target
/.idea
client_secret.json

85
migration/src/m20230904_141851_create_monzo_tables.rs
View File

@ -6,54 +6,40 @@ pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager.create_table(
Table::create()
.table(Transaction::Table)
.if_not_exists()
.col(
ColumnDef::new(Transaction::Id)
.string()
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(Transaction::TransactionType)
.string()
.not_null(),
)
.col(
ColumnDef::new(Transaction::TotalAmount)
.decimal()
.not_null(),
)
.col(
ColumnDef::new(Transaction::Timestamp)
.timestamp()
.not_null(),
)
.col(
ColumnDef::new(Transaction::Title)
.string()
.not_null(),
)
.col(
ColumnDef::new(Transaction::Emoji)
.string(),
)
.col(
ColumnDef::new(Transaction::Notes)
.string(),
)
.col(
ColumnDef::new(Transaction::Receipt)
.string(),
)
.col(
ColumnDef::new(Transaction::Description)
.string(),
)
.to_owned()
).await?;
manager
.create_table(
Table::create()
.table(Transaction::Table)
.if_not_exists()
.col(
ColumnDef::new(Transaction::Id)
.string()
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(Transaction::TransactionType)
.string()
.not_null(),
)
.col(
ColumnDef::new(Transaction::TotalAmount)
.decimal()
.not_null(),
)
.col(
ColumnDef::new(Transaction::Timestamp)
.timestamp()
.not_null(),
)
.col(ColumnDef::new(Transaction::Title).string().not_null())
.col(ColumnDef::new(Transaction::Emoji).string())
.col(ColumnDef::new(Transaction::Notes).string())
.col(ColumnDef::new(Transaction::Receipt).string())
.col(ColumnDef::new(Transaction::Description).string())
.to_owned(),
)
.await?;
manager
.create_table(
@ -73,7 +59,8 @@ impl MigrationTrait for Migration {
)
.col(ColumnDef::new(Expenditure::Amount).decimal().not_null())
.to_owned(),
).await?;
)
.await?;
Ok(())
}

28
src/ingestion/db.rs
View File

@ -15,28 +15,24 @@ pub async fn insert(db: &DatabaseConnection, insertions: Vec<Insertion>) -> Resu
for insertions in insertions.chunks(400) {
let tx = db.begin().await?;
transaction::Entity::insert_many(
insertions.iter().map(|i| &i.transaction).cloned(),
)
.on_conflict(
OnConflict::column(transaction::Column::Id)
.update_columns(transaction::Column::iter())
.to_owned(),
)
.exec(&tx)
.await?;
transaction::Entity::insert_many(insertions.iter().map(|i| &i.transaction).cloned())
.on_conflict(
OnConflict::column(transaction::Column::Id)
.update_columns(transaction::Column::iter())
.to_owned(),
)
.exec(&tx)
.await?;
// Expenditures can change as we recagegorise them, so we delete all the old ones and insert
// an entirely new set to ensure we don't end up leaving old ones around.
expenditure::Entity::delete_many()
.filter(
expenditure::Column::TransactionId.is_in(
insertions
.iter()
.map(|i| i.transaction.id.as_ref()),
),
expenditure::Column::TransactionId
.is_in(insertions.iter().map(|i| i.transaction.id.as_ref())),
)
.exec(&tx).await?;
.exec(&tx)
.await?;
expenditure::Entity::insert_many(
insertions

4
src/ingestion/ingestion_logic.rs
View File

@ -1,13 +1,13 @@
use crate::ingestion::db::Insertion;
use anyhow::Context;
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime};
use csv::StringRecord;
use entity::expenditure::ActiveModel;
use entity::transaction;
use num_traits::FromPrimitive;
use sea_orm::prelude::Decimal;
use sea_orm::ActiveValue::*;
use sea_orm::IntoActiveModel;
use crate::ingestion::db::Insertion;
use csv::StringRecord;
#[allow(dead_code)]
mod headings {

2
src/ingestion/mod.rs
View File

@ -1,3 +1,3 @@
pub mod db;
pub mod routes;
pub mod ingestion_logic;
pub mod routes;

6
src/ingestion/routes.rs
View File

@ -1,11 +1,11 @@
use crate::error::AppError;
use crate::ingestion::db;
use crate::ingestion::ingestion_logic::{from_csv_row, from_json_row};
use anyhow::anyhow;
use axum::extract::{Extension, Json, Multipart};
use sea_orm::DatabaseConnection;
use serde_json::Value;
use std::io::Cursor;
use crate::error::AppError;
use crate::ingestion::db;
use crate::ingestion::ingestion_logic::{from_csv_row, from_json_row};
pub async fn monzo_updated(
Extension(db): Extension<DatabaseConnection>,

17
src/main.rs
View File

@ -1,14 +1,14 @@
mod ingestion;
mod error;
mod ingestion;
use axum::{Extension, Router};
use std::net::SocketAddr;
use axum::routing::{get, post};
use clap::Parser;
use sea_orm::{ConnectionTrait, DatabaseConnection};
use migration::{Migrator, MigratorTrait};
use crate::error::AppError;
use crate::ingestion::routes::{monzo_batched_csv, monzo_batched_json, monzo_updated};
use axum::routing::{get, post};
use axum::{Extension, Router};
use clap::Parser;
use migration::{Migrator, MigratorTrait};
use sea_orm::{ConnectionTrait, DatabaseConnection};
use std::net::SocketAddr;
#[derive(Debug, clap::Parser)]
struct Config {
@ -28,8 +28,7 @@ struct Config {
async fn health_check(
Extension(db): Extension<DatabaseConnection>,
) -> Result<&'static str, AppError> {
db.execute_unprepared("SELECT 1")
.await?;
db.execute_unprepared("SELECT 1").await?;
Ok("Ok")
}