Investigate cargo-chef to cache things

.github/workflows/build.yml

@@ -10,17 +10,11 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: catthehacker/ubuntu:act-latest
-    env:
-      RUNNER_TOOL_CACHE: /toolcache
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
-# Note to self: maybe look at this if we want to build outside docker?
-#      - name: Build with cache
-#        uses: Swatinem/rust-cache@v2
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
@@ -36,6 +30,7 @@ jobs:
         with:
           context: .
           push: true
+          file: ./Dockerfile.cache
           tags: git.joshuacoles.me/personal/monzo-ingestion:latest
           cache-from: type=registry,ref=user/app:latest
           cache-to: type=inline

Dockerfile

@@ -75,4 +75,4 @@ HEALTHCHECK --interval=5s --timeout=3s --retries=3 \
     CMD curl -f http://localhost:3000/health || exit 1
 
 # What the container should run when it is started.
-CMD ["/bin/server"]
+CMD ["/bin/server", "web", "--addr", "0.0.0.0:3000"]

Dockerfile.cache

@@ -0,0 +1,58 @@
+# Stage 1: Build
+ARG RUST_VERSION=1.76.0
+FROM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION} as chef
+WORKDIR /build/
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    lld \
+    clang \
+    libclang-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM chef as planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef as builder
+COPY --from=planner /build/recipe.json recipe.json
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release -p monzo-ingestion --recipe-path recipe.json
+# Build application
+COPY . .
+RUN cargo build --release -p monzo-ingestion
+
+# Stage 2: Run
+FROM debian:bullseye-slim AS final
+
+RUN set -ex; \
+    apt-get update && \
+    apt-get -y install --no-install-recommends \
+        ca-certificates curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+USER appuser
+
+# Copy the executable from the "build" stage.
+COPY --from=builder /build/target/release/monzo-ingestion /bin/
+
+# Expose the port that the application listens on.
+EXPOSE 3000
+
+HEALTHCHECK --interval=5s --timeout=3s --retries=3 \
+    CMD curl -f http://localhost:3000/health || exit 1
+
+# What the container should run when it is started.
+CMD ["/bin/server"]

src/main.rs

@@ -2,29 +2,56 @@ mod error;
 mod ingestion;
 
 use crate::error::AppError;
+use crate::ingestion::db;
+use crate::ingestion::ingestion_logic::from_csv_row;
 use crate::ingestion::routes::{monzo_batched_csv, monzo_batched_json};
 use axum::routing::{get, post};
 use axum::{Extension, Router};
-use clap::Parser;
+use clap::{Parser, Subcommand};
 use migration::{Migrator, MigratorTrait};
 use sea_orm::{ConnectionTrait, DatabaseConnection};
+use std::fs::File;
 use std::net::SocketAddr;
+use std::path::PathBuf;
 use tower_http::trace::TraceLayer;
 use tracing::log::LevelFilter;
 
+#[derive(Debug, Subcommand)]
+enum Commands {
+    Migrate {
+        /// Number of migration steps to perform. If not provided, all migrations will be run.
+        #[arg(long)]
+        steps: Option<u32>,
+
+        /// If we should perform migration down.
+        #[arg(long)]
+        down: bool,
+    },
+
+    Run {
+        /// If we should perform migration on startup.
+        #[clap(short, long, env, default_value_t = true)]
+        migrate: bool,
+
+        /// The server address to bind to.
+        #[clap(short, long, env, default_value = "0.0.0.0:3000")]
+        addr: SocketAddr,
+    },
+
+    Csv {
+        /// The path to the CSV file to ingest.
+        csv_file: PathBuf,
+    },
+}
+
 #[derive(Debug, clap::Parser)]
-struct Config {
-    /// If we should perform migration on startup.
-    #[clap(short, long, env, default_value_t = true)]
-    migrate: bool,
-
-    /// The server address to bind to.
-    #[clap(short, long, env, default_value = "0.0.0.0:3000")]
-    addr: SocketAddr,
-
+struct Cli {
     /// URL to PostgreSQL database.
     #[clap(short, long = "db", env)]
     database_url: String,
+
+    #[command(subcommand)]
+    command: Commands,
 }
 
 async fn health_check(
@@ -37,18 +64,48 @@ async fn health_check(
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    let config: Config = Config::parse();
-    let connection = sea_orm::ConnectOptions::new(&config.database_url)
+    tracing_subscriber::fmt::init();
+
+    let cli: Cli = Cli::parse();
+    let connection = sea_orm::ConnectOptions::new(&cli.database_url)
         .sqlx_logging_level(LevelFilter::Debug)
         .to_owned();
 
     let connection = sea_orm::Database::connect(connection).await?;
 
-    if config.migrate {
-        Migrator::up(&connection, None).await?;
+    match cli.command {
+        Commands::Migrate { steps, down } => {
+            if down {
+                Migrator::down(&connection, steps).await?;
+            } else {
+                Migrator::up(&connection, steps).await?
+            }
+        }
+
+        Commands::Run { migrate, addr } => {
+            if migrate {
+                Migrator::up(&connection, None).await?;
+            }
+
+            serve_web(addr, connection).await?;
+        }
+
+        Commands::Csv { csv_file } => {
+            let mut csv = csv::Reader::from_reader(File::open(csv_file)?);
+            let data = csv.records();
+            let data = data
+                .filter_map(|f| f.ok())
+                .map(from_csv_row)
+                .collect::<Result<_, _>>()?;
+
+            db::insert(&connection, data).await?;
+        }
     }
 
-    tracing_subscriber::fmt::init();
+    Ok(())
+}
+
+async fn serve_web(address: SocketAddr, connection: DatabaseConnection) -> anyhow::Result<()> {
     let app = Router::new()
         .route("/health", get(health_check))
         .route("/monzo-batch-export", post(monzo_batched_json))
@@ -56,9 +113,9 @@ async fn main() -> anyhow::Result<()> {
         .layer(Extension(connection.clone()))
         .layer(TraceLayer::new_for_http());
 
-    tracing::debug!("listening on {}", &config.addr);
-    let listener = tokio::net::TcpListener::bind(&config.addr).await.unwrap();
-    axum::serve(listener, app).await.unwrap();
+    tracing::info!("listening on {}", &address);
+    let listener = tokio::net::TcpListener::bind(&address).await?;
+    axum::serve(listener, app).await?;
 
     Ok(())
 }