From 261baa6b5bd5091b62ad82131ad970d6daedd843 Mon Sep 17 00:00:00 2001 From: Joshua Coles Date: Thu, 25 Apr 2024 14:08:41 +0100 Subject: [PATCH] Add a little bit of tracing --- Cargo.lock | 133 ++++++++++++++++++++++++++++++++++++---------------- Cargo.toml | 5 +- src/main.rs | 8 ++++ 3 files changed, 103 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f105ecf..2916fa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,7 +110,7 @@ dependencies = [ [[package]] name = "arc-ingester" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "chrono", @@ -118,7 +118,6 @@ dependencies = [ "flate2", "futures", "itertools", - "rayon", "serde", "serde_json", "sha256", @@ -127,6 +126,8 @@ dependencies = [ "testcontainers-modules", "thiserror", "tokio", + "tracing", + "tracing-subscriber", "uuid", ] @@ -366,25 +367,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -948,6 +930,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1021,6 +1013,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1158,26 +1156,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rayon" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -1380,6 +1358,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1771,6 +1758,16 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1857,6 +1854,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -1936,6 +1959,12 @@ dependencies = [ "serde", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -2020,6 +2049,28 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index cb98361..a797fa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "arc-ingester" -version = "0.1.1" +version = "0.1.2" edition = "2021" authors = ["Joshua Coles"] license = "MIT OR Apache-2.0" @@ -23,7 +23,6 @@ thiserror = "1" sha256 = "1" serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.113" -rayon = "1.8.1" futures = "0.3.30" itertools = "0.12.1" chrono = { version = "0.4.33", features = ["serde"] } @@ -31,3 +30,5 @@ uuid = { version = "1.7.0", features = ["serde"] } testcontainers = "0.15.0" testcontainers-modules = { version = "0.3.6", features = ["postgres"] } anyhow = { version = "1.0.82", features = ["backtrace"] } +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/src/main.rs b/src/main.rs index a8c69ed..07b2238 100644 --- a/src/main.rs +++ b/src/main.rs @@ -95,10 +95,16 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec { let new_hashes = hash_files(files); new_hashes.filter_map(|(date, path, new_hash)| { + tracing::debug!("Considering file for updates {path} (read as date: {date})"); if let Some(existing_hash) = date_hashes.get(&date) { if new_hash == *existing_hash { + tracing::debug!("Found and matched to to existing hash"); return None; + } else { + tracing::debug!("Found existing hash but file does not match"); } + } else { + tracing::debug!("No existing hash found"); } let bytes = std::fs::read(&path).unwrap(); @@ -123,6 +129,8 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec { #[tokio::main] async fn main() { + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); let daily_exports = cli.root.join("Export/JSON/Daily"); let files = std::fs::read_dir(daily_exports)