joshuacoles/arc-ingester
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add test containers to allow for unit testing SQL files

Browse Source
This commit is contained in:
Joshua Coles 2024-04-05 15:35:02 +01:00
parent bcfa0c2d89
commit 48f4614b1c
4 changed files with 131 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
Cargo.lock generated
View File

@ -113,6 +113,8 @@ dependencies = [
"serde_json",
"sha256",
"sqlx",
"testcontainers",
"testcontainers-modules",
"thiserror",
"tokio",
"uuid",
@ -205,6 +207,16 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bollard-stubs"
version = "1.42.0-rc.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed59b5c00048f48d7af971b71f800fdf23e858844a6f9e4d32ca72e9399e7864"
dependencies = [
"serde",
"serde_with",
]
[[package]]
name = "bumpalo"
version = "3.14.0"
@ -388,6 +400,41 @@ dependencies = [
"typenum",
]
[[package]]
name = "darling"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
dependencies = [
"darling_core",
"quote",
"syn 1.0.109",
]
[[package]]
name = "der"
version = "0.7.8"
@ -492,6 +539,12 @@ dependencies = [
"spin 0.9.8",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@ -718,6 +771,12 @@ dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.5.0"
@ -1254,6 +1313,28 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_with"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff"
dependencies = [
"serde",
"serde_with_macros",
]
[[package]]
name = "serde_with_macros"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "sha1"
version = "0.10.6"
@ -1634,6 +1715,32 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "testcontainers"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d2931d7f521af5bae989f716c3fa43a6af9af7ec7a5e21b59ae40878cec00"
dependencies = [
"bollard-stubs",
"futures",
"hex",
"hmac",
"log",
"rand",
"serde",
"serde_json",
"sha2",
]
[[package]]
name = "testcontainers-modules"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "204d1c7516bfdc8a01bb85d3e30145e5bbeb2351812e5e8aa6971769109b45b5"
dependencies = [
"testcontainers",
]
[[package]]
name = "thiserror"
version = "1.0.56"

2
Cargo.toml
View File

@ -28,3 +28,5 @@ futures = "0.3.30"
itertools = "0.12.1"
chrono = { version = "0.4.33", features = ["serde"] }
uuid = { version = "1.7.0", features = ["serde"] }
testcontainers = "0.15.0"
testcontainers-modules = { version = "0.3.6", features = ["postgres"] }

2
src/main.rs
View File

@ -133,7 +133,7 @@ async fn main() {
// Refresh the database with the new files
for updated_file in &need_refresh {
sqlx::query("INSERT INTO raw_files (date, sha256, json) VALUES ($1, $2, $3 :: jsonb) ON CONFLICT (date) DO UPDATE SET sha256 = $2, json = $3 :: jsonb")
sqlx::query("INSERT INTO raw_files (date, sha256, json) VALUES ($1, $2, $3 :: jsonb) ON CONFLICT (date) DO UPDATE SET sha256 = excluded.sha256, json = excluded.json")
.bind(&updated_file.date)
.bind(&updated_file.sha256)
.bind(&serde_json::to_value(&updated_file.json).unwrap())

27
src/t1.rs
View File

@ -1,7 +1,22 @@
fn main() {
let files = std::fs::read_dir("/Users/joshuacoles/Library/Mobile Documents/iCloud~com~bigpaua~LearnerCoacher/Documents/Export/JSON/Daily/").unwrap();
for file in files {
let file = file.unwrap().path();
println!("{}\t{}", file.display(), sha256::try_digest(&file).unwrap())
}
use sqlx::{Connection, Executor};
use testcontainers::core::env::Os;
#[tokio::main]
async fn main() {
// let files = std::fs::read_dir("/Users/joshuacoles/Library/Mobile Documents/iCloud~com~bigpaua~LearnerCoacher/Documents/Export/JSON/Daily/").unwrap();
// for file in files {
// let file = file.unwrap().path();
// println!("{}\t{}", file.display(), sha256::try_digest(&file).unwrap())
// }
//
let tc = testcontainers::clients::Cli::new::<Os>();
let pg_spec = testcontainers_modules::postgres::Postgres::default();
let pg_container = tc.run(pg_spec);
pg_container.start();
println!("postgres running");
let mut pg = sqlx::postgres::PgConnection::connect(&format!("postgres://postgres:postgres@localhost:{}/postgres", pg_container.get_host_port_ipv4(5432)),)
.await
.unwrap();
dbg!(pg.execute("select 1").await.unwrap());
}