joshuacoles/arc-ingester
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Try to fix memory issues
All checks were successful
Rust / build (push) Successful in 2m40s
Details

Browse Source
This commit is contained in:
Joshua Coles 2024-04-04 11:15:47 +01:00
parent 3581797ebe
commit 7dee90f09f
2 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main.rs
View File

@ -85,7 +85,7 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
// the hash stored in the database. // the hash stored in the database.
let need_refresh = files.into_iter() let need_refresh = files.into_iter()
.map(|f| f.unwrap().path()) .map(|f| f.unwrap().path())
.map(|path| async move { .filter_map(|path| {
// Extract the date from the file name // Extract the date from the file name
let date = { let date = {
let file_name = path.file_name().unwrap().to_str().unwrap(); let file_name = path.file_name().unwrap().to_str().unwrap();
@ -93,8 +93,7 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
&file_name[..i] &file_name[..i]
}; };
let bytes = tokio::fs::read(&path).await.unwrap(); let current_hash = sha256::try_digest(&path).unwrap();
let current_hash = sha256::digest(&bytes);
let existing_hash = date_hashes.get(date); let existing_hash = date_hashes.get(date);
if let Some(existing_hash) = existing_hash { if let Some(existing_hash) = existing_hash {
@ -103,12 +102,11 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
} }
} }
return Some((date.to_string(), current_hash, bytes)); let bytes = std::fs::read(&path).unwrap();
});
let need_refresh = futures::future::join_all(need_refresh).await.into_iter() return Some((date.to_string(), current_hash, bytes));
.filter_map(|x| x) })
.collect::<Vec<_>>(); .collect_vec();
let decompressed = need_refresh.par_iter().map(|(date, new_hash, bytes)| { let decompressed = need_refresh.par_iter().map(|(date, new_hash, bytes)| {
let mut decoder = flate2::bufread::GzDecoder::new(&bytes[..]); let mut decoder = flate2::bufread::GzDecoder::new(&bytes[..]);