joshuacoles/arc-ingester
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Try to fix memory issues
All checks were successful
Rust / build (push) Successful in 2m40s
Details

Browse Source
This commit is contained in:
Joshua Coles 2024-04-04 11:15:47 +01:00
parent 3581797ebe
commit 7dee90f09f
2 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build.yml vendored
View File

@ -50,4 +50,4 @@ jobs:
uses: actions/upload-artifact@v3
with:
name: binary
path: target/release/arc-ingester
path: target/release/arc-ingester

14
src/main.rs
View File

@ -85,7 +85,7 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
// the hash stored in the database.
let need_refresh = files.into_iter()
.map(|f| f.unwrap().path())
.map(|path| async move {
.filter_map(|path| {
// Extract the date from the file name
let date = {
let file_name = path.file_name().unwrap().to_str().unwrap();
@ -93,8 +93,7 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
&file_name[..i]
};
let bytes = tokio::fs::read(&path).await.unwrap();
let current_hash = sha256::digest(&bytes);
let current_hash = sha256::try_digest(&path).unwrap();
let existing_hash = date_hashes.get(date);
if let Some(existing_hash) = existing_hash {
@ -103,12 +102,11 @@ async fn find_updated(db: &sqlx::PgPool, files: ReadDir) -> Vec<UpdatedFile> {
}
}
return Some((date.to_string(), current_hash, bytes));
});
let bytes = std::fs::read(&path).unwrap();
let need_refresh = futures::future::join_all(need_refresh).await.into_iter()
.filter_map(|x| x)
.collect::<Vec<_>>();
return Some((date.to_string(), current_hash, bytes));
})
.collect_vec();
let decompressed = need_refresh.par_iter().map(|(date, new_hash, bytes)| {
let mut decoder = flate2::bufread::GzDecoder::new(&bytes[..]);