SOA for batch inserts using UNNEST as per docs

This commit is contained in:
Joshua Coles 2024-07-27 20:28:39 +01:00
parent ce46739f30
commit 4efbb95204
4 changed files with 53 additions and 28 deletions

21
Cargo.lock generated
View File

@ -1912,6 +1912,26 @@ version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "soa-rs"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c76685dff09a60d416b82a23e192e14cd47d147858b9aca2ce5ca05877acd93"
dependencies = [
"soa-rs-derive",
]
[[package]]
name = "soa-rs-derive"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4281fd25ca7e6cc6e84cadaf296ffabac12719e830e1af160cd243dca831577"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.71",
]
[[package]]
name = "socket2"
version = "0.5.7"
@ -2343,6 +2363,7 @@ dependencies = [
"serde_json",
"serde_json_path_to_error",
"serde_with",
"soa-rs",
"sqlx",
"thiserror",
"tokio",

View File

@ -25,3 +25,4 @@ futures = "0.3.30"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
itertools = "0.13.0"
soa-rs = "0.6.1"

View File

@ -1,8 +1,9 @@
use chrono::{DateTime, DurationRound, NaiveDate, TimeDelta, Utc};
use sqlx::{Connection, PgConnection, Postgres, QueryBuilder};
use toggl::TogglApi;
use crate::toggl::types::{TimeEntry, TogglReportFilters};
use crate::toggl::types::{Tag, TimeEntry, TogglReportFilters};
use itertools::Itertools;
use soa_rs::Soa;
mod toggl;
mod sensitive;
@ -256,31 +257,32 @@ impl Worker {
async fn update_tags(&mut self) -> Result<(), AppError> {
let tags = self.toggl_api.get_tags().await?;
let tags: Soa<Tag> = Soa::from(tags.as_slice());
for tag in tags {
sqlx::query!(
r#"
INSERT INTO tags (id, name, workspace_id, creator_id, updated_at, deleted_at, permissions)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (id) DO UPDATE SET
name = excluded.name,
workspace_id = excluded.workspace_id,
creator_id = excluded.creator_id,
updated_at = excluded.updated_at,
deleted_at = excluded.deleted_at,
permissions = excluded.permissions
"#,
tag.id as i64,
tag.name,
tag.workspace_id as i64,
tag.creator_id as i64,
tag.updated_at,
tag.deleted_at,
tag.permissions,
)
.execute(&mut self.db)
.await?;
}
sqlx::query!(
r#"
INSERT INTO tags (id, name, workspace_id, creator_id, updated_at, deleted_at, permissions)
SELECT * FROM UNNEST($1::bigint[], $2::text[], $3::bigint[], $4::bigint[], $5::timestamptz[], $6::timestamptz[], $7::text[])
ON CONFLICT (id) DO UPDATE SET
name = excluded.name,
workspace_id = excluded.workspace_id,
creator_id = excluded.creator_id,
updated_at = excluded.updated_at,
deleted_at = excluded.deleted_at,
permissions = excluded.permissions
"#,
&tags.id().iter().map(|id| *id as i64).collect::<Vec<_>>()[..],
tags.name(),
&tags.workspace_id().iter().map(|id| *id as i64).collect::<Vec<_>>()[..],
&tags.creator_id().iter().map(|id| *id as i64).collect::<Vec<_>>()[..],
tags.updated_at(),
// Nullable fields fail to type check with UNNEST batch inserts so we silence the
// errors using `: _`.
tags.deleted_at(): _,
tags.permissions(): _,
)
.execute(&mut self.db)
.await?;
Ok(())
}
@ -349,7 +351,7 @@ async fn main() {
toggl_api: api,
};
worker.update_clients()
worker.update_tags()
.await
.unwrap();

View File

@ -349,7 +349,7 @@ pub mod types {
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Soars, Serialize, Deserialize, Debug, Clone)]
pub struct TrackingClient {
/// The unique identifier for the client.
pub id: i64,
@ -378,7 +378,7 @@ pub mod types {
pub permissions: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Soars, Serialize, Deserialize, Debug, Clone)]
pub struct Tag {
pub id: u64,
pub name: String,
@ -493,6 +493,7 @@ pub mod types {
}
use std::fmt;
use soa_rs::Soars;
impl fmt::Debug for TogglReportFilters {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {