From 7759632848199185729d0e635a7f3ad8f25d1826 Mon Sep 17 00:00:00 2001 From: Joshua Coles Date: Sun, 3 Mar 2024 20:35:53 +0000 Subject: [PATCH] Migrate database updates to use `TimeEntry`s over `ReportRow`s --- src/db.rs | 50 +++++++++-------------- src/poll.rs | 81 +++++++++++++------------------------ src/routes.rs | 59 ++++++++++++++++----------- src/sync_service.rs | 71 ++++++++++++++++++++++++++------ src/toggl_api/api_client.rs | 15 ++++++- src/toggl_api/types.rs | 2 +- 6 files changed, 153 insertions(+), 125 deletions(-) diff --git a/src/db.rs b/src/db.rs index 4ca6806..370db3b 100644 --- a/src/db.rs +++ b/src/db.rs @@ -4,38 +4,24 @@ use sea_orm::sea_query::OnConflict; use sea_orm::{NotSet, Set}; impl ReportRow { - pub(crate) fn as_models(&self) -> Vec { - self.time_entries - .iter() - .map(|inner| time_entry::ActiveModel { - id: NotSet, - toggl_id: Set(inner.id as i64), - description: Set(self.description.clone()), - project_id: Set(self.project_id.map(|id| id as i64)), - start: Set(inner.start.fixed_offset()), - stop: Set(inner.stop.fixed_offset()), - raw_json: Set(serde_json::to_value(inner).unwrap()), - server_deleted_at: NotSet, - server_updated_at: Set(inner.at.fixed_offset()), - // TODO: tags on report row import, need to track in separate table - tags: NotSet, - }) - .collect() - } - - pub fn grafting_conflict_statement() -> OnConflict { - OnConflict::column(time_entry::Column::TogglId) - .update_columns(vec![ - time_entry::Column::Description, - time_entry::Column::ProjectId, - time_entry::Column::Start, - time_entry::Column::Stop, - time_entry::Column::RawJson, - time_entry::Column::ServerUpdatedAt, - time_entry::Column::ServerDeletedAt, - // time_entry::Column::Tags, TODO add tags to conflict resolution when implemented - ]) - .to_owned() + pub fn to_time_entries(&self, workspace_id: i64) -> Vec { + self.time_entries.iter() + .map(|inner| TimeEntry { + id: inner.id as i64, + description: self.description.clone(), + project_id: self.project_id.map(|id| id as i64), + task_id: self.task_id.map(|id| id as i64), + billable: self.billable, + start: inner.start, + stop: Some(inner.stop), + at: inner.at, + server_deleted_at: None, + tags: vec![], // TODO: tags on report row import, need to track in separate table + workspace_id, + duration: inner.seconds as i64, + tag_ids: self.tag_ids.iter().map(|ids| *ids as i64).collect(), + user_id: self.user_id as i64, + }).collect() } } diff --git a/src/poll.rs b/src/poll.rs index 98af866..dabab8f 100644 --- a/src/poll.rs +++ b/src/poll.rs @@ -1,5 +1,5 @@ use crate::entity::prelude::TimeEntry; -use crate::entity::{client, project}; +use crate::entity::{client, project, time_entry}; use crate::toggl_api::types::{Client, Project, TogglReportQuery}; use crate::toggl_api::TogglApiClient; use crate::utils; @@ -11,7 +11,9 @@ use sea_orm::{ Statement, }; use std::ops::Sub; +use migration::Order; use tracing::instrument; +use crate::sync_service::update_database; #[tracing::instrument(skip(client, db))] pub async fn poll_job(client: TogglApiClient, db: DatabaseConnection, poll_period: u64) { @@ -37,65 +39,36 @@ pub async fn poll_job(client: TogglApiClient, db: DatabaseConnection, poll_perio } } -#[instrument(skip(client, db))] +#[instrument(skip(toggl_client, db))] pub async fn perform_poll( - client: &TogglApiClient, + toggl_client: &TogglApiClient, db: &DatabaseConnection, ) -> utils::Result { - let since = db - .query_one(Statement::from_string( - Postgres, - r#"select server_updated_at from time_entry order by server_updated_at desc limit 1"#, - )) - .await? - .map(|row| row.try_get_by::, _>("server_updated_at")) - .transpose()? - .unwrap_or( - chrono::Utc::now() - .sub(chrono::Duration::days(1)) - .fixed_offset(), - ); + let since = time_entry::Entity::find() + .select_only() + .column(time_entry::Column::ServerUpdatedAt) + .order_by(time_entry::Column::ServerUpdatedAt, Order::Desc) + .into_tuple::>() + .one(db) + .await?; - let time_entries = client + let since = since.unwrap_or( + chrono::Utc::now() + .sub(chrono::Duration::days(1)) + .fixed_offset(), + ); + + let time_entries = toggl_client .fetch_time_entries_modified_since(since.to_utc()) .await?; - let existing_project_ids = project::Entity::find() - .select_only() - .column(project::Column::TogglId) - .into_tuple::() - .all(db) - .await?; + // These are changes only so there is no need to enforce exclusivity + update_database( + db, + toggl_client, + &time_entries, + None, + ).await?; - let new_projects = time_entries - .iter() - .filter_map(|entry| entry.project_id) - .any(|project_id| !existing_project_ids.contains(&(project_id as i64))); - - if new_projects { - let clients = client.fetch_clients().await?; - - client::Entity::insert_many(clients.iter().map(Client::as_model)) - .on_conflict(Client::grafting_conflict_statement()) - .exec(db) - .await?; - - let projects = client.fetch_projects().await?; - - project::Entity::insert_many(projects.iter().map(Project::as_model)) - .on_conflict(Project::grafting_conflict_statement()) - .exec(db) - .await?; - } - - crate::sync_service::cache_report( - &db, - &report, - Some(day_exclusivity_condition( - now.date_naive(), - now.date_naive(), - )), - ) - .await?; - Ok(report.len()) + Ok(time_entries.len()) } diff --git a/src/routes.rs b/src/routes.rs index a56a131..5a61a4b 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -1,5 +1,3 @@ -use crate::entity::time_entry; -use crate::entity::time_entry::Entity as TimeEntry; use crate::toggl_api::types::{self, Client, Project, ReportRow, TogglReportQuery}; use crate::toggl_api::TogglApiClient; use crate::{entity, sync_service, utils}; @@ -8,13 +6,12 @@ use axum::extract::{Multipart, Query}; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::{Extension, Json}; -use chrono::NaiveDate; -use migration::Condition; -use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use chrono::{DateTime, Utc}; +use sea_orm::{DatabaseConnection, EntityTrait}; use serde::Deserialize; use serde_json::Value; use std::collections::HashMap; -use tracing::{debug, instrument}; +use tracing::{instrument}; #[instrument(skip(db, toggl_client))] pub async fn report( @@ -23,11 +20,13 @@ pub async fn report( Json(query): Json, ) -> utils::Result>> { let report = toggl_client.full_report(&query).await?; - debug!("Returned results: {:?}", report); - - // We don't perform any deletes on report-fetched entries as they aren't necessarily exclusive - // on their time range. - sync_service::cache_report(&db, &report, None).await?; + let time_entries = report.iter().flat_map(|entry| entry.to_time_entries(toggl_client.workspace_id())).collect::>(); + sync_service::update_database( + &db, + &toggl_client, + &time_entries, + None, + ).await?; Ok(Json(report)) } @@ -111,29 +110,41 @@ pub async fn import_csv( #[derive(Debug, Clone, Deserialize)] pub struct RefreshQuery { - start_date: Option, + start_date: Option>, + end_date: Option>, } #[instrument(skip(toggl_client, db))] pub async fn refresh( Extension(toggl_client): Extension, Extension(db): Extension, - Query(RefreshQuery { start_date }): Query, + Query(RefreshQuery { start_date, end_date }): Query, ) -> utils::Result<&'static str> { - let end_date = chrono::Utc::now(); - let end_date_query_string = end_date.date_naive().format("%Y-%m-%d").to_string(); - let start_date_query_string = start_date.unwrap_or(end_date_query_string.clone()); - let start_date = NaiveDate::parse_from_str(&start_date_query_string, "%Y-%m-%d")?; + let time_entries = match (start_date, end_date) { + (Some(start_date), Some(end_date)) => { + toggl_client.fetch_time_entries_in_range(start_date, end_date).await? + } - let query = TogglReportQuery { - start_date: Some(start_date_query_string), - end_date: Some(end_date_query_string), - ..Default::default() + (Some(start_date), None) => { + let end_date = Utc::now(); + toggl_client.fetch_time_entries_in_range(start_date, end_date).await? + } + + (None, Some(_)) => { + return Err(anyhow!("start_date must be provided if end_date is provided").into()); + } + + _ => { + toggl_client.fetch_recent_time_entries().await? + } }; - let report = toggl_client.full_report(&query).await?; - let exclusivity_condition = utils::day_exclusivity_condition(start_date, end_date.date_naive()); - sync_service::cache_report(&db, &report, Some(exclusivity_condition)).await?; + sync_service::update_database( + &db, + &toggl_client, + &time_entries, + None, + ).await?; Ok("Ok") } diff --git a/src/sync_service.rs b/src/sync_service.rs index d9f08fe..83a5e0e 100644 --- a/src/sync_service.rs +++ b/src/sync_service.rs @@ -1,24 +1,69 @@ -use crate::entity::time_entry; +use crate::entity::{client, project, time_entry}; use crate::entity::time_entry::Entity as TimeEntry; -use crate::toggl_api::types::ReportRow; +use crate::toggl_api::types::{Client, Project, ReportRow, TimeEntry as ToggleApiTimeEntry}; use crate::utils; use migration::Condition; -use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QuerySelect}; use tracing::{debug, instrument}; +use crate::toggl_api::TogglApiClient; -#[instrument(skip_all)] -pub async fn cache_report( +pub async fn update_database( db: &DatabaseConnection, - models: &Vec, + toggl_client: &TogglApiClient, + time_entries: &[ToggleApiTimeEntry], exclusive_on: Option, ) -> utils::Result<()> { - let models = models.iter().flat_map(|entry| entry.as_models()); - let models = models.collect::>(); - let ids = models + let (deleted_entries, time_entries) = time_entries .iter() - .map(|entry| entry.toggl_id.clone().unwrap()) + .partition::, _>(|entry| entry.server_deleted_at.is_some()); + + let deleted_ids = deleted_entries.iter() + .map(|entry| entry.id) + .collect::>(); + + if !deleted_ids.is_empty() { + debug!("Deleting time entries: {:?}", deleted_ids); + TimeEntry::delete_many() + .filter(time_entry::Column::TogglId.is_in(deleted_ids)) + .exec(db) + .await?; + } + + let existing_project_ids = project::Entity::find() + .select_only() + .column(project::Column::TogglId) + .into_tuple::() + .all(db) + .await?; + + let new_projects = time_entries + .iter() + .filter_map(|entry| entry.project_id) + .any(|project_id| !existing_project_ids.contains(&(project_id as i64))); + + if new_projects { + let clients = toggl_client.fetch_clients().await?; + + client::Entity::insert_many(clients.iter().map(Client::as_model)) + .on_conflict(Client::grafting_conflict_statement()) + .exec(db) + .await?; + + let projects = toggl_client.fetch_projects().await?; + + project::Entity::insert_many(projects.iter().map(Project::as_model)) + .on_conflict(Project::grafting_conflict_statement()) + .exec(db) + .await?; + } + + let ids = time_entries.iter() + .map(|entry| entry.id) + .collect::>(); + + let models = time_entries.into_iter() + .map(|entry| entry.as_model()) .collect::>(); - debug!("Caching report entries: {:?}", models); // TODO: Why is this needed? if models.is_empty() { @@ -26,7 +71,7 @@ pub async fn cache_report( } TimeEntry::insert_many(models) - .on_conflict(ReportRow::grafting_conflict_statement()) + .on_conflict(ToggleApiTimeEntry::grafting_conflict_statement()) .exec(db) .await?; @@ -42,4 +87,4 @@ pub async fn cache_report( } Ok(()) -} +} \ No newline at end of file diff --git a/src/toggl_api/api_client.rs b/src/toggl_api/api_client.rs index 883375c..2d79539 100644 --- a/src/toggl_api/api_client.rs +++ b/src/toggl_api/api_client.rs @@ -40,6 +40,10 @@ impl TogglApiClient { } } + pub fn workspace_id(&self) -> i64 { + self.workspace_id.parse().unwrap() + } + pub async fn check_health(&self) -> bool { true } @@ -100,6 +104,15 @@ impl TogglApiClient { Ok(clients) } + pub async fn fetch_recent_time_entries(&self) -> crate::Result> { + let url = format!("{base_url}/me/time_entries", base_url = self.base_url); + + Ok(self.make_request(self.client.get(url)) + .await? + .json::>() + .await?) + } + pub async fn fetch_time_entries_modified_since( &self, date_time: DateTime, @@ -119,7 +132,7 @@ impl TogglApiClient { pub async fn fetch_time_entries_in_range( &self, - (start, end): (DateTime, DateTime), + start: DateTime, end: DateTime, ) -> crate::Result> { let url = format!("{base_url}/me/time_entries", base_url = self.base_url); diff --git a/src/toggl_api/types.rs b/src/toggl_api/types.rs index fa6b190..7ed3a8c 100644 --- a/src/toggl_api/types.rs +++ b/src/toggl_api/types.rs @@ -35,7 +35,7 @@ pub struct TimeEntry { pub id: i64, pub workspace_id: i64, pub project_id: Option, - pub task_id: Option, + pub task_id: Option, pub billable: bool, pub start: DateTime, pub stop: Option>,