joshuacoles/arc-ingester
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify code to determine if files need to be changed and prep to move data transformation into PSQL

Browse Source
This commit is contained in:
joshuacoles
2024-04-04 21:49:52 +01:00
parent 99523430c6
commit bcfa0c2d89
5 changed files with 97 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/update_places.sql
+24
View File
@@ -0,0 +1,24 @@
with timelineItems as (select jsonb_array_elements(raw_files.json -> 'timelineItems') as timelineItem
from raw_files),
places as (select distinct on (md5(timelineItem ->> 'place' :: text)) timelineItem -> 'place' as place,
timelineItem -> 'place' ->> 'placeId' as placeId,
(timelineItem -> 'place' ->> 'lastSaved') :: timestamptz as lastSaved
from timelineItems
where timelineItem ->> 'place' is not null),
places_with_max_last_saved as (select place -> 'placeId' as placeId,
max((place ->> 'lastSaved') :: timestamptz) as latest_last_saved
from places
group by place -> 'placeId'),
latest_places as (select places.*
from places_with_max_last_saved
inner join places on places.place -> 'placeId' = places_with_max_last_saved.placeId and
places.lastSaved =
places_with_max_last_saved.latest_last_saved)
insert
into public.place (place_id, json, last_saved, server_last_updated)
select (placeId :: uuid) as place_id, place as json, lastSaved as last_saved, now() as server_last_updated
from latest_places
on conflict (place_id) do update set json = excluded.json,
last_saved = excluded.last_saved,
server_last_updated = excluded.server_last_updated
where excluded.last_saved > public.place.last_saved;
functions/update_timeline_items.sql
+28
View File
@@ -0,0 +1,28 @@
with timelineItems as (select jsonb_array_elements(raw_files.json -> 'timelineItems') as timelineItem
from raw_files),
max_last_saved as (select timelineItem ->> 'itemId' as itemId,
max((timelineItem ->> 'lastSaved') :: timestamptz) as latest_last_saved
from timelineItems
group by timelineItem ->> 'itemId'),
unique_timline_items as (select distinct on (max_last_saved.itemId) *
from max_last_saved
inner join timelineItems
on timelineItems.timelineItem ->> 'itemId' = max_last_saved.itemId
and (timelineItems.timelineItem ->> 'lastSaved') :: timestamptz =
max_last_saved.latest_last_saved)
insert
into public.timeline_item (item_id, json, place_id, end_date, last_saved, server_last_updated)
select unique_timline_items.itemId :: uuid as item_id,
unique_timline_items.timelineItem as json,
(unique_timline_items.timelineItem -> 'place' ->> 'placeId') :: uuid as place_id,
(unique_timline_items.timelineItem ->> 'endDate') :: timestamptz as end_date,
unique_timline_items.latest_last_saved :: timestamptz as last_saved,
now() as server_last_updated
from unique_timline_items
on conflict (item_id) do update set json = excluded.json,
place_id = excluded.place_id,
end_date = excluded.end_date,
last_saved = excluded.last_saved,
server_last_updated = excluded.server_last_updated
where excluded.last_saved > public.timeline_item.last_saved
returning item_id;