fix(mimic-iii): merge overlapping line duration intervals

.github/workflows/transpile.yml

@@ -19,6 +19,8 @@ jobs:
         run: pip install -e . --no-deps
       - name: Run transpiler unit tests
         run: pytest tests/test_transpile.py -q
+      - name: Test line duration regressions
+        run: pytest tests/test_line_durations.py -q
 
   generated-up-to-date:
     name: Check cross-dialect concepts match BQ

mimic-iii/concepts/durations/arterial_line_durations.sql

@@ -30,6 +30,49 @@ with mv as
     -- , 228286 -- Intraosseous Device | None | 12 | Processes
   )
 )
+-- Collapse overlapping or touching MetaVision intervals to time with a line present.
+-- Use the running maximum: LAG(endtime) alone fails for nested intervals.
+, mv_intervals as
+(
+  select distinct icustay_id, starttime, endtime
+  from mv
+  where arterial_line = 1
+)
+, mv_ordered as
+(
+  select icustay_id, starttime, endtime
+    , MAX(endtime) OVER
+      (
+        partition by icustay_id order by starttime, endtime
+        rows between unbounded preceding and 1 preceding
+      ) as prior_endtime
+  from mv_intervals
+)
+, mv_marked as
+(
+  select icustay_id, starttime, endtime
+    , case when prior_endtime is null or starttime > prior_endtime
+        then 1 else 0 end as new_interval
+  from mv_ordered
+)
+, mv_grouped as
+(
+  select icustay_id, starttime, endtime
+    , SUM(new_interval) OVER
+      (
+        partition by icustay_id order by starttime, endtime
+        rows between unbounded preceding and current row
+      ) as interval_group
+  from mv_marked
+)
+, mv_dur as
+(
+  select icustay_id
+    , MIN(starttime) as starttime
+    , MAX(endtime) as endtime
+  from mv_grouped
+  group by icustay_id, interval_group
+)
 , cv_grp as
 (
   -- group type+site
@@ -168,11 +211,9 @@ select icustay_id
   , starttime, endtime, duration_hours
 from cv_dur
 UNION ALL
---TODO: collapse metavision durations if they overlap
 select icustay_id
   -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum
   , starttime, endtime
   , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours
-from mv
-where arterial_line = 1
+from mv_dur
 order by icustay_id, starttime;

mimic-iii/concepts/durations/central_line_durations.sql

@@ -25,6 +25,49 @@ with mv as
     , 224270 -- Dialysis Catheter
   )
 )
+-- Collapse overlapping or touching MetaVision intervals to time with a line present.
+-- Use the running maximum: LAG(endtime) alone fails for nested intervals.
+, mv_intervals as
+(
+  select distinct icustay_id, starttime, endtime
+  from mv
+  where central_line = 1
+)
+, mv_ordered as
+(
+  select icustay_id, starttime, endtime
+    , MAX(endtime) OVER
+      (
+        partition by icustay_id order by starttime, endtime
+        rows between unbounded preceding and 1 preceding
+      ) as prior_endtime
+  from mv_intervals
+)
+, mv_marked as
+(
+  select icustay_id, starttime, endtime
+    , case when prior_endtime is null or starttime > prior_endtime
+        then 1 else 0 end as new_interval
+  from mv_ordered
+)
+, mv_grouped as
+(
+  select icustay_id, starttime, endtime
+    , SUM(new_interval) OVER
+      (
+        partition by icustay_id order by starttime, endtime
+        rows between unbounded preceding and current row
+      ) as interval_group
+  from mv_marked
+)
+, mv_dur as
+(
+  select icustay_id
+    , MIN(starttime) as starttime
+    , MAX(endtime) as endtime
+  from mv_grouped
+  group by icustay_id, interval_group
+)
 , cv_grp as
 (
   -- group type+site
@@ -163,11 +206,9 @@ select icustay_id
   , starttime, endtime, duration_hours
 from cv_dur
 UNION ALL
---TODO: collapse metavision durations if they overlap
 select icustay_id
   -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum
   , starttime, endtime
   , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours
-from mv
-where central_line = 1
+from mv_dur
 order by icustay_id, starttime;

mimic-iii/concepts_duckdb/durations/arterial_line_durations.sql

@@ -17,6 +17,52 @@ WITH mv AS (
   FROM mimiciii.procedureevents_mv AS pe
   WHERE
     pe.itemid IN (224263, 224267, 224268, 225199, 225752, 225789, 224272)
+), mv_intervals AS (
+  SELECT DISTINCT
+    icustay_id,
+    starttime,
+    endtime
+  FROM mv
+  WHERE
+    arterial_line = 1
+), mv_ordered AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    MAX(endtime) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND 1 preceding
+    ) AS prior_endtime
+  FROM mv_intervals
+), mv_marked AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    CASE WHEN prior_endtime IS NULL OR starttime > prior_endtime THEN 1 ELSE 0 END AS new_interval
+  FROM mv_ordered
+), mv_grouped AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    SUM(new_interval) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND CURRENT ROW
+    ) AS interval_group
+  FROM mv_marked
+), mv_dur AS (
+  SELECT
+    icustay_id,
+    MIN(starttime) AS starttime,
+    MAX(endtime) AS endtime
+  FROM mv_grouped
+  GROUP BY
+    icustay_id,
+    interval_group
 ), cv_grp AS (
   SELECT
     ce.icustay_id,
@@ -136,9 +182,7 @@ SELECT
   starttime,
   endtime,
   DATE_DIFF('HOUR', starttime, endtime) AS duration_hours
-FROM mv
-WHERE
-  arterial_line = 1
+FROM mv_dur
 ORDER BY
   icustay_id NULLS FIRST,
   starttime NULLS FIRST

mimic-iii/concepts_duckdb/durations/central_line_durations.sql

@@ -27,6 +27,52 @@ WITH mv AS (
       227719,
       224270
     )
+), mv_intervals AS (
+  SELECT DISTINCT
+    icustay_id,
+    starttime,
+    endtime
+  FROM mv
+  WHERE
+    central_line = 1
+), mv_ordered AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    MAX(endtime) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND 1 preceding
+    ) AS prior_endtime
+  FROM mv_intervals
+), mv_marked AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    CASE WHEN prior_endtime IS NULL OR starttime > prior_endtime THEN 1 ELSE 0 END AS new_interval
+  FROM mv_ordered
+), mv_grouped AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    SUM(new_interval) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND CURRENT ROW
+    ) AS interval_group
+  FROM mv_marked
+), mv_dur AS (
+  SELECT
+    icustay_id,
+    MIN(starttime) AS starttime,
+    MAX(endtime) AS endtime
+  FROM mv_grouped
+  GROUP BY
+    icustay_id,
+    interval_group
 ), cv_grp AS (
   SELECT
     ce.icustay_id,
@@ -234,9 +280,7 @@ SELECT
   starttime,
   endtime,
   DATE_DIFF('HOUR', starttime, endtime) AS duration_hours
-FROM mv
-WHERE
-  central_line = 1
+FROM mv_dur
 ORDER BY
   icustay_id NULLS FIRST,
   starttime NULLS FIRST

mimic-iii/concepts_postgres/durations/arterial_line_durations.sql

@@ -25,6 +25,52 @@ WITH mv AS (
       225789, /* Sheath */
       224272 /* IABP Line */
     ) /* , 227719 -- AVA Line | None | 12 | Processes */ /* , 228286 -- Intraosseous Device | None | 12 | Processes */
+), mv_intervals /* Collapse overlapping or touching MetaVision intervals to time with a line present. */ /* Use the running maximum: LAG(endtime) alone fails for nested intervals. */ AS (
+  SELECT DISTINCT
+    icustay_id,
+    starttime,
+    endtime
+  FROM mv
+  WHERE
+    arterial_line = 1
+), mv_ordered AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    MAX(endtime) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND 1 preceding
+    ) AS prior_endtime
+  FROM mv_intervals
+), mv_marked AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    CASE WHEN prior_endtime IS NULL OR starttime > prior_endtime THEN 1 ELSE 0 END AS new_interval
+  FROM mv_ordered
+), mv_grouped AS (
+  SELECT
+    icustay_id,
+    starttime,
+    endtime,
+    SUM(new_interval) OVER (
+      PARTITION BY icustay_id
+      ORDER BY starttime NULLS FIRST, endtime NULLS FIRST
+      rows BETWEEN UNBOUNDED preceding AND CURRENT ROW
+    ) AS interval_group
+  FROM mv_marked
+), mv_dur AS (
+  SELECT
+    icustay_id,
+    MIN(starttime) AS starttime,
+    MAX(endtime) AS endtime
+  FROM mv_grouped
+  GROUP BY
+    icustay_id,
+    interval_group
 ), cv_grp AS (
   /* group type+site */
   SELECT
@@ -144,15 +190,12 @@ SELECT
   duration_hours
 FROM cv_dur
 UNION ALL
-/* TODO: collapse metavision durations if they overlap */
 SELECT
   icustay_id, /* , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum */
   starttime,
   endtime,
   CAST(EXTRACT(EPOCH FROM DATE_TRUNC('hour', endtime) - DATE_TRUNC('hour', starttime)) / 3600 AS BIGINT) AS duration_hours
-FROM mv
-WHERE
-  arterial_line = 1
+FROM mv_dur
 ORDER BY
   icustay_id NULLS FIRST,
   starttime NULLS FIRST