From 47ddd2e40a4d974618ecfd370763c38333b92d18 Mon Sep 17 00:00:00 2001 From: Michael Gaffney Date: Wed, 24 May 2023 16:22:25 -0400 Subject: [PATCH] fix(db): Fix generated series to yield consistent results The hcp billing `_all` views built a range of buckets using the `generate_series` function using the minimum value of the `session_pending_time` column from the `wh_session_accumulating_fact` table as the start time and `now()` as the end time. It then truncated values in the series using the `date_trunc` function. This allowed the generated series of buckets returned to vary and could result in one additional bucket being included in the series depending on the current time and the start time. As an illustrative example, the `hourly_all` query used: select date_trunc('hour',time) from generate_series( (select min(session_pending_time) -- start time from wh_session_accumulating_fact), now(), '1 hour'::interval ) as time; Substituting the start time with a hard coded value to show the issue: -- Show current time, where the minutes portion is 55 select now(); now ------------------------------- 2023-05-22 14:55:40.567605+00 (1 row) -- generate series with start time's minutes portion at 57 select date_trunc('hour',time) from generate_series( '2023-05-22 01:57:00'::timestamptz, now(), '1 hour'::interval ) as time; date_trunc ------------------------ 2023-05-22 01:00:00+00 2023-05-22 02:00:00+00 2023-05-22 03:00:00+00 2023-05-22 04:00:00+00 2023-05-22 05:00:00+00 2023-05-22 06:00:00+00 2023-05-22 07:00:00+00 2023-05-22 08:00:00+00 2023-05-22 09:00:00+00 2023-05-22 10:00:00+00 2023-05-22 11:00:00+00 2023-05-22 12:00:00+00 2023-05-22 13:00:00+00 (13 rows) -- Wait a few minutes select now(); now ------------------------------ 2023-05-22 14:58:48.75287+00 (1 row) -- Now run again and get an extra result, even though still -- within the same hour select date_trunc('hour',time) from generate_series( '2023-05-22 01:57:00'::timestamptz, now(), '1 hour'::interval ) as time; date_trunc ------------------------ 2023-05-22 01:00:00+00 2023-05-22 02:00:00+00 2023-05-22 03:00:00+00 2023-05-22 04:00:00+00 2023-05-22 05:00:00+00 2023-05-22 06:00:00+00 2023-05-22 07:00:00+00 2023-05-22 08:00:00+00 2023-05-22 09:00:00+00 2023-05-22 10:00:00+00 2023-05-22 11:00:00+00 2023-05-22 12:00:00+00 2023-05-22 13:00:00+00 2023-05-22 14:00:00+00 (14 rows) To fix this, we call the `generate_series` function truncating the start time argument to the bucket size: -- Hourly select bucket from generate_series( date_trunc('hour', select min(session_pending_time) from wh_session_accumulating_fact)), now(), '1 hour'::interval ) as bucket; -- Monthly select bucket from generate_series( date_trunc('month', select min(session_pending_time) from wh_session_accumulating_fact)), now(), '1 month'::interval ) as bucket; Fixes: 41b1b5b4b74b38141ddf934ff274dfb5c5b53d0e --- .../postgres/64/02_hcp_billing_hourly.up.sql | 1 + .../postgres/64/03_hcp_billing_monthly.up.sql | 2 + .../postgres/70/02_hcp_billing_hourly.up.sql | 36 +++++++++++++++ .../postgres/70/03_hcp_billing_monthly.up.sql | 44 +++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 internal/db/schema/migrations/oss/postgres/70/02_hcp_billing_hourly.up.sql create mode 100644 internal/db/schema/migrations/oss/postgres/70/03_hcp_billing_monthly.up.sql diff --git a/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql b/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql index e80ec96f92..7b206d75ae 100644 --- a/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql +++ b/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql @@ -72,6 +72,7 @@ begin; '169 rows are returned: 1 for the current hour plus 168 for the previous 7 days. ' 'Rows are sorted by the hour in descending order.'; + -- replaced in 70/02_hcp_billing_hourly.up.sql drop view if exists hcp_billing_hourly_sessions_all; create view hcp_billing_hourly_sessions_all as with diff --git a/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql b/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql index 665029345e..0e9618bc62 100644 --- a/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql +++ b/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql @@ -71,6 +71,7 @@ begin; 'The current month is a sum from the beginning of the current month ' 'until the start of the current hour (exclusive).'; + -- replaced in 70/03_hcp_billing_monthly.up.sql drop view if exists hcp_billing_monthly_sessions_all; create view hcp_billing_monthly_sessions_all as with @@ -107,4 +108,5 @@ begin; 'the sum of pending sessions for the current month and all previous months. ' 'The current month is a sum from the beginning of the current month ' 'until the start of the current hour (exclusive).'; + commit; diff --git a/internal/db/schema/migrations/oss/postgres/70/02_hcp_billing_hourly.up.sql b/internal/db/schema/migrations/oss/postgres/70/02_hcp_billing_hourly.up.sql new file mode 100644 index 0000000000..903bb65e30 --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/70/02_hcp_billing_hourly.up.sql @@ -0,0 +1,36 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + + -- replaces view from 64/02_hcp_billing_hourly.up.sql + drop view hcp_billing_hourly_sessions_all; + create view hcp_billing_hourly_sessions_all as + with + hourly_counts (hour, sessions_pending_count) as ( + select date_trunc('hour', session_pending_time), count(*) + from wh_session_accumulating_fact + group by date_trunc('hour', session_pending_time) + ), + hourly_range (hour) as ( + select bucket + from generate_series( + date_trunc('hour', ( select min(session_pending_time) from wh_session_accumulating_fact ) ), + now(), + '1 hour'::interval + ) as bucket + ), + final (hour, sessions_pending_count) as ( + select hourly_range.hour, coalesce(hourly_counts.sessions_pending_count, 0) + from hourly_range + left join hourly_counts on hourly_range.hour = hourly_counts.hour + ) + select hour, sessions_pending_count + from final + order by hour desc; + comment on view hcp_billing_hourly_sessions_all is + 'hcp_billing_hourly_sessions_all is a view where each row contains the timestamp for an hour and the sum of the pending sessions created in that hour. ' + 'A row is returned for each hour since the first session was created up to and including the current hour. ' + 'Rows are sorted by the hour in descending order.'; + +commit; diff --git a/internal/db/schema/migrations/oss/postgres/70/03_hcp_billing_monthly.up.sql b/internal/db/schema/migrations/oss/postgres/70/03_hcp_billing_monthly.up.sql new file mode 100644 index 0000000000..6ddab358cd --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/70/03_hcp_billing_monthly.up.sql @@ -0,0 +1,44 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + + -- replaces view from 64/03_hcp_billing_monthly.up.sql + drop view hcp_billing_monthly_sessions_all; + create view hcp_billing_monthly_sessions_all as + with + monthly_counts (month, sessions_pending_count) as ( + select date_trunc('month', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time < date_trunc('hour', now()) + group by date_trunc('month', session_pending_time) + ), + monthly_range (month) as ( + select bucket + from generate_series( + date_trunc('month', ( select min(session_pending_time) from wh_session_accumulating_fact ) ), + now(), + '1 month'::interval + ) as bucket + ), + final (start_time, end_time, sessions_pending_count) as ( + -- select monthly_range.month - interval '1 month', -- start + select monthly_range.month, -- start + case when monthly_range.month = date_trunc('month', now()) + then date_trunc('hour', now()) + else monthly_range.month + interval '1 month' + end, + coalesce(monthly_counts.sessions_pending_count, 0) + from monthly_range + left join monthly_counts on monthly_range.month = monthly_counts.month + ) + select start_time, end_time, sessions_pending_count + from final + order by start_time desc; + comment on view hcp_billing_monthly_sessions_all is + 'hcp_billing_monthly_sessions_all is a view that contains ' + 'the sum of pending sessions for the current month and all previous months. ' + 'The current month is a sum from the beginning of the current month ' + 'until the start of the current hour (exclusive).'; + +commit;