From ace2def49dce2971ce37db44d94faac0e2289bb7 Mon Sep 17 00:00:00 2001 From: Michael Milton Date: Wed, 24 May 2023 15:52:34 -0400 Subject: [PATCH] Add daily sessions pending view and accompanying snapshot table (#3217) * feat: add a new daily billing view * update views and table based on early feedback * address review comments * update CTE to be an accurate result of what we expect * remove 'with time zone' from daily range * remove timezone * Update internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql Co-authored-by: Michael Gaffney * Apply suggestions from code review Co-authored-by: Timothy Messier * go mod updates * fix tests --------- Co-authored-by: Michael Gaffney Co-authored-by: Timothy Messier --- .../postgres/70/01_hcp_billing_daily.up.sql | 64 ++++++++++++++++ .../tests/hcp/billing/daily_sessions_all.sql | 70 +++++++++++++++++ .../hcp/billing/daily_sessions_yesterday.sql | 75 +++++++++++++++++++ .../sessions_pending_daily_snapshot.sql | 28 +++++++ 4 files changed, 237 insertions(+) create mode 100644 internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql create mode 100644 internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql create mode 100644 internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql create mode 100644 internal/db/sqltest/tests/hcp/billing/sessions_pending_daily_snapshot.sql diff --git a/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql b/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql new file mode 100644 index 0000000000..372b06680d --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql @@ -0,0 +1,64 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + create table sessions_pending_daily_snapshot ( + snapshot_date date primary key, + sessions_pending_count bigint not null + ); + + create view hcp_billing_daily_sessions_yesterday as + with + daily_counts (day, sessions_pending_count) as ( + select date_trunc('day', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time >= date_trunc('day', timestamp 'yesterday') + and session_pending_time < date_trunc('day', timestamp 'today') + group by date_trunc('day', session_pending_time) + ), + daily_range (day) as ( + select date_trunc('day', timestamp 'yesterday') + ), + final (day, sessions_pending_count) as ( + select daily_range.day, coalesce(daily_counts.sessions_pending_count, 0) + from daily_range + left join daily_counts on daily_range.day = daily_counts.day + ) + select day, sessions_pending_count + from final + order by day desc; + comment on view hcp_billing_daily_sessions_yesterday is + 'hcp_billing_daily_sessions_yesterday is a view that contains ' + 'the sum of pending sessions ' + 'from the beginning of the previous day ' + 'until the start of the current day (exclusive).'; + + create view hcp_billing_daily_sessions_all as + with + daily_counts (day, sessions_pending_count) as ( + select date_trunc('day', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time < date_trunc('day', timestamp 'today') + group by date_trunc('day', session_pending_time) + ), + daily_range (day) as ( + select bucket + from generate_series( + date_trunc('day', (select min(session_pending_time) from wh_session_accumulating_fact)), + timestamp 'yesterday', + '1 day'::interval + ) as bucket + ), + final (day, sessions_pending_count) as ( + select daily_range.day::timestamp, + coalesce(daily_counts.sessions_pending_count, 0) + from daily_range + left join daily_counts on daily_range.day = daily_counts.day + ) + select day, sessions_pending_count + from final + order by day desc; + comment on view hcp_billing_daily_sessions_all is + 'hcp_billing_daily_sessions_all is a view that contains ' + 'the sum of pending sessions for yesterday and all previous days.'; +commit; diff --git a/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql b/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql new file mode 100644 index 0000000000..96426fe7f8 --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql @@ -0,0 +1,70 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; +select plan(9); + +select has_view('hcp_billing_daily_sessions_all', 'view for hcp billing does not exist'); + +select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + +-- validate the warehouse fact tables are empty +select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; +select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + +select is(count(*), 0::bigint, 'hcp_billing_daily_sessions_all should always return 0 rows when there are no sessions') from hcp_billing_daily_sessions_all; + +-- insert one session per minute for the past 171 hours +-- 171 is 7.125 days, should return 8 days +with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + now() - interval '171 hours', + now(), + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) +insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) +select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + +select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 rows') from hcp_billing_daily_sessions_yesterday; +select is(count(*), 7::bigint, 'hcp_billing_daily_sessions_all should return 7 rows') from hcp_billing_daily_sessions_all; + +select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_daily_sessions_all limit 1', + 'select 1440::bigint', + 'hcp_billing_daily_sessions_all: session count for the previous day is incorrect' + ); + +select results_eq( + 'select count(*)::bigint from wh_session_accumulating_fact', + 'select (select sum(sessions_pending_count)::bigint from hcp_billing_daily_sessions_all) + (select (select extract(minute from now())::bigint + 1) + (select extract(hour from now())::bigint * 60))', + 'hcp_billing_daily_sessions_all sum of sessions is incorrect' + ); + +select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql b/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql new file mode 100644 index 0000000000..03165f60d9 --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql @@ -0,0 +1,75 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; +select plan(9); + +select has_view('hcp_billing_daily_sessions_yesterday', 'view for hcp billing does not exist'); + +select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + +-- validate the warehouse fact tables are empty +select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; +select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + +select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday; + +-- insert one session per minute from one minute before midnight of yesterday until one minute past midnight today +-- 1442 total sessions +with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + timestamp 'yesterday' - interval '1 minute', + timestamp 'today' + interval '1 minute', + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) +insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) +select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + +select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday; + +select results_eq( + 'select count(*)::bigint from wh_session_accumulating_fact', + 'select sum(sessions_pending_count)::bigint + 3 from hcp_billing_daily_sessions_yesterday', + 'hcp_billing_daily_sessions_yesterday: the sum of sessions is incorrect' + ); + +select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_daily_sessions_yesterday limit 1', + 'select 1440::bigint', + 'hcp_billing_daily_sessions_yesterday: session count for the previous day is incorrect' + ); + +select results_eq( + 'select * from hcp_billing_daily_sessions_yesterday', + 'select * from hcp_billing_daily_sessions_all limit 1', + 'hcp_billing_daily_sessions_yesterday and hcp_billing_daily_sessions_all: latest day should be equal' + ); + +select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/sessions_pending_daily_snapshot.sql b/internal/db/sqltest/tests/hcp/billing/sessions_pending_daily_snapshot.sql new file mode 100644 index 0000000000..af331f2a5f --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/sessions_pending_daily_snapshot.sql @@ -0,0 +1,28 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(4); + + insert into sessions_pending_daily_snapshot + (snapshot_date, sessions_pending_count) + values + (timestamp 'yesterday', 100), + (timestamp 'yesterday' - interval '1 day', 2000); + + select is(count(*), 1::bigint) from sessions_pending_daily_snapshot where snapshot_date = timestamp 'yesterday'; + select is(count(*), 1::bigint) from sessions_pending_daily_snapshot where snapshot_date = timestamp 'yesterday' - interval '1 day'; + select results_eq( + 'select sessions_pending_count::bigint from sessions_pending_daily_snapshot limit 1', + 'select 100::bigint', + 'sessions_pending_daily_snapshot: session count for yesterday day is incorrect' + ); + + select results_eq( + 'select sum(sessions_pending_count)::bigint from sessions_pending_daily_snapshot', + 'select 2100::bigint', + 'sessions_pending_daily_snapshot sum of sessions is incorrect' + ); + + select * from finish(); +rollback;