diff --git a/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql b/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql index 372b06680d..228a569251 100644 --- a/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql +++ b/internal/db/schema/migrations/oss/postgres/70/01_hcp_billing_daily.up.sql @@ -2,63 +2,80 @@ -- SPDX-License-Identifier: MPL-2.0 begin; + create table sessions_pending_daily_snapshot ( snapshot_date date primary key, sessions_pending_count bigint not null + constraint sessions_pending_count_must_be_zero_or_positive + check(sessions_pending_count >= 0) ); + comment on table sessions_pending_daily_snapshot is + 'sessions_pending_count is a table where each row contains the count of ' + 'sessions pending for snapshot_date for that date.'; + + create function update_sessions_pending_daily_snapshot() + returns setof sessions_pending_daily_snapshot + as $$ + begin - create view hcp_billing_daily_sessions_yesterday as - with - daily_counts (day, sessions_pending_count) as ( - select date_trunc('day', session_pending_time), count(*) - from wh_session_accumulating_fact - where session_pending_time >= date_trunc('day', timestamp 'yesterday') - and session_pending_time < date_trunc('day', timestamp 'today') - group by date_trunc('day', session_pending_time) - ), - daily_range (day) as ( - select date_trunc('day', timestamp 'yesterday') - ), - final (day, sessions_pending_count) as ( - select daily_range.day, coalesce(daily_counts.sessions_pending_count, 0) - from daily_range - left join daily_counts on daily_range.day = daily_counts.day - ) - select day, sessions_pending_count - from final - order by day desc; - comment on view hcp_billing_daily_sessions_yesterday is - 'hcp_billing_daily_sessions_yesterday is a view that contains ' - 'the sum of pending sessions ' - 'from the beginning of the previous day ' - 'until the start of the current day (exclusive).'; + -- already ran for today + if (date_trunc('day', now()) - '1 day'::interval) = (select max(snapshot_date) from sessions_pending_daily_snapshot) + then return; + end if; - create view hcp_billing_daily_sessions_all as - with - daily_counts (day, sessions_pending_count) as ( - select date_trunc('day', session_pending_time), count(*) - from wh_session_accumulating_fact - where session_pending_time < date_trunc('day', timestamp 'today') - group by date_trunc('day', session_pending_time) - ), - daily_range (day) as ( - select bucket - from generate_series( - date_trunc('day', (select min(session_pending_time) from wh_session_accumulating_fact)), - timestamp 'yesterday', - '1 day'::interval + -- never run before and there are only sessions starting from today + if (select count(*) from sessions_pending_daily_snapshot) = 0 + and date_trunc('day', now()) = (select min(session_pending_time) from wh_session_accumulating_fact) + then return query + insert into sessions_pending_daily_snapshot + (snapshot_date, sessions_pending_count) + values + (date_trunc('day', now()) - '1 day'::interval, 0) + returning *; + return; + end if; + + return query + with + daily_counts (day, sessions_pending_count) as ( + select date_trunc('day', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time < date_trunc('day', now()) -- before midnight today + and session_pending_time >= coalesce((select max(snapshot_date) from sessions_pending_daily_snapshot), '-infinity') + group by date_trunc('day', session_pending_time) + ), + daily_range (day) as ( + select bucket + from generate_series( + coalesce(date_trunc('day', (select min(session_pending_time) from wh_session_accumulating_fact)), + date_trunc('day', now()) - '1 day'::interval), + now() - '1 day'::interval, + '1 day'::interval ) as bucket - ), - final (day, sessions_pending_count) as ( - select daily_range.day::timestamp, - coalesce(daily_counts.sessions_pending_count, 0) - from daily_range - left join daily_counts on daily_range.day = daily_counts.day - ) - select day, sessions_pending_count - from final - order by day desc; - comment on view hcp_billing_daily_sessions_all is - 'hcp_billing_daily_sessions_all is a view that contains ' - 'the sum of pending sessions for yesterday and all previous days.'; + ), + missing (day, sessions_pending_count) as ( + select daily_range.day::timestamp with time zone, + coalesce(daily_counts.sessions_pending_count, 0) + from daily_range + left join daily_counts on daily_range.day = daily_counts.day + ), + final (day, sessions_pending_count) as ( + insert into sessions_pending_daily_snapshot + (snapshot_date, sessions_pending_count) + select day::date, sessions_pending_count + from missing + returning * + ) + select day, sessions_pending_count + from final + order by day desc; + end; + $$ language plpgsql + set timezone to 'utc'; + comment on function update_sessions_pending_daily_snapshot is + 'update_sessions_pending_daily_snapshot is a function that updates the sessions_pending_daily_snapshot table by ' + 'querying the data warehouse and inserting the session pending counts for any days since the max snapshot_date ' + 'and yesterday. ' + 'update_sessions_pending_daily_snapshot returns the rows inserted or null if no rows are inserted.'; + commit; diff --git a/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql b/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql deleted file mode 100644 index 96426fe7f8..0000000000 --- a/internal/db/sqltest/tests/hcp/billing/daily_sessions_all.sql +++ /dev/null @@ -1,70 +0,0 @@ --- Copyright (c) HashiCorp, Inc. --- SPDX-License-Identifier: MPL-2.0 - -begin; -select plan(9); - -select has_view('hcp_billing_daily_sessions_all', 'view for hcp billing does not exist'); - -select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', - 'Truncate tables in preparation for testing'); - --- validate the warehouse fact tables are empty -select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; -select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; - -select is(count(*), 0::bigint, 'hcp_billing_daily_sessions_all should always return 0 rows when there are no sessions') from hcp_billing_daily_sessions_all; - --- insert one session per minute for the past 171 hours --- 171 is 7.125 days, should return 8 days -with - dim_keys (host_key, user_key, credential_group_key) as ( - select h.key, u.key, 'no credentials' - from (select key from wh_host_dimension limit 1) as h, - (select key from wh_user_dimension limit 1) as u - ), - time_series (date_key, time_key, time) as ( - select wh_date_key(time), wh_time_key(time), time - from generate_series( - now() - interval '171 hours', - now(), - interval '1 minute' - ) as time - ), - fake_sessions (session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time) as ( - select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), - k.host_key, k.user_key, k.credential_group_key, - t.date_key, t.time_key,t.time - from dim_keys as k, - time_series as t - ) -insert into wh_session_accumulating_fact - (session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time - ) -select session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time - from fake_sessions; - -select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 rows') from hcp_billing_daily_sessions_yesterday; -select is(count(*), 7::bigint, 'hcp_billing_daily_sessions_all should return 7 rows') from hcp_billing_daily_sessions_all; - -select results_eq( - 'select sessions_pending_count::bigint from hcp_billing_daily_sessions_all limit 1', - 'select 1440::bigint', - 'hcp_billing_daily_sessions_all: session count for the previous day is incorrect' - ); - -select results_eq( - 'select count(*)::bigint from wh_session_accumulating_fact', - 'select (select sum(sessions_pending_count)::bigint from hcp_billing_daily_sessions_all) + (select (select extract(minute from now())::bigint + 1) + (select extract(hour from now())::bigint * 60))', - 'hcp_billing_daily_sessions_all sum of sessions is incorrect' - ); - -select * from finish(); - -rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql b/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql deleted file mode 100644 index 03165f60d9..0000000000 --- a/internal/db/sqltest/tests/hcp/billing/daily_sessions_yesterday.sql +++ /dev/null @@ -1,75 +0,0 @@ --- Copyright (c) HashiCorp, Inc. --- SPDX-License-Identifier: MPL-2.0 - -begin; -select plan(9); - -select has_view('hcp_billing_daily_sessions_yesterday', 'view for hcp billing does not exist'); - -select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', - 'Truncate tables in preparation for testing'); - --- validate the warehouse fact tables are empty -select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; -select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; - -select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday; - --- insert one session per minute from one minute before midnight of yesterday until one minute past midnight today --- 1442 total sessions -with - dim_keys (host_key, user_key, credential_group_key) as ( - select h.key, u.key, 'no credentials' - from (select key from wh_host_dimension limit 1) as h, - (select key from wh_user_dimension limit 1) as u - ), - time_series (date_key, time_key, time) as ( - select wh_date_key(time), wh_time_key(time), time - from generate_series( - timestamp 'yesterday' - interval '1 minute', - timestamp 'today' + interval '1 minute', - interval '1 minute' - ) as time - ), - fake_sessions (session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time) as ( - select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), - k.host_key, k.user_key, k.credential_group_key, - t.date_key, t.time_key,t.time - from dim_keys as k, - time_series as t - ) -insert into wh_session_accumulating_fact - (session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time - ) -select session_id, auth_token_id, - host_key, user_key, credential_group_key, - session_pending_date_key, session_pending_time_key, session_pending_time - from fake_sessions; - -select is(count(*), 1::bigint, 'hcp_billing_daily_sessions_yesterday should always return 1 row') from hcp_billing_daily_sessions_yesterday; - -select results_eq( - 'select count(*)::bigint from wh_session_accumulating_fact', - 'select sum(sessions_pending_count)::bigint + 3 from hcp_billing_daily_sessions_yesterday', - 'hcp_billing_daily_sessions_yesterday: the sum of sessions is incorrect' - ); - -select results_eq( - 'select sessions_pending_count::bigint from hcp_billing_daily_sessions_yesterday limit 1', - 'select 1440::bigint', - 'hcp_billing_daily_sessions_yesterday: session count for the previous day is incorrect' - ); - -select results_eq( - 'select * from hcp_billing_daily_sessions_yesterday', - 'select * from hcp_billing_daily_sessions_all limit 1', - 'hcp_billing_daily_sessions_yesterday and hcp_billing_daily_sessions_all: latest day should be equal' - ); - -select * from finish(); - -rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/update_sessions_pending_daily_snapshot.sql b/internal/db/sqltest/tests/hcp/billing/update_sessions_pending_daily_snapshot.sql new file mode 100644 index 0000000000..654e2ddb22 --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/update_sessions_pending_daily_snapshot.sql @@ -0,0 +1,116 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(27); + + select has_function('update_sessions_pending_daily_snapshot'); + select volatility_is('update_sessions_pending_daily_snapshot', 'volatile'); + select isnt_strict('update_sessions_pending_daily_snapshot'); + + prepare call_update_sessions_pending_daily_snapshot + as select * from update_sessions_pending_daily_snapshot(); + + create function test_add_session(ts timestamptz) returns void + as $$ + with time_series (time) as ( + select ts + ), + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + dim_time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from time_series + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + dim_time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + $$ language sql; + + create function today() returns timestamptz + as $$ + select date_trunc('day', now(), 'utc'); + $$ language sql; + + create function yesterday() returns timestamptz + as $$ + select today() - '1 day'::interval; + $$ language sql; + + create function has_empty_table(table_name name) returns text + as $$ + declare + result text; + begin + execute format('select is(count(*), 0::bigint) from %I', table_name) into result; + return result; + end; + $$ language plpgsql; + + create table test_table_data ( + snapshot_date date primary key, + sessions_pending_count bigint not null + ); + prepare select_test_table_data as select * from test_table_data order by snapshot_date desc; + + -- add 5 to plan every time this is called + create function reset_data() returns text + as $$ + select * from collect_tap( + lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact, sessions_pending_daily_snapshot, test_table_data'), + has_empty_table('wh_session_connection_accumulating_fact'), + has_empty_table('wh_session_accumulating_fact'), + has_empty_table('sessions_pending_daily_snapshot'), + has_empty_table('test_table_data') + ); + $$ language sql; + + -- new install, no sessions + select reset_data(); + -- no sessions + insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 0; + select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data'); + select is(t.*, null, 'update_sessions_pending_daily_snapshot should return null when it has already been run for the day') from update_sessions_pending_daily_snapshot() as t; + + -- new install, only sessions are for today + select reset_data(); + select test_add_session(today()); + insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 0; + select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data'); + + -- upgrade install, sessions are for today and yesterday + select reset_data(); + select test_add_session(yesterday()); + select test_add_session(today()); + insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 1; + select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data'); + + -- upgrade install, sessions are for today, yesterday, and 3 days ago + select reset_data(); + select test_add_session(yesterday() - '1 day'::interval); + select test_add_session(yesterday()); + select test_add_session(today()); + insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date - '1 day'::interval, 1; + insert into test_table_data (snapshot_date, sessions_pending_count) select yesterday()::date, 1; + select results_eq('call_update_sessions_pending_daily_snapshot', 'select_test_table_data'); + + select * from finish(); +rollback;