diff --git a/internal/db/schema/migrations/oss/postgres/64/01_ssh_targets.up.sql b/internal/db/schema/migrations/oss/postgres/64/01_ssh_targets.up.sql new file mode 100644 index 0000000000..9055b00f7d --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/64/01_ssh_targets.up.sql @@ -0,0 +1,364 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + create table if not exists target_ssh ( + public_id wt_public_id primary key + constraint target_fkey + references target(public_id) + on delete cascade + on update cascade, + project_id wt_scope_id not null, + name text not null, -- name is not optional for a target subtype + description text, + default_port int, -- default_port can be null + -- max duration of the session in seconds. + -- default is 8 hours + session_max_seconds int not null default 28800 + constraint session_max_seconds_must_be_greater_than_0 + check(session_max_seconds > 0), + -- limit on number of session connections allowed. -1 equals no limit + session_connection_limit int not null default 1 + constraint session_connection_limit_must_be_greater_than_0_or_negative_1 + check(session_connection_limit > 0 or session_connection_limit = -1), + create_time wt_timestamp, + update_time wt_timestamp, + version wt_version, + worker_filter wt_bexprfilter, + egress_worker_filter wt_bexprfilter, + ingress_worker_filter wt_bexprfilter, + constraint target_ssh_project_id_name_uq + unique(project_id, name) -- name must be unique within a project scope. + ); + comment on table target_ssh is + 'target_ssh is a table where each row is a resource that represents an ssh target. ' + 'It is a target subtype.'; + + drop trigger if exists insert_target_subtype on target_ssh; + create trigger insert_target_subtype before insert on target_ssh + for each row execute procedure insert_target_subtype(); + + drop trigger if exists delete_target_subtype on target_ssh; + create trigger delete_target_subtype after delete on target_ssh + for each row execute procedure delete_target_subtype(); + + -- define the immutable fields for target + drop trigger if exists immutable_columns on target_ssh; + create trigger immutable_columns before update on target_ssh + for each row execute procedure immutable_columns('public_id', 'project_id', 'create_time'); + + drop trigger if exists update_version_column on target_ssh; + create trigger update_version_column after update on target_ssh + for each row execute procedure update_version_column(); + + drop trigger if exists update_time_column on target_ssh; + create trigger update_time_column before update on target_ssh + for each row execute procedure update_time_column(); + + drop trigger if exists default_create_time_column on target_ssh; + create trigger default_create_time_column before insert on target_ssh + for each row execute procedure default_create_time(); + + drop trigger if exists update_ssh_target_filter_validate on target_ssh; + create trigger update_ssh_target_filter_validate before update on target_ssh + for each row execute procedure validate_filter_values_on_update(); + + drop trigger if exists insert_ssh_target_filter_validate on target_ssh; + create trigger insert_ssh_target_filter_validate before insert on target_ssh + for each row execute procedure validate_filter_values_on_insert(); + + insert into oplog_ticket + (name, version) + values + ('target_ssh', 1) + on conflict do nothing; + + -- The whx_* views here depend on target_all_subtypes, so we need to drop + -- these first. + drop view if exists whx_host_dimension_source; + drop view if exists whx_credential_dimension_source; + drop view if exists target_all_subtypes; + + create view target_all_subtypes as + select + public_id, + project_id, + name, + description, + default_port, + session_max_seconds, + session_connection_limit, + version, + create_time, + update_time, + worker_filter, + egress_worker_filter, + ingress_worker_filter, + 'tcp' as type + from target_tcp + union + select + public_id, + project_id, + name, + description, + default_port, + session_max_seconds, + session_connection_limit, + version, + create_time, + update_time, + worker_filter, + egress_worker_filter, + ingress_worker_filter, + 'ssh' as type + from + target_ssh; + + -- replaces view from oss/60/03_wh_sessions.up.sql + create view whx_host_dimension_source as + with + host_sources ( + host_id, host_type, host_name, host_description, + host_set_id, host_set_type, host_set_name, host_set_description, + host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, + target_id, target_type, target_name, target_description, + target_default_port_number, target_session_max_seconds, target_session_connection_limit, + project_id, project_name, project_description, + organization_id, organization_name, organization_description + ) as ( + select -- id is the first column in the target view + h.public_id as host_id, + case when sh.public_id is not null then 'static host' + when ph.public_id is not null then 'plugin host' + else 'Unknown' end as host_type, + case when sh.public_id is not null then coalesce(sh.name, 'None') + when ph.public_id is not null then coalesce(ph.name, 'None') + else 'Unknown' end as host_name, + case when sh.public_id is not null then coalesce(sh.description, 'None') + when ph.public_id is not null then coalesce(ph.description, 'None') + else 'Unknown' end as host_description, + hs.public_id as host_set_id, + case when shs.public_id is not null then 'static host set' + when phs.public_id is not null then 'plugin host set' + else 'Unknown' end as host_set_type, + case + when shs.public_id is not null then coalesce(shs.name, 'None') + when phs.public_id is not null then coalesce(phs.name, 'None') + else 'None' + end as host_set_name, + case + when shs.public_id is not null then coalesce(shs.description, 'None') + when phs.public_id is not null then coalesce(phs.description, 'None') + else 'None' + end as host_set_description, + hc.public_id as host_catalog_id, + case when shc.public_id is not null then 'static host catalog' + when phc.public_id is not null then 'plugin host catalog' + else 'Unknown' end as host_catalog_type, + case + when shc.public_id is not null then coalesce(shc.name, 'None') + when phc.public_id is not null then coalesce(phc.name, 'None') + else 'None' + end as host_catalog_name, + case + when shc.public_id is not null then coalesce(shc.description, 'None') + when phc.public_id is not null then coalesce(phc.description, 'None') + else 'None' + end as host_catalog_description, + t.public_id as target_id, + case + when t.type = 'tcp' then 'tcp target' + when t.type = 'ssh' then 'ssh target' + else 'Unknown' + end as target_type, + coalesce(t.name, 'None') as target_name, + coalesce(t.description, 'None') as target_description, + coalesce(t.default_port, 0) as target_default_port_number, + t.session_max_seconds as target_session_max_seconds, + t.session_connection_limit as target_session_connection_limit, + p.public_id as project_id, + coalesce(p.name, 'None') as project_name, + coalesce(p.description, 'None') as project_description, + o.public_id as organization_id, + coalesce(o.name, 'None') as organization_name, + coalesce(o.description, 'None') as organization_description + from host as h + join host_catalog as hc on h.catalog_id = hc.public_id + join host_set as hs on h.catalog_id = hs.catalog_id + join target_host_set as ts on hs.public_id = ts.host_set_id + join target_all_subtypes as t on ts.target_id = t.public_id + join iam_scope as p on t.project_id = p.public_id and p.type = 'project' + join iam_scope as o on p.parent_id = o.public_id and o.type = 'org' + + left join static_host as sh on sh.public_id = h.public_id + left join host_plugin_host as ph on ph.public_id = h.public_id + left join static_host_catalog as shc on shc.public_id = hc.public_id + left join host_plugin_catalog as phc on phc.public_id = hc.public_id + left join static_host_set as shs on shs.public_id = hs.public_id + left join host_plugin_set as phs on phs.public_id = hs.public_id + ), + host_target_address ( + host_id, host_type, host_name, host_description, + host_set_id, host_set_type, host_set_name, host_set_description, + host_catalog_id, host_catalog_type, host_catalog_name, host_catalog_description, + target_id, target_type, target_name, target_description, + target_default_port_number, target_session_max_seconds, target_session_connection_limit, + project_id, project_name, project_description, + organization_id, organization_name, organization_description + ) as ( + select + 'Not Applicable' as host_id, + 'direct address' as host_type, + 'Not Applicable' as host_name, + 'Not Applicable' as host_description, + 'Not Applicable' as host_set_id, + 'Not Applicable' as host_set_type, + 'Not Applicable' as host_set_name, + 'Not Applicable' as host_set_description, + 'Not Applicable' as host_catalog_id, + 'Not Applicable' as host_catalog_type, + 'Not Applicable' as host_catalog_name, + 'Not Applicable' as host_catalog_description, + t.public_id as target_id, + case + when t.type = 'tcp' then 'tcp target' + when t.type = 'ssh' then 'ssh target' + else 'Unknown' + end as target_type, + coalesce(t.name, 'None') as target_name, + coalesce(t.description, 'None') as target_description, + coalesce(t.default_port, 0) as target_default_port_number, + t.session_max_seconds as target_session_max_seconds, + t.session_connection_limit as target_session_connection_limit, + p.public_id as project_id, + coalesce(p.name, 'None') as project_name, + coalesce(p.description, 'None') as project_description, + o.public_id as organization_id, + coalesce(o.name, 'None') as organization_name, + coalesce(o.description, 'None') as organization_description + from target_all_subtypes as t + right join target_address as ta on t.public_id = ta.target_id + left join iam_scope as p on p.public_id = t.project_id + left join iam_scope as o on o.public_id = p.parent_id + ) + select * from host_sources + union + select * from host_target_address; + + -- The whx_credential_dimension_source view shows the current values in the + -- operational tables of the credential dimension. + -- Replaces whx_credential_dimension_source defined in oss/63/03_wh_ssh_cert_library.up.sql + create view whx_credential_dimension_source as + with vault_generic_library as ( + select vcl.public_id as public_id, + 'vault generic credential library' as type, + coalesce(vcl.name, 'None') as name, + coalesce(vcl.description, 'None') as description, + vcl.vault_path as vault_path, + vcl.http_method as http_method, + case + when vcl.http_method = 'GET' then 'Not Applicable' + else coalesce(vcl.http_request_body::text, 'None') + end as http_request_body, + 'Not Applicable' as username, + 'Not Applicable' as key_type_and_bits + from credential_vault_library as vcl + ), + vault_ssh_cert_library as ( + select vsccl.public_id as public_id, + 'vault ssh certificate credential library' as type, + coalesce(vsccl.name, 'None') as name, + coalesce(vsccl.description, 'None') as description, + vsccl.vault_path as vault_path, + 'Not Applicable' as http_method, + 'Not Applicable' as http_request_body, + vsccl.username as username, + case + when vsccl.key_type = 'ed25519' then vsccl.key_type + else vsccl.key_type || '-' || vsccl.key_bits::text + end as key_type_and_bits + from credential_vault_ssh_cert_library as vsccl + ), + final as ( + select s.public_id as session_id, + scd.credential_purpose as credential_purpose, + cl.public_id as credential_library_id, + coalesce(vcl.type, vsccl.type) as credential_library_type, + coalesce(vcl.name, vsccl.name) as credential_library_name, + coalesce(vcl.description, vsccl.description) as credential_library_description, + coalesce(vcl.vault_path, vsccl.vault_path) as credential_library_vault_path, + coalesce(vcl.http_method, vsccl.http_method) as credential_library_vault_http_method, + coalesce(vcl.http_request_body, vsccl.http_request_body) as credential_library_vault_http_request_body, + coalesce(vcl.username, vsccl.username) as credential_library_username, + coalesce(vcl.key_type_and_bits, vsccl.key_type_and_bits) as credential_library_key_type_and_bits, + cs.public_id as credential_store_id, + case + when vcs is null then 'None' + else 'vault credential store' + end as credential_store_type, + coalesce(vcs.name, 'None') as credential_store_name, + coalesce(vcs.description, 'None') as credential_store_description, + coalesce(vcs.namespace, 'None') as credential_store_vault_namespace, + coalesce(vcs.vault_address, 'None') as credential_store_vault_address, + t.public_id as target_id, + case + when tt.type = 'tcp' then 'tcp target' + when tt.type = 'ssh' then 'ssh target' + else 'Unknown' + end as target_type, + coalesce(tt.name, 'None') as target_name, + coalesce(tt.description, 'None') as target_description, + coalesce(tt.default_port, 0) as target_default_port_number, + tt.session_max_seconds as target_session_max_seconds, + tt.session_connection_limit as target_session_connection_limit, + p.public_id as project_id, + coalesce(p.name, 'None') as project_name, + coalesce(p.description, 'None') as project_description, + o.public_id as organization_id, + coalesce(o.name, 'None') as organization_name, + coalesce(o.description, 'None') as organization_description + from session_credential_dynamic as scd + join session as s on scd.session_id = s.public_id + join credential_library as cl on scd.library_id = cl.public_id + join credential_store as cs on cl.store_id = cs.public_id + join target as t on s.target_id = t.public_id + join iam_scope as p on p.public_id = t.project_id and p.type = 'project' + join iam_scope as o on p.parent_id = o.public_id and o.type = 'org' + left join vault_generic_library as vcl on cl.public_id = vcl.public_id + left join vault_ssh_cert_library as vsccl on cl.public_id = vsccl.public_id + left join credential_vault_store as vcs on cs.public_id = vcs.public_id + left join target_all_subtypes as tt on t.public_id = tt.public_id + ) + select session_id, + credential_purpose, + credential_library_id, + credential_library_type, + credential_library_name, + credential_library_description, + credential_library_vault_path, + credential_library_vault_http_method, + credential_library_vault_http_request_body, + credential_library_username, + credential_library_key_type_and_bits, + credential_store_id, + credential_store_type, + credential_store_name, + credential_store_description, + credential_store_vault_namespace, + credential_store_vault_address, + target_id, + target_type, + target_name, + target_description, + target_default_port_number, + target_session_max_seconds, + target_session_connection_limit, + project_id, + project_name, + project_description, + organization_id, + organization_name, + organization_description + from final; +commit; diff --git a/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql b/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql new file mode 100644 index 0000000000..e80ec96f92 --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/64/02_hcp_billing_hourly.up.sql @@ -0,0 +1,158 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + drop index if exists wh_session_accumulating_fact_session_pending_time_idx; + create index wh_session_accumulating_fact_session_pending_time_idx on wh_session_accumulating_fact (session_pending_time); + analyze wh_session_accumulating_fact; + +/* + Implementation Note 1 + + SQL is very flexible. I'm 87% sure there are approximately 1,962 different + ways to write the queries for each of the views below (plus or minus 2). I + chose the ones below based on the explain plans for each view after filling + the wh_session_accumulating_fact table with more than 250,000 rows of test + data. + + The query I used to populate the test data is at the bottom of this file. If + you need to alter these views, please generate test data and use the explain + plan to guide your decisions. +*/ + + drop view if exists hcp_billing_hourly_sessions_last_3_hours; + create view hcp_billing_hourly_sessions_last_3_hours as + with + hourly_counts (hour, sessions_pending_count) as ( + select date_trunc('hour', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time >= date_trunc('hour', now() - '3 hours'::interval) + group by date_trunc('hour', session_pending_time) + ), + hourly_range (hour) as ( + select date_trunc('hour',time) + from generate_series(now() - '3 hours'::interval, now(), '1 hour'::interval) as time + ), + final (hour, sessions_pending_count) as ( + select hourly_range.hour, coalesce(hourly_counts.sessions_pending_count, 0) + from hourly_range + left join hourly_counts on hourly_range.hour = hourly_counts.hour + ) + select hour, sessions_pending_count + from final + order by hour desc; + comment on view hcp_billing_hourly_sessions_last_3_hours is + 'hcp_billing_hourly_sessions_last_3_hours is a view where each row contains the timestamp for an hour and the sum of the pending sessions created in that hour. ' + '4 rows are returned: 1 for the current hour plus 3 for the previous 3 hours. ' + 'Rows are sorted by the hour in descending order.'; + + drop view if exists hcp_billing_hourly_sessions_last_7_days; + create view hcp_billing_hourly_sessions_last_7_days as + with + hourly_counts (hour, sessions_pending_count) as ( + select date_trunc('hour', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time >= date_trunc('hour', now() - '7 days'::interval) + group by date_trunc('hour', session_pending_time) + ), + hourly_range (hour) as ( + select date_trunc('hour',time) + from generate_series(now() - '7 days'::interval, now(), '1 hour'::interval) as time + ), + final (hour, sessions_pending_count) as ( + select hourly_range.hour, coalesce(hourly_counts.sessions_pending_count, 0) + from hourly_range + left join hourly_counts on hourly_range.hour = hourly_counts.hour + ) + select hour, sessions_pending_count + from final + order by hour desc; + comment on view hcp_billing_hourly_sessions_last_7_days is + 'hcp_billing_hourly_sessions_last_7_days is a view where each row contains the timestamp for an hour and the sum of the pending sessions created in that hour. ' + '169 rows are returned: 1 for the current hour plus 168 for the previous 7 days. ' + 'Rows are sorted by the hour in descending order.'; + + drop view if exists hcp_billing_hourly_sessions_all; + create view hcp_billing_hourly_sessions_all as + with + hourly_counts (hour, sessions_pending_count) as ( + select date_trunc('hour', session_pending_time), count(*) + from wh_session_accumulating_fact + group by date_trunc('hour', session_pending_time) + ), + hourly_range (hour) as ( + select date_trunc('hour',time) + from generate_series( + (select min(session_pending_time) from wh_session_accumulating_fact), + now(), + '1 hour'::interval + ) as time + ), + final (hour, sessions_pending_count) as ( + select hourly_range.hour, coalesce(hourly_counts.sessions_pending_count, 0) + from hourly_range + left join hourly_counts on hourly_range.hour = hourly_counts.hour + ) + select hour, sessions_pending_count + from final + order by hour desc; + comment on view hcp_billing_hourly_sessions_all is + 'hcp_billing_hourly_sessions_all is a view where each row contains the timestamp for an hour and the sum of the pending sessions created in that hour. ' + 'A row is returned for each hour since the first session was created up to and including the current hour. ' + 'Rows are sorted by the hour in descending order.'; + +/* + Implementation Note 2: Generating test data + + Step 1: Start and connect to the sqltest docker container (see sqltest/README.md). + + The sqltest container initializes with a few sessions already in the data + warehouse. These sessions will populate rows in the wh_host_dimension and + wh_user_dimension tables which will be used in the query below. + + Step 2: Truncate the warehouse fact tables: + + truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact; + + Step 3: Fill the wh_session_accumulating_fact table with 261961 rows of data: + + with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + now() - interval '6 months', + now() - interval '2 hours', + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select substr(md5(random()::text), 0, 15), substr(md5(random()::text), 0, 15), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + + Step 4: Collect the execution plan for each view to establish a baseline before making changes: + + explain (analyze, buffers) select * from hcp_billing_hourly_sessions_last_3_hours; + explain (analyze, buffers) select * from hcp_billing_hourly_sessions_last_7_days; + explain (analyze, buffers) select * from hcp_billing_hourly_sessions_all; +*/ +commit; diff --git a/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql b/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql new file mode 100644 index 0000000000..665029345e --- /dev/null +++ b/internal/db/schema/migrations/oss/postgres/64/03_hcp_billing_monthly.up.sql @@ -0,0 +1,110 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + drop view if exists hcp_billing_monthly_sessions_current_month; + create view hcp_billing_monthly_sessions_current_month as + with + monthly_counts (month, sessions_pending_count) as ( + select date_trunc('month', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time >= date_trunc('month', now()) + and session_pending_time < date_trunc('hour', now()) + group by date_trunc('month', session_pending_time) + ), + monthly_range (month) as ( + select date_trunc('month',now()) + ), + final (start_time, end_time, sessions_pending_count) as ( + select monthly_range.month, -- start + case when monthly_range.month = date_trunc('month', now()) + then date_trunc('hour', now()) + else monthly_range.month + interval '1 month' + end, + coalesce(monthly_counts.sessions_pending_count, 0) + from monthly_range + left join monthly_counts on monthly_range.month = monthly_counts.month + ) + select start_time, end_time, sessions_pending_count + from final + order by start_time desc; + comment on view hcp_billing_monthly_sessions_current_month is + 'hcp_billing_monthly_sessions_current_month is a view that contains ' + 'the sum of pending sessions ' + 'from the beginning of the current month ' + 'until the start of the current hour (exclusive).'; + + drop view if exists hcp_billing_monthly_sessions_last_2_months; + create view hcp_billing_monthly_sessions_last_2_months as + with + monthly_counts (month, sessions_pending_count) as ( + select date_trunc('month', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time >= date_trunc('month', now() - interval '1 month') + and session_pending_time < date_trunc('hour', now()) + group by date_trunc('month', session_pending_time) + ), + monthly_range (month) as ( + select date_trunc('month', time) + from generate_series( + date_trunc('month', now() - interval '1 month'), + now(), + '1 month'::interval + ) as time + ), + final (start_time, end_time, sessions_pending_count) as ( + select monthly_range.month, -- start + case when monthly_range.month = date_trunc('month', now()) + then date_trunc('hour', now()) + else monthly_range.month + interval '1 month' + end, + coalesce(monthly_counts.sessions_pending_count, 0) + from monthly_range + left join monthly_counts on monthly_range.month = monthly_counts.month + ) + select start_time, end_time, sessions_pending_count + from final + order by start_time desc; + comment on view hcp_billing_monthly_sessions_last_2_months is + 'hcp_billing_monthly_sessions_last_2_months is a view that contains ' + 'the sum of pending sessions for the current month and the previous month. ' + 'The current month is a sum from the beginning of the current month ' + 'until the start of the current hour (exclusive).'; + + drop view if exists hcp_billing_monthly_sessions_all; + create view hcp_billing_monthly_sessions_all as + with + monthly_counts (month, sessions_pending_count) as ( + select date_trunc('month', session_pending_time), count(*) + from wh_session_accumulating_fact + where session_pending_time < date_trunc('hour', now()) + group by date_trunc('month', session_pending_time) + ), + monthly_range (month) as ( + select date_trunc('month',time) + from generate_series( + (select min(session_pending_time) from wh_session_accumulating_fact), + now(), + '1 month'::interval + ) as time + ), + final (start_time, end_time, sessions_pending_count) as ( + -- select monthly_range.month - interval '1 month', -- start + select monthly_range.month, -- start + case when monthly_range.month = date_trunc('month', now()) + then date_trunc('hour', now()) + else monthly_range.month + interval '1 month' + end, + coalesce(monthly_counts.sessions_pending_count, 0) + from monthly_range + left join monthly_counts on monthly_range.month = monthly_counts.month + ) + select start_time, end_time, sessions_pending_count + from final + order by start_time desc; + comment on view hcp_billing_monthly_sessions_all is + 'hcp_billing_monthly_sessions_all is a view that contains ' + 'the sum of pending sessions for the current month and all previous months. ' + 'The current month is a sum from the beginning of the current month ' + 'until the start of the current hour (exclusive).'; +commit; diff --git a/internal/db/sqltest/Makefile b/internal/db/sqltest/Makefile index c813725bb1..b149746f95 100644 --- a/internal/db/sqltest/Makefile +++ b/internal/db/sqltest/Makefile @@ -26,6 +26,7 @@ TESTS ?= tests/setup/*.sql \ tests/account/*/*.sql \ tests/target/*.sql \ tests/controller/*.sql \ + tests/hcp/*/*.sql \ tests/kms/*.sql POSTGRES_DOCKER_IMAGE_BASE ?= postgres diff --git a/internal/db/sqltest/tests/hcp/billing/hourly_sessions_all.sql b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_all.sql new file mode 100644 index 0000000000..0b177f18f9 --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_all.sql @@ -0,0 +1,84 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(12); + + select has_view('hcp_billing_hourly_sessions_all', 'view for hcp billing does not exist'); + + select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + + -- validate the warehouse fact tables are empty + select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; + select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + + select is(count(*), 0::bigint, 'hcp_billing_hourly_sessions_all should always return 0 rows when there are no sessions') from hcp_billing_hourly_sessions_all; + + -- insert one session per minute for the past 171 hours + -- 171 is 2 more than the hcp_billing_hourly_sessions_last_7_days view should return + -- total is 10,261 = 60 minutes * 171 + 1 for the current minute + with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + now() - interval '171 hours', + now(), + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + + select is(count(*), 4::bigint, 'hcp_billing_hourly_sessions_last_3_hours should always return 4 rows') from hcp_billing_hourly_sessions_last_3_hours; + select is(count(*), 169::bigint, 'hcp_billing_hourly_sessions_last_7_days should always return 169 rows') from hcp_billing_hourly_sessions_last_7_days; + select is(count(*), 172::bigint, 'hcp_billing_hourly_sessions_all should return 172 rows') from hcp_billing_hourly_sessions_all; + + select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_hourly_sessions_all limit 1', + 'select extract(minute from now())::bigint + 1', + 'hcp_billing_hourly_sessions_all: session count for the current hour is incorrect' + ); + + select results_eq( + 'select count(*)::bigint from wh_session_accumulating_fact', + 'select sum(sessions_pending_count)::bigint from hcp_billing_hourly_sessions_all', + 'hcp_billing_hourly_sessions_all sum of sessions is incorrect' + ); + + select results_eq( + 'select * from hcp_billing_hourly_sessions_all limit 4', + 'select * from hcp_billing_hourly_sessions_last_3_hours', + 'hcp_billing_hourly_sessions_all and hcp_billing_hourly_sessions_last_3_hours: latest 3 hours should be equal' + ); + + select results_eq( + 'select * from hcp_billing_hourly_sessions_all limit 169', + 'select * from hcp_billing_hourly_sessions_last_7_days', + 'hcp_billing_hourly_sessions_all and hcp_billing_hourly_sessions_last_7_days: last 7 days should be equal' + ); + + select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_3_hours.sql b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_3_hours.sql new file mode 100644 index 0000000000..c3a066eece --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_3_hours.sql @@ -0,0 +1,81 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(10); + + select has_view('hcp_billing_hourly_sessions_last_3_hours', 'view for hcp billing does not exist'); + + select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + + -- validate the warehouse fact tables are empty + select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; + select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + + select is(count(*), 4::bigint, 'hcp_billing_hourly_sessions_last_3_hours should always return 4 rows') from hcp_billing_hourly_sessions_last_3_hours; + + -- insert one session per minute for the past 2 hours + -- total is 121 = 60 minutes * 2 plus 1 for current minute + with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + now() - interval '2 hours', + now(), + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + + select is(count(*), 4::bigint, 'hcp_billing_hourly_sessions_last_3_hours should always return 4 rows') from hcp_billing_hourly_sessions_last_3_hours; + + select results_eq( + 'select count(*)::bigint from wh_session_accumulating_fact', + 'select sum(sessions_pending_count)::bigint from hcp_billing_hourly_sessions_last_3_hours', + 'hcp_billing_hourly_sessions_last_3_hours: the sum of sessions is incorrect' + ); + + select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_hourly_sessions_last_3_hours limit 1', + 'select extract(minute from now())::bigint + 1', + 'hcp_billing_hourly_sessions_last_3_hours: session count for the current hour is incorrect' + ); + + select results_eq( + 'select * from hcp_billing_hourly_sessions_last_3_hours', + 'select * from hcp_billing_hourly_sessions_last_7_days limit 4', + 'hcp_billing_hourly_sessions_last_3_hours and hcp_billing_hourly_sessions_last_7_days: latest 3 hours should be equal' + ); + + select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_hourly_sessions_last_3_hours order by hour limit 1', + 'select 0::bigint', + 'hcp_billing_hourly_sessions_last_3_hours: session count for the last hour is incorrect' + ); + + select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_7_days.sql b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_7_days.sql new file mode 100644 index 0000000000..7e94bd44ea --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/hourly_sessions_last_7_days.sql @@ -0,0 +1,82 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(10); + + select has_view('hcp_billing_hourly_sessions_last_7_days', 'view for hcp billing does not exist'); + + select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + + -- validate the warehouse fact tables are empty + select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; + select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + + select is(count(*), 169::bigint, 'hcp_billing_hourly_sessions_last_7_days should always return 169 rows') from hcp_billing_hourly_sessions_last_7_days; + + -- insert one session per minute for the past 167 hours + -- 167 hours = 24*7 - 1 hour to test edge cases + -- total is 10,021 = 60 minutes * 167 + 1 for the current minute + with + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from generate_series( + now() - interval '167 hours', + now(), + interval '1 minute' + ) as time + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + + select is(count(*), 169::bigint, 'hcp_billing_hourly_sessions_last_7_days should always return 169 rows') from hcp_billing_hourly_sessions_last_7_days; + + select results_eq( + 'select count(*)::bigint from wh_session_accumulating_fact', + 'select sum(sessions_pending_count)::bigint from hcp_billing_hourly_sessions_last_7_days', + 'hcp_billing_hourly_sessions_last_7_days: the sum of sessions is incorrect' + ); + + select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_hourly_sessions_last_7_days limit 1', + 'select extract(minute from now())::bigint + 1', + 'hcp_billing_hourly_sessions_last_7_days: session count for the current hour is incorrect' + ); + + select results_eq( + 'select * from hcp_billing_hourly_sessions_last_7_days limit 4', + 'select * from hcp_billing_hourly_sessions_last_3_hours', + 'hcp_billing_hourly_sessions_last_7_days and hcp_billing_hourly_sessions_last_3_hours: latest 3 hours should be equal' + ); + + select results_eq( + 'select sessions_pending_count::bigint from hcp_billing_hourly_sessions_last_7_days order by hour limit 1', + 'select 0::bigint', + 'hcp_billing_hourly_sessions_last_7_days: session count for the last hour is incorrect' + ); + + select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/indexes.sql b/internal/db/sqltest/tests/hcp/billing/indexes.sql new file mode 100644 index 0000000000..419443a329 --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/indexes.sql @@ -0,0 +1,13 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(1); + + select has_index('wh_session_accumulating_fact', + 'wh_session_accumulating_fact_session_pending_time_idx', + 'session_pending_time', + 'index for hcp billing views is missing' ); + + select * from finish(); +rollback; diff --git a/internal/db/sqltest/tests/hcp/billing/monthly_sessions_all.sql b/internal/db/sqltest/tests/hcp/billing/monthly_sessions_all.sql new file mode 100644 index 0000000000..ea4fb2fd5b --- /dev/null +++ b/internal/db/sqltest/tests/hcp/billing/monthly_sessions_all.sql @@ -0,0 +1,294 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(47); + + create function test_get_hours_between(start_time timestamptz, end_time timestamptz) returns int + as $$ + select count(time)::int + from generate_series(start_time, end_time, interval '1 hour') as time; + $$ language sql + immutable + returns null on null input; + + create function test_get_hours_between(start_time timestamptz) returns int + as $$ + select test_get_hours_between(start_time, now()); + $$ language sql + immutable + returns null on null input; + + select has_view('hcp_billing_monthly_sessions_all', 'monthly view for hcp billing does not exist'); + select has_view('hcp_billing_monthly_sessions_current_month', 'monthly view for hcp billing does not exist'); + select has_view('hcp_billing_monthly_sessions_last_2_months', 'monthly view for hcp billing does not exist'); + + select lives_ok('truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact', + 'Truncate tables in preparation for testing'); + + -- validate the warehouse fact tables are empty + select is(count(*), 0::bigint, 'wh_session_connection_accumulating_fact is not empty') from wh_session_connection_accumulating_fact; + select is(count(*), 0::bigint, 'wh_session_accumulating_fact is not empty' ) from wh_session_accumulating_fact; + + -- validate the view returns no rows + select is(count(*), 0::bigint, 'hcp_billing_monthly_sessions_all should return 0 rows when there are no sessions') from hcp_billing_monthly_sessions_all; + select is(count(*), 1::bigint, 'hcp_billing_monthly_sessions_current_month should return 1 rows when there are no sessions') from hcp_billing_monthly_sessions_current_month; + select is(count(*), 2::bigint, 'hcp_billing_monthly_sessions_last_2_months should return 1 rows when there are no sessions') from hcp_billing_monthly_sessions_last_2_months; + + create function test_setup_data(start_time timestamptz, end_time timestamptz) returns int + as $$ + declare + insert_count int; + tmp timestamptz; + begin + truncate wh_session_connection_accumulating_fact, wh_session_accumulating_fact; + + if start_time = end_time then + return 0::int; + end if; + + if start_time > end_time then + -- swap + tmp := start_time; start_time := end_time; end_time := tmp; + end if; + + with + vars (start_one, start_two) as ( + select date_trunc('hour', start_time), + (date_trunc('hour', start_time) + interval '1 hour') - interval '1 microsecond' + ), + time_series (time) as ( + select generate_series(vars.start_one, end_time, interval '1 hour') as ts + from vars + union + select generate_series(vars.start_two, end_time, interval '1 hour') as ts + from vars + order by ts + ), + dim_keys (host_key, user_key, credential_group_key) as ( + select h.key, u.key, 'no credentials' + from (select key from wh_host_dimension limit 1) as h, + (select key from wh_user_dimension limit 1) as u + ), + dim_time_series (date_key, time_key, time) as ( + select wh_date_key(time), wh_time_key(time), time + from time_series + ), + fake_sessions (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time) as ( + select concat('s__________', t.date_key, t.time_key), concat('a__________', t.date_key, t.time_key), + k.host_key, k.user_key, k.credential_group_key, + t.date_key, t.time_key,t.time + from dim_keys as k, + dim_time_series as t + ) + insert into wh_session_accumulating_fact + (session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + ) + select session_id, auth_token_id, + host_key, user_key, credential_group_key, + session_pending_date_key, session_pending_time_key, session_pending_time + from fake_sessions; + + select count(*) into insert_count + from wh_session_accumulating_fact; + + return insert_count; + end; + $$ language plpgsql; + + create function test_setup_data(start_time timestamptz) returns int + as $$ + begin + return test_setup_data(start_time, now()); + end; + $$ language plpgsql; + + prepare select_hcp_billing_monthly_sessions_all as + select * + from hcp_billing_monthly_sessions_all; + + prepare select_hcp_billing_monthly_sessions_current_month as + select * + from hcp_billing_monthly_sessions_current_month; + + prepare select_hcp_billing_monthly_sessions_last_2_months as + select * + from hcp_billing_monthly_sessions_last_2_months; + + prepare select_hcp_billing_monthly_sessions_last_2_months_1_row as + select * + from hcp_billing_monthly_sessions_last_2_months + limit 1; + + select is(test_setup_data(now()), 0::int, 'hcp billing: test_setup_data start_time of now() should insert 0 data'); + select is(count(*), 0::bigint, 'hcp_billing_monthly_sessions_all should return 0 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_all; + select is_empty('select_hcp_billing_monthly_sessions_all', 'hcp_billing_monthly_sessions_all should have no rows when there are no sessions in the warehouse'); + select is(count(*), 1::bigint, 'hcp_billing_monthly_sessions_current_month should return 1 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_current_month; + select is(count(*), 2::bigint, 'hcp_billing_monthly_sessions_last_2_months should return 2 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_last_2_months; + + select is(test_setup_data(now(), now()), 0::int, 'hcp billing: test_setup_data start_time of now should insert 0 data'); + select is(count(*), 0::bigint, 'hcp_billing_monthly_sessions_all should return 0 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_all; + select is_empty('select_hcp_billing_monthly_sessions_all', 'hcp_billing_monthly_sessions_all should have no rows when there are no sessions in the warehouse'); + select is(count(*), 1::bigint, 'hcp_billing_monthly_sessions_current_month should return 1 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_current_month; + select is(count(*), 2::bigint, 'hcp_billing_monthly_sessions_last_2_months should return 2 rows when there no rows in the warehouse') from hcp_billing_monthly_sessions_last_2_months; + + -- only sessions in this hour + select is(test_setup_data(date_trunc('hour', now())), 1::int, + 'hcp billing: test_setup_data: start_time of this hour should insert 1'); + select is(count(*), 1::bigint, + 'hcp_billing_monthly_sessions_all should return 1 row when there are only sessions in this hour') from hcp_billing_monthly_sessions_all; + select row_eq('select_hcp_billing_monthly_sessions_all', row(date_trunc('month', now()), date_trunc('hour', now()), 0::bigint), + 'hcp_billing_monthly_sessions_all should have 1 row with 0 sessions_pending_count when there are only sessions for this hour'); + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_all', + 'hcp_billing_monthly_sessions_current_month and hcp_billing_monthly_sessions_all should be equal'); + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_last_2_months_1_row', + 'hcp_billing_monthly_sessions_current_month and the first row of hcp_billing_monthly_sessions_last_2_months should be equal'); + + -- only sessions for yesterday + select is(test_setup_data( 'yesterday'::timestamptz, 'today'::timestamptz - interval '1 microsecond' ), 48::int, + 'hcp billing: test_setup_data: should be 48 sessions for yesterday'); + select is(count(*), 1::bigint, + 'hcp_billing_monthly_sessions_all should return 1 row when there are only sessions in this month') from hcp_billing_monthly_sessions_all; + select row_eq('select_hcp_billing_monthly_sessions_all', row(date_trunc('month', now()), date_trunc('hour', now()), 48::bigint), + 'hcp_billing_monthly_sessions_all should have 1 row with 48 sessions_pending_count when there are only sessions for yesterday'); + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_all', + 'hcp_billing_monthly_sessions_current_month and hcp_billing_monthly_sessions_all should be equal'); + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_last_2_months_1_row', + 'hcp_billing_monthly_sessions_current_month and the first row of hcp_billing_monthly_sessions_last_2_months should be equal'); + + -- only sessions for this month + -- every hour gets 2 sessions + -- 1 at the start of the hour 01:00:00 + -- 1 at the end of the hour (start of next hour - 1 microsecond) + -- the current hour only gets one because the end of the hour has not occurred + -- every day gets 48 sesions (2 for each hour) + -- every month gets 48 * number of days in the month + -- current month gets 48 * number of hours since the start of the month - 1 + -- the current month from the hcp_billing view returns + -- number of hours from the start of the month until the hour before now * 2 + -- when reporting the current hour is not included in the view + select is( test_setup_data( date_trunc('month', now()) ), + -- +1 for the current hour + (test_get_hours_between( date_trunc('month', now()), now() - interval '1 hour') * 2)::int + 1, + 'hcp billing: test_setup_data: wrong number of sessions for the current month'); + + select is(count(*), 1::bigint, + 'hcp_billing_monthly_sessions_all should return 1 row when there are only sessions in this month') from hcp_billing_monthly_sessions_all; + + select row_eq('select_hcp_billing_monthly_sessions_all', + row( date_trunc('month', now()), + date_trunc('hour', now()), + -- 2 sessions per hour, the current hour is not included + (test_get_hours_between( date_trunc('month', now()), now() - interval '1 hour') * 2)::bigint + ), + 'hcp_billing_monthly_sessions_all should have 1 row with 48 sessions_pending_count when there are only sessions for yesterday'); + + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_all', + 'hcp_billing_monthly_sessions_current_month and hcp_billing_monthly_sessions_all should be equal'); + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_last_2_months_1_row', + 'hcp_billing_monthly_sessions_current_month and the first row of hcp_billing_monthly_sessions_last_2_months should be equal'); + + -- only sessions for this month and last month + -- same rules as above for the current month + -- the previous month gets 48 sessions per day * the number of days in the month + + create table test_hcp_billing ( + start_time timestamptz not null, + end_time timestamptz not null, + sessions_pending_count bigint not null, + primary key(start_time, end_time) + ); + + prepare insert_2_month_results as + insert into test_hcp_billing + (start_time, end_time, sessions_pending_count) + select date_trunc('month', now()), + date_trunc('hour', now()), + (test_get_hours_between( date_trunc('month', now()), now() - interval '1 hour' ) * 2)::bigint + union + select date_trunc('month', now() - interval '1 month' ), + date_trunc('month', now()), + extract(days from date_trunc('month', now()) - interval '1 day')::int * 48; + + prepare select_test_hcp_billing as + select * from test_hcp_billing + order by start_time desc; + + prepare select_hcp_billing_monthly_sessions_all_1_row as + select * + from hcp_billing_monthly_sessions_all + limit 1; + + select lives_ok('insert_2_month_results', 'insert rows into test_hcp_billing'); + + select is( test_setup_data( date_trunc('month', now() - interval '1 month' ) ), + (test_get_hours_between( date_trunc('month', now() - interval '1 month' )) * 2)::int - 1, + 'hcp billing: test_setup_data: wrong number of sessions for 2 months'); + + select is(count(*), 2::bigint, + 'hcp_billing_monthly_sessions_all should return 2 rows when there are only sessions for the last 2 months') from hcp_billing_monthly_sessions_all; + + select results_eq('select_hcp_billing_monthly_sessions_all', 'select_test_hcp_billing', + 'hcp_billing_monthly_sessions_all should have 2 rows'); + + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_all_1_row', + 'hcp_billing_monthly_sessions_current_month and hcp_billing_monthly_sessions_all should be equal'); + + select results_eq('select_hcp_billing_monthly_sessions_all', 'select_hcp_billing_monthly_sessions_last_2_months', + 'hcp_billing_monthly_sessions_all and hcp_billing_monthly_sessions_last_2_months should be equal'); + -- sessions for the last 13 months + -- same rules as above for the current month + -- the all previous months get 48 sessions per day * the number of days in the month + + truncate test_hcp_billing; + + prepare insert_13_months_results as + with + expected (start_time, end_time, session_count) as ( + select date_trunc('month', time - interval '1 month'), + date_trunc('month', time), + extract(days from date_trunc('month', time) - interval '1 day')::bigint * 48 + from generate_series( now() - interval '12 months', now(), '1 month'::interval) as time + union + select date_trunc('month', now()), + date_trunc('hour', now()), + (test_get_hours_between( date_trunc('month', now()), now() - interval '1 hour' ) * 2)::bigint + ) + insert into test_hcp_billing + (start_time, end_time, sessions_pending_count) + select start_time, end_time, session_count + from expected; + + prepare select_hcp_billing_monthly_sessions_all_2_rows as + select * + from hcp_billing_monthly_sessions_all + limit 2; + + select lives_ok('insert_13_months_results', 'insert rows into test_hcp_billing'); + + select is(count(*), 14::bigint, + 'test_hcp_billing should return 14 rows') from test_hcp_billing; + + select is( test_setup_data( date_trunc('month', now() - interval '13 month' ) ), + (test_get_hours_between( date_trunc('month', now() - interval '13 month' )) * 2)::int - 1, + 'hcp billing: test_setup_data: wrong number of sessions for 13 months'); + + select is(count(*), 14::bigint, + 'hcp_billing_monthly_sessions_all should return 14 rows') from hcp_billing_monthly_sessions_all; + + select results_eq('select_hcp_billing_monthly_sessions_all', 'select_test_hcp_billing', + 'hcp_billing_monthly_sessions_all should have 14 rows'); + + select results_eq('select_hcp_billing_monthly_sessions_current_month', 'select_hcp_billing_monthly_sessions_all_1_row', + 'hcp_billing_monthly_sessions_current_month and hcp_billing_monthly_sessions_all should be equal'); + + select results_eq('select_hcp_billing_monthly_sessions_all_2_rows', 'select_hcp_billing_monthly_sessions_last_2_months', + 'hcp_billing_monthly_sessions_last_2_months should be equal to the first 2 rows of hcp_billing_monthly_sessions_all'); + + select * from finish(); + +rollback; diff --git a/internal/db/sqltest/tests/target/target_ssh.sql b/internal/db/sqltest/tests/target/target_ssh.sql new file mode 100644 index 0000000000..6cd0f9e784 --- /dev/null +++ b/internal/db/sqltest/tests/target/target_ssh.sql @@ -0,0 +1,21 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(4); + + select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets'); + + insert into target_ssh + (project_id, public_id, name) + values + ('p____bwidget', 'tssh______wb', 'Big Widget SSH Target'), + ('p____swidget', 'tssh______ws', 'Small Widget SSH Target'); + + select is(count(*), 1::bigint) from target_all_subtypes where public_id = 'tssh______wb'; + select is(type, 'ssh') from target_all_subtypes where public_id = 'tssh______wb'; + select is(count(*), 1::bigint) from target_all_subtypes where public_id = 't_________wb'; + select is(type, 'tcp') from target_all_subtypes where public_id = 't_________wb'; + + select * from finish(); +rollback; diff --git a/internal/db/sqltest/tests/wh/credential_dimension/ssh_target.sql b/internal/db/sqltest/tests/wh/credential_dimension/ssh_target.sql new file mode 100644 index 0000000000..581d8f3f60 --- /dev/null +++ b/internal/db/sqltest/tests/wh/credential_dimension/ssh_target.sql @@ -0,0 +1,41 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(2); + + select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets', 'credentials'); + + insert into target_ssh + (project_id, public_id, name) + values + ('p____bwidget', 'tssh______wb', 'Big Widget SSH Target'); + + insert into target_host_set + (project_id, target_id, host_set_id) + values + ('p____bwidget', 'tssh______wb', 's___1wb-sths'); + + insert into target_credential_library + (project_id, target_id, credential_library_id, credential_purpose) + values + ('p____bwidget', 'tssh______wb', 'vl______wvl1', 'brokered'); + + -- ensure no existing dimensions + select is(count(*), 0::bigint) from wh_credential_dimension where organization_id = 'o_____widget'; + + -- insert session, should result in a new credentials dimension with an ssh target type + insert into session + ( project_id, target_id, user_id, auth_token_id, certificate, endpoint, public_id) + values + ('p____bwidget', 'tssh______wb', 'u_____walter', 'tok___walter', 'abc'::bytea, 'ep1', 's1____walter'); + insert into session_host_set_host + (session_id, host_set_id, host_id) + values + ('s1____walter', 's___1wb-sths', 'h_____wb__01'); + insert into session_credential_dynamic + ( session_id, library_id, credential_id, credential_purpose) + values + ('s1____walter', 'vl______wvl1', null, 'brokered'); + select is(count(*), 1::bigint) from wh_credential_dimension where organization_id = 'o_____widget' and target_type = 'ssh target'; +rollback; diff --git a/internal/db/sqltest/tests/wh/host_dimension/targets.sql b/internal/db/sqltest/tests/wh/host_dimension/targets.sql new file mode 100644 index 0000000000..f15f7ebefb --- /dev/null +++ b/internal/db/sqltest/tests/wh/host_dimension/targets.sql @@ -0,0 +1,45 @@ +-- Copyright (c) HashiCorp, Inc. +-- SPDX-License-Identifier: MPL-2.0 + +begin; + select plan(8); + + select wtt_load('widgets', 'iam', 'kms', 'auth', 'hosts', 'targets'); + + insert into target_ssh + (project_id, public_id, name) + values + ('p____bwidget', 'tssh____wbwh', 'Test SSH Target Type W/ HostSet'), + ('p____bwidget', 'tssh___wbwha', 'Test SSH Target Type W/ Address'); + + insert into target_tcp + (project_id, public_id, name) + values + ('p____bwidget', 'ttcp____wbwh', 'Test TCP Target Type W/ HostSet'), + ('p____bwidget', 'ttcp___wbwha', 'Test TCP Target Type W/ Address'); + + insert into target_host_set + (project_id, target_id, host_set_id) + values + ('p____bwidget', 'tssh____wbwh', 's___1wb-plghs'), + ('p____bwidget', 'ttcp____wbwh', 's___1wb-plghs'); + + insert into target_address + (target_id, address) + values + ('tssh___wbwha', '8.6.4.2'), + ('ttcp___wbwha', '8.6.4.2'); + + -- validate ssh target type with host set + select is(target_type, 'ssh target') from whx_host_dimension_source where target_id = 'tssh____wbwh'; + + -- validate ssh target type with address + select is(target_type, 'ssh target') from whx_host_dimension_source where target_id = 'tssh___wbwha'; + + -- validate tcp target type with host set + select is(target_type, 'tcp target') from whx_host_dimension_source where target_id = 'ttcp____wbwh'; + + -- validate tcp target type with address + select is(target_type, 'tcp target') from whx_host_dimension_source where target_id = 'ttcp___wbwha'; + +rollback; \ No newline at end of file diff --git a/internal/target/repository.go b/internal/target/repository.go index cd83ed8c8f..d14d658155 100644 --- a/internal/target/repository.go +++ b/internal/target/repository.go @@ -275,6 +275,12 @@ func (r *Repository) ListTargets(ctx context.Context, opt ...Option) ([]Target, address = v } subtype, err := t.targetSubtype(ctx, address) + if errors.Is(err, errTargetSubtypeNotFound) { + // In cases where we have mixed target types and the controller + // doesn't support all of them, we want to ignore if we can't find + // the target subtype and continue listing the others we do support. + continue + } if err != nil { return nil, errors.Wrap(ctx, err, op) } diff --git a/internal/target/target.go b/internal/target/target.go index 7607e6dc4f..94342906a2 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -5,6 +5,7 @@ package target import ( "context" + goerrs "errors" "fmt" "github.com/hashicorp/boundary/internal/boundary" @@ -54,7 +55,11 @@ const ( targetsViewDefaultTable = "target_all_subtypes" ) -var _ boundary.AuthzProtectedEntity = (*targetView)(nil) +var ( + _ boundary.AuthzProtectedEntity = (*targetView)(nil) + + errTargetSubtypeNotFound = goerrs.New("target subtype not found") +) // targetView provides a common way to return targets regardless of their // underlying type. @@ -117,7 +122,12 @@ func (t *targetView) targetSubtype(ctx context.Context, address string) (Target, alloc, ok := subtypeRegistry.allocFunc(t.Subtype()) if !ok { - return nil, errors.New(ctx, errors.InvalidParameter, op, fmt.Sprintf("%s is an unknown target subtype of %s", t.PublicId, t.Type)) + return nil, errors.Wrap(ctx, + errTargetSubtypeNotFound, + op, + errors.WithCode(errors.InvalidParameter), + errors.WithMsg(fmt.Sprintf("%s is an unknown target subtype of %s", t.PublicId, t.Type)), + ) } tt := alloc()