From fb65b2f1e75720e79119c9e6197e0462b86933e5 Mon Sep 17 00:00:00 2001 From: Maksym Buz Date: Thu, 2 Apr 2026 17:02:18 +0000 Subject: [PATCH] Enhance partitioning logic and update Zabbix config template --- postgresql/procedures/00_schema_create.sql | 2 +- postgresql/procedures/01_maintenance.sql | 51 +++++++++++++++---- postgresql/procedures/03_monitoring_view.sql | 8 ++- postgresql/template/partitions.get_all.sql | 2 + .../zbx_pg_partitions_monitor_agent2.yaml | 32 ++++++++++++ 5 files changed, 81 insertions(+), 14 deletions(-) diff --git a/postgresql/procedures/00_schema_create.sql b/postgresql/procedures/00_schema_create.sql index 811ed3c..b194b90 100644 --- a/postgresql/procedures/00_schema_create.sql +++ b/postgresql/procedures/00_schema_create.sql @@ -8,7 +8,7 @@ CREATE SCHEMA IF NOT EXISTS partitions; -- Configuration table to store partitioning settings per table CREATE TABLE IF NOT EXISTS partitions.config ( table_name text NOT NULL, - period text NOT NULL CHECK (period IN ('day', 'week', 'month', 'year')), + period text NOT NULL, keep_history interval NOT NULL, future_partitions integer NOT NULL DEFAULT 5, last_updated timestamp WITH TIME ZONE DEFAULT (now() AT TIME ZONE 'UTC'), diff --git a/postgresql/procedures/01_maintenance.sql b/postgresql/procedures/01_maintenance.sql index c51a3fd..9be5ada 100644 --- a/postgresql/procedures/01_maintenance.sql +++ b/postgresql/procedures/01_maintenance.sql @@ -43,6 +43,8 @@ BEGIN IF p_period = 'month' THEN v_suffix := to_char(p_start_time, 'YYYYMM'); + ELSIF p_period LIKE '%hour%' THEN + v_suffix := to_char(p_start_time, 'YYYYMMDDHH24'); ELSE v_suffix := to_char(p_start_time, 'YYYYMMDD'); END IF; @@ -91,26 +93,47 @@ BEGIN BEGIN IF length(v_suffix) = 6 THEN -- YYYYMM v_partition_date := to_timestamp(v_suffix || '01', 'YYYYMMDD') AT TIME ZONE 'UTC'; - -- For monthly, we check if the END of the month is older than retention? - -- Or just strict retention. - -- To be safe, adding 1 month to check vs cutoff. - IF extract(epoch from (v_partition_date + '1 month'::interval)) < v_cutoff_ts THEN + ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD + v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC'; + ELSIF length(v_suffix) = 10 THEN -- YYYYMMDDHH + v_partition_date := to_timestamp(v_suffix, 'YYYYMMDDHH24') AT TIME ZONE 'UTC'; + ELSE + CONTINUE; -- Ignore non-matching suffix lengths + END IF; + EXCEPTION WHEN OTHERS THEN + -- Safely ignore parsing errors for oddly named partitions + CONTINUE; + END; + + -- Now check retention and execute DROP TABLE (so dropping errors are correctly raised!) + IF length(v_suffix) = 6 THEN -- YYYYMM + IF extract(epoch from (v_partition_date + '1 month'::interval)) < v_cutoff_ts THEN + RAISE NOTICE 'Dropping old partition %', v_partition.partition_name; + EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name); + COMMIT; -- Release lock immediately + END IF; + ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD + -- If period is weekly, the partition spans an entire week. Otherwise, it spans one day. + IF p_period = 'week' THEN + IF extract(epoch from (v_partition_date + '1 week'::interval)) < v_cutoff_ts THEN RAISE NOTICE 'Dropping old partition %', v_partition.partition_name; EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name); COMMIT; -- Release lock immediately END IF; - ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD - v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC'; + ELSE IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN RAISE NOTICE 'Dropping old partition %', v_partition.partition_name; EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name); COMMIT; -- Release lock immediately END IF; END IF; - EXCEPTION WHEN OTHERS THEN - -- Ignore parsing errors for non-standard partitions - NULL; - END; + ELSIF length(v_suffix) = 10 THEN -- YYYYMMDDHH + IF extract(epoch from (v_partition_date + p_period::interval)) < v_cutoff_ts THEN + RAISE NOTICE 'Dropping old partition %', v_partition.partition_name; + EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name); + COMMIT; -- Release lock immediately + END IF; + END IF; END LOOP; END; $$; @@ -145,8 +168,14 @@ BEGIN v_start_time := date_trunc('month', now() AT TIME ZONE 'UTC'); -- Approximate 30 days per month (2592000 seconds) v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer; + + ELSIF p_period LIKE '%hour%' THEN + v_period_interval := p_period::interval; + v_start_time := date_trunc('hour', now() AT TIME ZONE 'UTC'); + v_past_iterations := ceil(extract(epoch from p_keep_history) / extract(epoch from v_period_interval))::integer; + ELSE - RETURN; + RAISE EXCEPTION 'Unsupported partitioning period: %', p_period; END IF; -- 1. Create Future Partitions (Current + Buffer) diff --git a/postgresql/procedures/03_monitoring_view.sql b/postgresql/procedures/03_monitoring_view.sql index 15c06a0..aa91444 100644 --- a/postgresql/procedures/03_monitoring_view.sql +++ b/postgresql/procedures/03_monitoring_view.sql @@ -12,9 +12,13 @@ SELECT count(child.relname) AS partition_count, count(child.relname) FILTER ( WHERE - (c.period = 'day' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMMDD'))) + (c.period = 'day' AND child.relname > (parent.relname || '_p' || to_char(now() AT TIME ZONE 'UTC', 'YYYYMMDD'))) OR - (c.period = 'month' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMM'))) + (c.period = 'month' AND child.relname > (parent.relname || '_p' || to_char(now() AT TIME ZONE 'UTC', 'YYYYMM'))) + OR + (c.period = 'week' AND child.relname > (parent.relname || '_p' || to_char(date_trunc('week', now() AT TIME ZONE 'UTC'), 'YYYYMMDD'))) + OR + (c.period LIKE '%hour%' AND child.relname > (parent.relname || '_p' || to_char(now() AT TIME ZONE 'UTC', 'YYYYMMDDHH24'))) ) AS future_partitions, sum(pg_total_relation_size(child.oid)) AS total_size_bytes, pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size, diff --git a/postgresql/template/partitions.get_all.sql b/postgresql/template/partitions.get_all.sql index 7c82e6f..3276b50 100644 --- a/postgresql/template/partitions.get_all.sql +++ b/postgresql/template/partitions.get_all.sql @@ -1,5 +1,7 @@ SELECT table_name, + period, + keep_history::text AS keep_history, future_partitions, total_size_bytes, EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds diff --git a/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml b/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml index ab17289..bcb52e6 100644 --- a/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml +++ b/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml @@ -86,6 +86,38 @@ zabbix_export: value: size - tag: table value: '{#TABLE_NAME}' + - uuid: ffa2b3c4d5e64f7a9b8c7d6e5f4a1001 + name: '{#TABLE_NAME}: Configured Partition Period' + type: DEPENDENT + key: 'db.partitions.period["{#TABLE_NAME}"]' + value_type: CHAR + preprocessing: + - type: JSONPATH + parameters: + - '$.[?(@.table_name == "{#TABLE_NAME}")].period.first()' + master_item: + key: 'pgsql.custom.query["{$PG.CONNSTRING.AGENT2}",,,"{$PG.DBNAME}","partitions.get_all"]' + tags: + - tag: metric + value: config + - tag: table + value: '{#TABLE_NAME}' + - uuid: ffa2b3c4d5e64f7a9b8c7d6e5f4a1002 + name: '{#TABLE_NAME}: Configured Retention (Keep History)' + type: DEPENDENT + key: 'db.partitions.retention["{#TABLE_NAME}"]' + value_type: CHAR + preprocessing: + - type: JSONPATH + parameters: + - '$.[?(@.table_name == "{#TABLE_NAME}")].keep_history.first()' + master_item: + key: 'pgsql.custom.query["{$PG.CONNSTRING.AGENT2}",,,"{$PG.DBNAME}","partitions.get_all"]' + tags: + - tag: metric + value: config + - tag: table + value: '{#TABLE_NAME}' master_item: key: 'pgsql.custom.query["{$PG.CONNSTRING.AGENT2}",,,"{$PG.DBNAME}","partitions.get_all"]' lld_macro_paths: