From 99e25f2efb090180ab9178ec2c080139cb2e3474 Mon Sep 17 00:00:00 2001 From: Maksym Buz Date: Thu, 19 Feb 2026 17:27:31 +0000 Subject: [PATCH] feat: introduce configurable future partition buffer and add monitoring for future partitions. --- PARTITIONING.md | 12 +++++++----- postgresql/procedures/00_partitions_init.sql | 1 + postgresql/procedures/02_maintenance.sql | 9 +++++---- postgresql/procedures/03_enable_partitioning.sql | 4 ++-- postgresql/procedures/04_monitoring_view.sql | 6 ++++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/PARTITIONING.md b/PARTITIONING.md index c82374b..ea5901e 100644 --- a/PARTITIONING.md +++ b/PARTITIONING.md @@ -1,22 +1,23 @@ # PostgreSQL Partitioning for Zabbix -This document describes the declarative partitioning implementation for Zabbix `history`, `trends`, and `auditlog` tables on PostgreSQL. This solution replaces standard Zabbix housekeeping for the configured tables. +This is the declarative (PostgreSQL procedures based) partitioning implementation for Zabbix `history`, `trends`, and `auditlog` tables on PostgreSQL. This solution is intended to replace standard Zabbix housekeeping for the configured tables. Partitioning is very useful for large environments because it completely eliminates the housekeeper from the process. Instead of huge DELETE queries on several million rows, fast DDL queries (ALTER TABLE) are executed, which drop an entire partition. ## Architecture The solution uses PostgreSQL native declarative partitioning (`PARTITION BY RANGE`). -All procedures and configuration are stored in the `partitions` schema to maintain separation from Zabbix schema. +All procedures, information, statistics and configuration are stored in the `partitions` schema to maintain full separation from Zabbix schema. ### Components 1. **Configuration Table**: `partitions.config` defines retention policies. 2. **Maintenance Procedure**: `partitions.run_maintenance()` manages partition lifecycle. 3. **Monitoring View**: `partitions.monitoring` provides system state visibility. +4. **Version Table**: `partitions.version` provides information about installed version of the partitioning solution. ## Installation The installation is performed by executing the SQL procedures in the following order: 1. Initialize schema (`00_partitions_init.sql`). -2. Prepare tables (e.g., `auditlog` PK adjustment) (`01_auditlog_prep.sql`). +2. Auditlog PK adjustment (`01_auditlog_prep.sql`). 3. Install maintenance procedures (`02_maintenance.sql`). 4. Enable partitioning on tables (`03_enable_partitioning.sql`). 5. Install monitoring views (`04_monitoring_view.sql`). @@ -30,6 +31,7 @@ Partitioning policies are defined in the `partitions.config` table. | `table_name` | text | Name of the Zabbix table (e.g., `history`, `trends`). | | `period` | text | Partition interval: `day`, `week`, or `month`. | | `keep_history` | interval | Data retention period (e.g., `30 days`, `12 months`). | +| `future_partitions` | integer | Number of future partitions to pre-create (buffer). Default: `5`. | | `last_updated` | timestamp | Timestamp of the last successful maintenance run. | ### Modifying Retention @@ -44,7 +46,7 @@ WHERE table_name = 'history'; ## Maintenance The maintenance procedure `partitions.run_maintenance()` is responsible for: -1. Creating future partitions (current period + 3 future buffers). +1. Creating future partitions (current period + `future_partitions` buffer). 2. Creating past partitions (backward coverage based on `keep_history`). 3. Dropping partitions older than `keep_history`. @@ -56,7 +58,7 @@ CALL partitions.run_maintenance(); ## Monitoring & Permissions -System state can be monitored via the `partitions.monitoring` view. +System state can be monitored via the `partitions.monitoring` view. It includes a `future_partitions` column which counts how many partitions exist *after* the current period. This is useful for alerting (e.g., trigger if `future_partitions < 2`). ```sql SELECT * FROM partitions.monitoring; diff --git a/postgresql/procedures/00_partitions_init.sql b/postgresql/procedures/00_partitions_init.sql index b9664ea..fec159a 100644 --- a/postgresql/procedures/00_partitions_init.sql +++ b/postgresql/procedures/00_partitions_init.sql @@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS partitions.config ( table_name text NOT NULL, period text NOT NULL CHECK (period IN ('day', 'week', 'month', 'year')), keep_history interval NOT NULL, + future_partitions integer NOT NULL DEFAULT 5, last_updated timestamp WITH TIME ZONE DEFAULT now(), PRIMARY KEY (table_name) ); diff --git a/postgresql/procedures/02_maintenance.sql b/postgresql/procedures/02_maintenance.sql index cbb8d2e..c79fb1e 100644 --- a/postgresql/procedures/02_maintenance.sql +++ b/postgresql/procedures/02_maintenance.sql @@ -105,7 +105,8 @@ $$; CREATE OR REPLACE PROCEDURE partitions.maintain_table( p_table_name text, p_period text, - p_keep_history interval + p_keep_history interval, + p_future_partitions integer DEFAULT 5 ) LANGUAGE plpgsql AS $$ DECLARE v_start_time timestamp with time zone; @@ -139,8 +140,8 @@ BEGIN RETURN; END IF; - -- 1. Create Future Partitions (Current + 3 ahead) - FOR i IN 0..3 LOOP + -- 1. Create Future Partitions (Current + Buffer) + FOR i IN 0..p_future_partitions LOOP CALL partitions.create_partition( p_table_name, v_start_time + (i * v_period_interval), @@ -176,7 +177,7 @@ DECLARE v_row record; BEGIN FOR v_row IN SELECT * FROM partitions.config LOOP - CALL partitions.maintain_table(v_row.table_name, v_row.period, v_row.keep_history); + CALL partitions.maintain_table(v_row.table_name, v_row.period, v_row.keep_history, v_row.future_partitions); END LOOP; END; $$; diff --git a/postgresql/procedures/03_enable_partitioning.sql b/postgresql/procedures/03_enable_partitioning.sql index 95ae3e3..cd6147b 100644 --- a/postgresql/procedures/03_enable_partitioning.sql +++ b/postgresql/procedures/03_enable_partitioning.sql @@ -27,9 +27,9 @@ BEGIN -- 3. Create initial partitions RAISE NOTICE 'Creating initial partitions for %...', v_table; - CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history); + CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history, v_row.future_partitions); - -- 4. (Optional) Copy data? + -- Optional: Migrate existing data -- EXECUTE format('INSERT INTO public.%I SELECT * FROM public.%I', v_table, v_old_table); ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN diff --git a/postgresql/procedures/04_monitoring_view.sql b/postgresql/procedures/04_monitoring_view.sql index 56f4fbd..80139f1 100644 --- a/postgresql/procedures/04_monitoring_view.sql +++ b/postgresql/procedures/04_monitoring_view.sql @@ -10,6 +10,12 @@ SELECT c.period, c.keep_history, count(child.relname) AS partition_count, + count(child.relname) FILTER ( + WHERE + (c.period = 'day' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMMDD'))) + OR + (c.period = 'month' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMM'))) + ) AS future_partitions, pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size, min(child.relname) AS oldest_partition, max(child.relname) AS newest_partition,