change: Initial version of procedures based partitioning.

2026-02-19 16:25:15 +00:00
parent bcd5cef7ce
commit b1c3cd579d
7 changed files with 405 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,5 +2,9 @@
 docker/
 z_gen_history_data.sql

+# Local docs
+QUICKSTART.md
+init_extra_users.sql
+
 # Schemas
 sql-scripts*/
--- a/PARTITIONING.md
+++ b/PARTITIONING.md
@@ -0,0 +1,90 @@
+# PostgreSQL Partitioning for Zabbix
+
+This document describes the declarative partitioning implementation for Zabbix `history`, `trends`, and `auditlog` tables on PostgreSQL. This solution replaces standard Zabbix housekeeping for the configured tables.
+
+## Architecture
+
+The solution uses PostgreSQL native declarative partitioning (`PARTITION BY RANGE`).
+All procedures and configuration are stored in the `partitions` schema to maintain separation from Zabbix schema.
+
+### Components
+1.  **Configuration Table**: `partitions.config` defines retention policies.
+2.  **Maintenance Procedure**: `partitions.run_maintenance()` manages partition lifecycle.
+3.  **Monitoring View**: `partitions.monitoring` provides system state visibility.
+
+## Installation
+
+The installation is performed by executing the SQL procedures in the following order:
+1.  Initialize schema (`00_partitions_init.sql`).
+2.  Prepare tables (e.g., `auditlog` PK adjustment) (`01_auditlog_prep.sql`).
+3.  Install maintenance procedures (`02_maintenance.sql`).
+4.  Enable partitioning on tables (`03_enable_partitioning.sql`).
+5.  Install monitoring views (`04_monitoring_view.sql`).
+
+## Configuration
+
+Partitioning policies are defined in the `partitions.config` table.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `table_name` | text | Name of the Zabbix table (e.g., `history`, `trends`). |
+| `period` | text | Partition interval: `day`, `week`, or `month`. |
+| `keep_history` | interval | Data retention period (e.g., `30 days`, `12 months`). |
+| `last_updated` | timestamp | Timestamp of the last successful maintenance run. |
+
+### Modifying Retention
+To change the retention period for a table, update the configuration:
+
+```sql
+UPDATE partitions.config 
+SET keep_history = '60 days' 
+WHERE table_name = 'history';
+```
+
+## Maintenance
+
+The maintenance procedure `partitions.run_maintenance()` is responsible for:
+1.  Creating future partitions (current period + 3 future buffers).
+2.  Creating past partitions (backward coverage based on `keep_history`).
+3.  Dropping partitions older than `keep_history`.
+
+This procedure should be scheduled to run periodically (e.g., daily via `pg_cron` or system cron).
+
+```sql
+CALL partitions.run_maintenance();
+```
+
+## Monitoring & Permissions
+
+System state can be monitored via the `partitions.monitoring` view.
+
+```sql
+SELECT * FROM partitions.monitoring;
+```
+
+### Least Privilege Access (`zbx_monitor`)
+For monitoring purposes, it is recommended to create a dedicated user with read-only access to the monitoring view.
+
+```sql
+CREATE USER zbx_monitor WITH PASSWORD 'secure_password';
+GRANT USAGE ON SCHEMA partitions TO zbx_monitor;
+GRANT SELECT ON partitions.monitoring TO zbx_monitor;
+```
+
+## Implementation Details
+
+### `auditlog` Table
+The standard `auditlog` table Primary Key is `(auditid)`. Partitioning by `clock` requires the partition key to be part of the Primary Key. The initialization script modifies the PK to `(auditid, clock)`.
+
+### Converting Existing Tables
+The enablement script renames the existing table to `table_name_old` and creates a new partitioned table with the same structure.
+*   **Note**: Data from the old table is NOT automatically migrated to minimize downtime.
+*   New data flows into the new partitioned table immediately.
+*   Old data remains accessible in `table_name_old` for manual query or migration if required.
+
+## Upgrades
+
+When upgrading Zabbix:
+1.  **Backup**: Ensure a full database backup exists.
+2.  **Compatibility**: Zabbix upgrade scripts may attempt to `ALTER` tables. PostgreSQL supports `ALTER TABLE` on partitioned tables for adding columns, which propagates to partitions.
+3.  **Failure Scenarios**: If an upgrade script fails due to partitioning, the table may need to be temporarily reverted or the partition structure manually adjusted.
--- a/postgresql/procedures/00_partitions_init.sql
+++ b/postgresql/procedures/00_partitions_init.sql
@@ -0,0 +1,37 @@
+-- ============================================================================
+-- SCRIPT: 00_partitions_init.sql
+-- DESCRIPTION: Creates the 'partitions' schema and configuration table.
+--              Defines the structure for managing Zabbix partitioning.
+-- ============================================================================
+
+CREATE SCHEMA IF NOT EXISTS partitions;
+
+-- Configuration table to store partitioning settings per table
+CREATE TABLE IF NOT EXISTS partitions.config (
+    table_name      text NOT NULL,
+    period          text NOT NULL CHECK (period IN ('day', 'week', 'month', 'year')),
+    keep_history    interval NOT NULL,
+    last_updated    timestamp WITH TIME ZONE DEFAULT now(),
+    PRIMARY KEY (table_name)
+);
+
+-- Default configuration for Zabbix tables (adjust as needed)
+-- History tables: Daily partitions, keep 30 days
+INSERT INTO partitions.config (table_name, period, keep_history) VALUES
+('history', 'day', '30 days'),
+('history_uint', 'day', '30 days'),
+('history_str', 'day', '30 days'),
+('history_log', 'day', '30 days'),
+('history_text', 'day', '30 days')
+ON CONFLICT (table_name) DO NOTHING;
+
+-- Trends tables: Monthly partitions, keep 12 months
+INSERT INTO partitions.config (table_name, period, keep_history) VALUES
+('trends', 'month', '12 months'),
+('trends_uint', 'month', '12 months')
+ON CONFLICT (table_name) DO NOTHING;
+
+-- Auditlog: Monthly partitions, keep 12 months
+INSERT INTO partitions.config (table_name, period, keep_history) VALUES
+('auditlog', 'month', '12 months')
+ON CONFLICT (table_name) DO NOTHING;
--- a/postgresql/procedures/01_auditlog_prep.sql
+++ b/postgresql/procedures/01_auditlog_prep.sql
@@ -0,0 +1,27 @@
+-- ============================================================================
+-- SCRIPT: 01_auditlog_prep.sql
+-- DESCRIPTION: Modifies the 'auditlog' table Primary Key to include 'clock'.
+--              This is REQUIRED for range partitioning by 'clock'.
+-- ============================================================================
+
+DO $$
+BEGIN
+    -- Check if PK needs modification
+    -- Original PK is typically on (auditid) named 'auditlog_pkey'
+    IF EXISTS (
+        SELECT 1 FROM pg_constraint 
+        WHERE conname = 'auditlog_pkey' 
+        AND conrelid = 'auditlog'::regclass
+    ) THEN
+        -- Verify if 'clock' is already in PK (basic check)
+        -- Realistically, if 'auditlog_pkey' exists on default Zabbix, it's just (auditid).
+        
+        RAISE NOTICE 'Dropping existing Primary Key on auditlog...';
+        ALTER TABLE auditlog DROP CONSTRAINT auditlog_pkey;
+        
+        RAISE NOTICE 'Creating new Primary Key on auditlog (auditid, clock)...';
+        ALTER TABLE auditlog ADD PRIMARY KEY (auditid, clock);
+    ELSE
+        RAISE NOTICE 'Constraint auditlog_pkey not found. Skipping or already modified.';
+    END IF;
+END $$;
--- a/postgresql/procedures/02_maintenance.sql
+++ b/postgresql/procedures/02_maintenance.sql
@@ -0,0 +1,182 @@
+-- ============================================================================
+-- SCRIPT: 02_maintenance.sql
+-- DESCRIPTION: Core functions for Zabbix partitioning (Create, Drop, Maintain).
+-- ============================================================================
+
+-- Function to check if a partition exists
+CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text) 
+RETURNS boolean AS $$
+BEGIN
+    RETURN EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = p_partition_name
+        AND n.nspname = 'public'
+    );
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to create a partition
+CREATE OR REPLACE PROCEDURE partitions.create_partition(
+    p_parent_table text,
+    p_start_time timestamp with time zone,
+    p_end_time timestamp with time zone,
+    p_period text
+) LANGUAGE plpgsql AS $$
+DECLARE
+    v_partition_name text;
+    v_start_ts bigint;
+    v_end_ts bigint;
+    v_suffix text;
+BEGIN
+    v_start_ts := extract(epoch from p_start_time)::bigint;
+    v_end_ts := extract(epoch from p_end_time)::bigint;
+    
+    IF p_period = 'month' THEN
+        v_suffix := to_char(p_start_time, 'YYYYMM');
+    ELSE
+        v_suffix := to_char(p_start_time, 'YYYYMMDD');
+    END IF;
+    
+    v_partition_name := p_parent_table || '_p' || v_suffix;
+
+    IF NOT partitions.partition_exists(v_partition_name) THEN
+        EXECUTE format(
+            'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES FROM (%s) TO (%s)',
+            v_partition_name, p_parent_table, v_start_ts, v_end_ts
+        );
+    END IF;
+END;
+$$;
+
+-- Function to drop old partitions
+CREATE OR REPLACE PROCEDURE partitions.drop_old_partitions(
+    p_parent_table text,
+    p_retention interval,
+    p_period text
+) LANGUAGE plpgsql AS $$
+DECLARE
+    v_cutoff_ts bigint;
+    v_partition record;
+    v_partition_date timestamp with time zone;
+    v_suffix text;
+BEGIN
+    -- Calculate cutoff timestamp
+    v_cutoff_ts := extract(epoch from (now() - p_retention))::bigint;
+
+    FOR v_partition IN 
+        SELECT 
+            child.relname AS partition_name
+        FROM pg_inherits
+        JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+        JOIN pg_class child ON pg_inherits.inhrelid = child.oid
+        WHERE parent.relname = p_parent_table
+    LOOP
+        -- Parse partition suffix to determine age
+        -- Format: parent_pYYYYMM or parent_pYYYYMMDD
+        v_suffix := substring(v_partition.partition_name from length(p_parent_table) + 3);
+        
+        BEGIN
+            IF length(v_suffix) = 6 THEN -- YYYYMM
+                v_partition_date := to_timestamp(v_suffix || '01', 'YYYYMMDD');
+                -- For monthly, we check if the END of the month is older than retention? 
+                -- Or just strict retention. 
+                -- To be safe, adding 1 month to check vs cutoff.
+                IF extract(epoch from (v_partition_date + '1 month'::interval)) < v_cutoff_ts THEN
+                    RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
+                    EXECUTE format('DROP TABLE public.%I', v_partition.partition_name);
+                END IF;
+            ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD
+                v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD');
+                IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN
+                    RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
+                    EXECUTE format('DROP TABLE public.%I', v_partition.partition_name);
+                END IF;
+            END IF;
+        EXCEPTION WHEN OTHERS THEN
+            -- Ignore parsing errors for non-standard partitions
+            NULL;
+        END;
+    END LOOP;
+END;
+$$;
+
+-- MAIN Procedure to maintain a single table
+CREATE OR REPLACE PROCEDURE partitions.maintain_table(
+    p_table_name text,
+    p_period text,
+    p_keep_history interval
+) LANGUAGE plpgsql AS $$
+DECLARE
+    v_start_time timestamp with time zone;
+    v_period_interval interval;
+    i integer;
+    v_past_iterations integer;
+BEGIN
+    IF p_period = 'day' THEN
+        v_period_interval := '1 day'::interval;
+        v_start_time := date_trunc('day', now());
+        -- Calculate how many past days cover the retention period
+        v_past_iterations := extract(day from p_keep_history)::integer;
+        -- Safety cap or ensure minimum? default 7 if null?
+        IF v_past_iterations IS NULL THEN v_past_iterations := 7; END IF;
+        
+    ELSIF p_period = 'week' THEN
+        v_period_interval := '1 week'::interval;
+        v_start_time := date_trunc('week', now());
+        v_past_iterations := (extract(day from p_keep_history) / 7)::integer;
+
+    ELSIF p_period = 'month' THEN
+        v_period_interval := '1 month'::interval;
+        v_start_time := date_trunc('month', now());
+        -- Approximate months
+        v_past_iterations := (extract(year from p_keep_history) * 12 + extract(month from p_keep_history))::integer;
+        -- Fallback if interval is just days (e.g. '365 days')
+        IF v_past_iterations = 0 THEN 
+             v_past_iterations := (extract(day from p_keep_history) / 30)::integer; 
+        END IF;
+    ELSE
+        RETURN;
+    END IF;
+
+    -- 1. Create Future Partitions (Current + 3 ahead)
+    FOR i IN 0..3 LOOP
+        CALL partitions.create_partition(
+            p_table_name,
+            v_start_time + (i * v_period_interval),
+            v_start_time + ((i + 1) * v_period_interval),
+            p_period
+        );
+    END LOOP;
+    
+    -- 2. Create Past Partitions (Covering retention period)
+    IF v_past_iterations > 0 THEN
+        FOR i IN 1..v_past_iterations LOOP
+            CALL partitions.create_partition(
+                p_table_name,
+                v_start_time - (i * v_period_interval),
+                v_start_time - ((i - 1) * v_period_interval),
+                p_period
+            );
+        END LOOP;
+    END IF;
+
+    -- 3. Drop Old Partitions
+    CALL partitions.drop_old_partitions(p_table_name, p_keep_history, p_period);
+    
+    -- 4. Update Metadata
+    UPDATE partitions.config SET last_updated = now() WHERE table_name = p_table_name;
+END;
+$$;
+
+-- Global Maintenance Procedure
+CREATE OR REPLACE PROCEDURE partitions.run_maintenance() 
+LANGUAGE plpgsql AS $$
+DECLARE
+    v_row record;
+BEGIN
+    FOR v_row IN SELECT * FROM partitions.config LOOP
+        CALL partitions.maintain_table(v_row.table_name, v_row.period, v_row.keep_history);
+    END LOOP;
+END;
+$$;
--- a/postgresql/procedures/03_enable_partitioning.sql
+++ b/postgresql/procedures/03_enable_partitioning.sql
@@ -0,0 +1,43 @@
+-- ============================================================================
+-- SCRIPT: 03_enable_partitioning.sql
+-- DESCRIPTION: Converts standard Zabbix tables to Partitioned tables.
+--              WARNING: This renames existing tables to *_old.
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_row record;
+    v_table text;
+    v_old_table text;
+    v_pk_sql text;
+BEGIN
+    FOR v_row IN SELECT * FROM partitions.config LOOP
+        v_table := v_row.table_name;
+        v_old_table := v_table || '_old';
+        
+        -- Check if table exists and is NOT already partitioned
+        IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
+            RAISE NOTICE 'Converting table % to partitioned table...', v_table;
+            
+            -- 1. Rename existing table
+            EXECUTE format('ALTER TABLE public.%I RENAME TO %I', v_table, v_old_table);
+            
+            -- 2. Create new partitioned table (copying structure)
+            EXECUTE format('CREATE TABLE public.%I (LIKE public.%I INCLUDING ALL) PARTITION BY RANGE (clock)', v_table, v_old_table);
+            
+            -- 3. Create initial partitions
+            RAISE NOTICE 'Creating initial partitions for %...', v_table;
+            CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history); 
+            
+            -- 4. (Optional) Copy data?
+            -- EXECUTE format('INSERT INTO public.%I SELECT * FROM public.%I', v_table, v_old_table);
+            
+        ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN
+             RAISE NOTICE 'Table % is already partitioned. Skipping conversion.', v_table;
+             -- Just run maintenance to ensure partitions exist
+             CALL partitions.run_maintenance();
+        ELSE
+             RAISE WARNING 'Table % not found!', v_table;
+        END IF;
+    END LOOP;
+END $$;
--- a/postgresql/procedures/04_monitoring_view.sql
+++ b/postgresql/procedures/04_monitoring_view.sql
@@ -0,0 +1,22 @@
+-- ============================================================================
+-- SCRIPT: 04_monitoring_view.sql
+-- DESCRIPTION: Creates a view to monitor partition status and sizes.
+-- ============================================================================
+
+CREATE OR REPLACE VIEW partitions.monitoring AS
+SELECT
+    parent.relname AS parent_table,
+    c.table_name,
+    c.period,
+    c.keep_history,
+    count(child.relname) AS partition_count,
+    pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size,
+    min(child.relname) AS oldest_partition,
+    max(child.relname) AS newest_partition,
+    c.last_updated
+FROM partitions.config c
+JOIN pg_class parent ON parent.relname = c.table_name
+LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid
+LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid
+WHERE parent.relkind = 'p' -- Only partitioned tables
+GROUP BY parent.relname, c.table_name, c.period, c.keep_history, c.last_updated;