Zabbix/partitioning
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add housekeeper task interceptor trigger and update docs

Browse Source
This commit is contained in:
Maksym Buz
2026-04-29 17:12:18 +00:00
parent a88cee43b6
commit 59b2780125
13 changed files with 158966 additions and 148395 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
postgresql/procedures/00_schema_create.sql
View File

@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS partitions.version (
description text
);
INSERT INTO partitions.version (version, description) VALUES ('7-1', 'Zabbix 7.4 and 7.0 compatible version')
INSERT INTO partitions.version (version, description) VALUES ('7-2', 'Added housekeeper task interceptor trigger to drop tasks for partitioned tables')
ON CONFLICT (version) DO NOTHING;
-- Default configuration for Zabbix tables (adjust as needed)

11
postgresql/procedures/01_maintenance.sql
View File

@@ -226,3 +226,14 @@ BEGIN
END LOOP;
END;
$$;
-- Trigger function to silently discard housekeeper tasks for partitioned tables
CREATE OR REPLACE FUNCTION partitions.housekeeper_insert_trigger()
RETURNS TRIGGER AS $$
BEGIN
IF EXISTS (SELECT 1 FROM partitions.config WHERE table_name = NEW.tablename) THEN
RETURN NULL;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;

7
postgresql/procedures/02_enable_partitioning.sql
View File

@@ -57,3 +57,10 @@ BEGIN
END IF;
END LOOP;
END $$;
-- Attach trigger to housekeeper table to silently discard tasks for partitioned tables
DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper;
CREATE TRIGGER housekeeper_filter
BEFORE INSERT ON housekeeper
FOR EACH ROW
EXECUTE FUNCTION partitions.housekeeper_insert_trigger();

69
postgresql/procedures/04_undo_partitioning.sql Normal file
View File

@@ -0,0 +1,69 @@
-- ============================================================================
-- Reverts Zabbix partitioned tables back to standard non-partitioned tables.
-- Existing partitioned tables will be renamed to *_part (data is preserved).
-- ============================================================================
DO $$
DECLARE
v_row record;
v_table text;
v_part_table text;
v_schema text;
BEGIN
FOR v_row IN SELECT * FROM partitions.config LOOP
v_table := v_row.table_name;
v_part_table := v_table || '_part';
-- Determine schema of the partitioned table
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = v_table AND c.relkind = 'p';
IF v_schema IS NOT NULL THEN
RAISE NOTICE 'Reverting partitioned table %...', v_table;
-- 1. Rename existing partitioned table to *_part
EXECUTE format('ALTER TABLE %I.%I RENAME TO %I', v_schema, v_table, v_part_table);
-- 2. Create standard (unpartitioned) replacement table based on the structure
IF v_table = 'auditlog' THEN
-- For auditlog, we need to try and restore the original single-column PK (auditid) if possible
EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING DEFAULTS INCLUDING COMMENTS)', v_schema, v_table, v_schema, v_part_table);
BEGIN
EXECUTE format('ALTER TABLE %I.%I ADD PRIMARY KEY (auditid)', v_schema, v_table);
EXCEPTION WHEN others THEN
RAISE WARNING 'Failed to create primary key on auditlog, might already exist or duplicates present.';
END;
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_1 ON %I.%I (userid, clock)', v_schema, v_table);
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_2 ON %I.%I (clock)', v_schema, v_table);
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_3 ON %I.%I (resourcetype, resourceid)', v_schema, v_table);
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_4 ON %I.%I (recordsetid)', v_schema, v_table);
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_5 ON %I.%I (ip)', v_schema, v_table);
ELSE
-- For others, copy everything including indexes
EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING ALL)', v_schema, v_table, v_schema, v_part_table);
END IF;
RAISE NOTICE 'SUCCESS: % reverted to default. Partitioned data stored in % (You can DROP TABLE % CASCADE; later).', v_table, v_part_table, v_part_table;
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
RAISE NOTICE 'Table % is already a regular table. Skipping.', v_table;
ELSE
RAISE WARNING 'Partitioned table % not found!', v_table;
END IF;
END LOOP;
-- Drop the housekeeper intercept trigger
DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper;
RAISE NOTICE '================================================================================';
RAISE NOTICE 'Undo complete. Partitioned tables have been renamed to *_part.';
RAISE NOTICE 'If you want to migrate your history back, you must do it manually:';
RAISE NOTICE ' INSERT INTO history SELECT * FROM history_part;';
RAISE NOTICE 'Once done, or if you do not need the data, drop the partitioned tables:';
RAISE NOTICE ' DROP TABLE history_part CASCADE;';
RAISE NOTICE 'After that, you can safely remove the partitions infrastructure:';
RAISE NOTICE ' DROP SCHEMA partitions CASCADE;';
RAISE NOTICE '================================================================================';
END $$;

56
postgresql/procedures/README.md
View File

@@ -21,6 +21,8 @@ This is the declarative partitioning implementation for Zabbix `history*`, `tren
- [Implementation Details](#implementation-details)
- [`auditlog` Table](#auditlog-table)
- [Converting Existing Tables](#converting-existing-tables)
- [PostgreSQL Tuning](#postgresql-tuning)
- [Uninstall / Reverting](#uninstall--reverting)
- [Upgrades](#upgrades)
## Architecture
@@ -267,9 +269,59 @@ The enablement script guarantees practically zero downtime by automatically rena
* New data flows into the new partitioned tables immediately.
* Old data remains accessible in `table_name_old` for manual lookup or migration if required.
## Upgrades
### Housekeeper Interceptor
Even when Zabbix Housekeeping is disabled in the UI for History and Trends, the Zabbix Server daemon may still generate and insert tasks into the `housekeeper` table (e.g., when an item or trigger is deleted, it schedules the deletion of its historical data). Without intervention, this results in the `housekeeper` table bloating massively over time, leading to slow sequential scans and `autovacuum` overhead.
When upgrading Zabbix:
To prevent this, this extension installs a `BEFORE INSERT` trigger on the `housekeeper` table.
* When Zabbix attempts to insert a housekeeper task, the trigger intercepts it and checks if the target table is managed in `partitions.config`.
* If the table is partitioned (like `history`), the trigger **silently discards the insert** (`RETURNS NULL`), preventing disk I/O and table bloat entirely.
* If the table is not partitioned (like `events` or `sessions`), the task is allowed to be recorded and is cleaned up naturally by Zabbix.
## PostgreSQL Tuning
Before or immediately after enabling partitioning, you should tune your `postgresql.conf`. The standard configuration is not optimized for partitioned tables and might cause performance degradation or out-of-memory errors.
| Parameter | Recommended | Description |
|-----------|-------------|-------------|
| `max_locks_per_transaction`| `512` (or higher) | **Requires DB Restart.** Default is `64`, which is far too low. PostgreSQL lock tables per partition. With many partitioned tables (e.g., history x 30 days), operations like `pg_dump`, `VACUUM`, or queries crossing multiple boundaries will fail with *“out of shared memory”*. |
| `jit` | `off` | **Highly Recommended.** JIT adds overhead to query planning. With many partitions, JIT can drastically increase CPU usage as PostgreSQL attempts to optimize simple queries across dozens of partitions. |
**Default parameters to verify:**
The following are usually set correctly by default, but you should verify them just in case:
* `enable_partition_pruning = on` : **Critical.** Ensures PostgreSQL only queries the necessary partitions instead of scanning everything.
* `enable_partitionwise_join = off` : Zabbix does not do massive joins on history tables; enabling this only wastes planner CPU time.
* `enable_partitionwise_aggregate = off` : Zabbix doesn't perform complex DB-side `GROUP BY` aggregations on history. Leave it disabled.
## Uninstall / Reverting
If you wish to stop using partitioning and revert back to standard, unpartitioned tables without data loss, carefully follow these steps.
> [!CAUTION]
> Reverting partitioning replaces your partitioned tables with standard empty tables. If you need to retain data from the partitioned period, you must manually migrate it before dropping the partition sets. **Always stop Zabbix Server before proceeding.**
1. **Stop Zabbix Server** to prevent new data from being inserted during the transition.
2. **Execute Undo Script:** Run the `04_undo_partitioning.sql` script to recreate non-partitioned tables matching your original Zabbix schema. This script will rename your current partitioned tables to `*_part` (`history_part`, `trends_part`, etc.) and automatically create native, clean tables (`history`, `trends`) in their place.
```bash
psql -h $DB_HOST -U zbxpart_admin -d zabbix -f 04_undo_partitioning.sql
```
3. **Data Migration (Optional):** If you want to keep the metrics collected during the partitioned period, you must manually insert them into the newly created regular tables. This step can take hours depending on table sizes.
```sql
INSERT INTO history SELECT * FROM history_part;
INSERT INTO trends SELECT * FROM trends_part;
-- Repeat for all tables you wish to restore
```
4. **Cleanup:** Once you have migrated the data you need (or if you don't need it at all), you can drop the heavy partitioned tables and remove the partitioning extensions completely.
```sql
DROP TABLE history_part CASCADE;
DROP TABLE history_uint_part CASCADE;
-- Repeat for all *_part tables ...
-- To drop the automatic maintenance infrastructure:
DROP SCHEMA partitions CASCADE;
```
5. **Start Zabbix Server & Re-enable Housekeeper:** Once the tables are replaced, you can start the server. *Don't forget to re-enable Housekeeping for History and Trends in the Zabbix UI!*
## Upgrades
1. **Backup**: Ensure a full database backup exists.
2. **Compatibility**: Zabbix upgrade scripts may attempt to `ALTER` tables. PostgreSQL supports `ALTER TABLE` on partitioned tables for adding columns, which propagates to partitions.
3. **Failure Scenarios**: If an upgrade script fails due to partitioning, the table may need to be temporarily reverted or the partition structure manually adjusted.