Zabbix/partitioning
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Initial commit for 8.0 branch. history_json was added.

Browse Source
This commit is contained in:
Maksym Buz
2026-02-20 21:35:11 +00:00
parent 9d1b84225c
commit 9581dbdb62
31 changed files with 177 additions and 853991 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
postgresql/procedures/00_partitions_init.sql
View File

@@ -1,7 +1,6 @@
-- ============================================================================
-- SCRIPT: 00_partitions_init.sql
-- DESCRIPTION: Creates the 'partitions' schema and configuration table.
-- Defines the structure for managing Zabbix partitioning.
-- Creates the 'partitions' schema and configuration table.
-- Defines the structure for managing Zabbix partitioning.
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS partitions;
@@ -34,7 +33,8 @@ INSERT INTO partitions.config (table_name, period, keep_history) VALUES
('history_uint', 'day', '30 days'),
('history_str', 'day', '30 days'),
('history_log', 'day', '30 days'),
('history_text', 'day', '30 days')
('history_text', 'day', '30 days'),
('history_json', 'day', '30 days')
ON CONFLICT (table_name) DO NOTHING;
-- Trends tables: Monthly partitions, keep 12 months

19
postgresql/procedures/01_auditlog_prep.sql
View File

@@ -1,24 +1,23 @@
-- ============================================================================
-- SCRIPT: 01_auditlog_prep.sql
-- DESCRIPTION: Modifies the 'auditlog' table Primary Key to include 'clock'.
-- This is REQUIRED for range partitioning by 'clock'.
-- Modifies the 'auditlog' table Primary Key to include 'clock'.
-- This is REQUIRED for range partitioning by 'clock'.
-- ============================================================================
DO $$
BEGIN
-- Check if PK needs modification
-- Original PK is typically on (auditid) named 'auditlog_pkey'
-- Original PK is on auditid named 'auditlog_pkey'
IF EXISTS (
SELECT 1 FROM pg_constraint
WHERE conname = 'auditlog_pkey'
SELECT 1 FROM pg_constraint
WHERE conname = 'auditlog_pkey'
AND conrelid = 'auditlog'::regclass
) THEN
-- Verify if 'clock' is already in PK (basic check)
-- Realistically, if 'auditlog_pkey' exists on default Zabbix, it's just (auditid).
-- Verify if 'clock' is already in PK (basic safety check)
-- Realistically, if 'auditlog_pkey' exists on default Zabbix, it's just auditid.
RAISE NOTICE 'Dropping existing Primary Key on auditlog...';
ALTER TABLE auditlog DROP CONSTRAINT auditlog_pkey;
RAISE NOTICE 'Creating new Primary Key on auditlog (auditid, clock)...';
ALTER TABLE auditlog ADD PRIMARY KEY (auditid, clock);
ELSE

28
postgresql/procedures/02_maintenance.sql
View File

@@ -1,6 +1,5 @@
-- ============================================================================
-- SCRIPT: 02_maintenance.sql
-- DESCRIPTION: Core functions for Zabbix partitioning (Create, Drop, Maintain).
-- Core functions for Zabbix partitioning (Create, Drop, Maintain).
-- ============================================================================
-- Function to check if a partition exists
@@ -11,7 +10,6 @@ BEGIN
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = p_partition_name
AND n.nspname = 'public'
);
END;
$$ LANGUAGE plpgsql;
@@ -28,8 +26,17 @@ DECLARE
v_start_ts bigint;
v_end_ts bigint;
v_suffix text;
v_parent_schema text;
BEGIN
-- (No changes needed for time here as passed params are already UTC-adjusted in caller)
-- Determine the schema of the parent table
SELECT n.nspname INTO v_parent_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = p_parent_table;
IF NOT FOUND THEN
RAISE EXCEPTION 'Parent table % not found', p_parent_table;
END IF;
v_start_ts := extract(epoch from p_start_time)::bigint;
v_end_ts := extract(epoch from p_end_time)::bigint;
@@ -43,8 +50,8 @@ BEGIN
IF NOT partitions.partition_exists(v_partition_name) THEN
EXECUTE format(
'CREATE TABLE public.%I PARTITION OF public.%I FOR VALUES FROM (%s) TO (%s)',
v_partition_name, p_parent_table, v_start_ts, v_end_ts
'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)',
v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts
);
END IF;
END;
@@ -61,16 +68,19 @@ DECLARE
v_partition record;
v_partition_date timestamp with time zone;
v_suffix text;
v_partition_schema text;
BEGIN
-- Calculate cutoff timestamp
v_cutoff_ts := extract(epoch from (now() - p_retention))::bigint;
FOR v_partition IN
SELECT
child.relname AS partition_name
child.relname AS partition_name,
n.nspname AS partition_schema
FROM pg_inherits
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
JOIN pg_namespace n ON child.relnamespace = n.oid
WHERE parent.relname = p_parent_table
LOOP
-- Parse partition suffix to determine age
@@ -85,14 +95,14 @@ BEGIN
-- To be safe, adding 1 month to check vs cutoff.
IF extract(epoch from (v_partition_date + '1 month'::interval)) < v_cutoff_ts THEN
RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
EXECUTE format('DROP TABLE public.%I', v_partition.partition_name);
EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name);
COMMIT; -- Release lock immediately
END IF;
ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD
v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC';
IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN
RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
EXECUTE format('DROP TABLE public.%I', v_partition.partition_name);
EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name);
COMMIT; -- Release lock immediately
END IF;
END IF;

19
postgresql/procedures/03_enable_partitioning.sql
View File

@@ -1,7 +1,6 @@
-- ============================================================================
-- SCRIPT: 03_enable_partitioning.sql
-- DESCRIPTION: Converts standard Zabbix tables to Partitioned tables.
-- WARNING: This renames existing tables to *_old.
-- Converts Zabbix tables to Partitioned tables.
-- WARNING: This renames existing tables to *_old.
-- ============================================================================
DO $$
@@ -10,27 +9,35 @@ DECLARE
v_table text;
v_old_table text;
v_pk_sql text;
v_schema text;
BEGIN
FOR v_row IN SELECT * FROM partitions.config LOOP
v_table := v_row.table_name;
v_old_table := v_table || '_old';
-- Determine schema
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = v_table;
-- Check if table exists and is NOT already partitioned
IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
RAISE NOTICE 'Converting table % to partitioned table...', v_table;
-- 1. Rename existing table
EXECUTE format('ALTER TABLE public.%I RENAME TO %I', v_table, v_old_table);
EXECUTE format('ALTER TABLE %I.%I RENAME TO %I', v_schema, v_table, v_old_table);
-- 2. Create new partitioned table (copying structure)
EXECUTE format('CREATE TABLE public.%I (LIKE public.%I INCLUDING ALL) PARTITION BY RANGE (clock)', v_table, v_old_table);
EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING ALL) PARTITION BY RANGE (clock)', v_schema, v_table, v_schema, v_old_table);
-- 3. Create initial partitions
RAISE NOTICE 'Creating initial partitions for %...', v_table;
CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history, v_row.future_partitions);
-- Optional: Migrate existing data
-- EXECUTE format('INSERT INTO public.%I SELECT * FROM public.%I', v_table, v_old_table);
-- EXECUTE format('INSERT INTO %I.%I SELECT * FROM %I.%I', v_schema, v_table, v_schema, v_old_table);
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN
RAISE NOTICE 'Table % is already partitioned. Skipping conversion.', v_table;

3
postgresql/procedures/04_monitoring_view.sql
View File

@@ -1,6 +1,5 @@
-- ============================================================================
-- SCRIPT: 04_monitoring_view.sql
-- DESCRIPTION: Creates a view to monitor partition status and sizes.
-- Creates a view to monitor partition status and sizes.
-- ============================================================================
CREATE OR REPLACE VIEW partitions.monitoring AS

62
postgresql/procedures/README.md
View File

@@ -61,7 +61,7 @@ This procedure should be scheduled to run periodically (e.g., daily via `pg_cron
```sql
CALL partitions.run_maintenance();
```
### Automatic Maintenance (Cron)
### Automatic Maintenance
To ensure partitions are created in advance and old data is cleaned up, the maintenance procedure should be scheduled to run automatically.
@@ -69,16 +69,70 @@ It is recommended to run the maintenance **twice a day** (e.g., at 05:30 and 23:
* **Primary Run**: Creates new future partitions and drops old ones.
* **Secondary Run**: Acts as a safety check. Since the procedure is idempotent (safe to run multiple times), a second run ensures everything is consistent if the first run failed or was interrupted.
There are three ways to schedule this, depending on your environment:
#### Option 1: `pg_cron` (If you use RDS/Aurora)
If you are running on managed PostgreSQL (like AWS Aurora) or prefer to keep scheduling inside the database, `pg_cron` is the way to go.
1. Ensure `pg_cron` is installed and loaded in `postgresql.conf` (`shared_preload_libraries = 'pg_cron'`).
2. Run the following to schedule the maintenance:
```sql
CREATE EXTENSION IF NOT EXISTS pg_cron;
SELECT cron.schedule('zabbix_maintenance', '30 5,23 * * *', 'CALL partitions.run_maintenance();');
```
*Where:*
* `'zabbix_maintenance'` - The name of the job (must be unique).
* `'30 5,23 * * *'` - The standard cron schedule (runs at 05:30 and 23:30 daily).
* `'CALL partitions.run_maintenance();'` - The SQL command to execute.
#### Option 2: `systemd` Timers
For standard Linux VM deployments, `systemd` timers are modern, prevent overlapping runs, and provide excellent logging.
1. Create a service file (`/etc/systemd/system/zabbix-partitioning.service`):
```ini
[Unit]
Description=Zabbix PostgreSQL Partition Maintenance
[Service]
Type=oneshot
User=zabbix
# Ensure .pgpass is configured for the zabbix user so it doesn't prompt for a password
ExecStart=/usr/bin/psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();"
```
2. Create a timer file (`/etc/systemd/system/zabbix-partitioning.timer`):
```ini
[Unit]
Description=Zabbix Partitioning twice a day
[Timer]
OnCalendar=*-*-* 05,23:30:00
Persistent=true
[Install]
WantedBy=timers.target
```
3. Enable and start the timer:
```bash
systemctl daemon-reload
systemctl enable --now zabbix-partitioning.timer
```
#### Option 3: Standard Cron
This is the legacy, simple method for standard VMs and containerized environments.
**Example Crontab Entry (`crontab -e`):**
```bash
# Run Zabbix partition maintenance twice daily (5:30 AM and 5:30 PM)
# Run Zabbix partition maintenance twice daily (5:30 AM and 11:30 PM)
30 5,23 * * * psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();" >> /var/log/zabbix_maintenance.log 2>&1
```
**Docker Environment:**
If running in Docker, you can execute it via the container:
If running in Docker, you can execute it via the container's host:
```bash
30 5,23 * * * docker exec zabbix-db-test psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();"
30 5,23 * * * docker exec zabbix-db psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();"
```
## Monitoring & Permissions