Synced procedure scripts from test
This commit is contained in:
@@ -1,7 +1,6 @@
|
|||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- SCRIPT: 00_partitions_init.sql
|
-- Creates the 'partitions' schema and configuration table.
|
||||||
-- DESCRIPTION: Creates the 'partitions' schema and configuration table.
|
-- Defines the structure for managing Zabbix partitioning.
|
||||||
-- Defines the structure for managing Zabbix partitioning.
|
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
|
|
||||||
CREATE SCHEMA IF NOT EXISTS partitions;
|
CREATE SCHEMA IF NOT EXISTS partitions;
|
||||||
@@ -12,14 +11,14 @@ CREATE TABLE IF NOT EXISTS partitions.config (
|
|||||||
period text NOT NULL CHECK (period IN ('day', 'week', 'month', 'year')),
|
period text NOT NULL CHECK (period IN ('day', 'week', 'month', 'year')),
|
||||||
keep_history interval NOT NULL,
|
keep_history interval NOT NULL,
|
||||||
future_partitions integer NOT NULL DEFAULT 5,
|
future_partitions integer NOT NULL DEFAULT 5,
|
||||||
last_updated timestamp WITH TIME ZONE DEFAULT now(),
|
last_updated timestamp WITH TIME ZONE DEFAULT (now() AT TIME ZONE 'UTC'),
|
||||||
PRIMARY KEY (table_name)
|
PRIMARY KEY (table_name)
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Table to track installed version of the partitioning solution
|
-- Table to track installed version of the partitioning solution
|
||||||
CREATE TABLE IF NOT EXISTS partitions.version (
|
CREATE TABLE IF NOT EXISTS partitions.version (
|
||||||
version text PRIMARY KEY,
|
version text PRIMARY KEY,
|
||||||
installed_at timestamp with time zone DEFAULT now(),
|
installed_at timestamp with time zone DEFAULT (now() AT TIME ZONE 'UTC'),
|
||||||
description text
|
description text
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
194
postgresql/procedures/01_maintenance.sql
Normal file
194
postgresql/procedures/01_maintenance.sql
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
-- ============================================================================
|
||||||
|
-- Core functions for Zabbix partitioning (Create, Drop, Maintain).
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Function to check if a partition exists
|
||||||
|
CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text)
|
||||||
|
RETURNS boolean AS $$
|
||||||
|
BEGIN
|
||||||
|
RETURN EXISTS (
|
||||||
|
SELECT 1 FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE c.relname = p_partition_name
|
||||||
|
);
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
-- Function to create a partition
|
||||||
|
CREATE OR REPLACE PROCEDURE partitions.create_partition(
|
||||||
|
p_parent_table text,
|
||||||
|
p_start_time timestamp with time zone,
|
||||||
|
p_end_time timestamp with time zone,
|
||||||
|
p_period text
|
||||||
|
) LANGUAGE plpgsql AS $$
|
||||||
|
DECLARE
|
||||||
|
v_partition_name text;
|
||||||
|
v_start_ts bigint;
|
||||||
|
v_end_ts bigint;
|
||||||
|
v_suffix text;
|
||||||
|
v_parent_schema text;
|
||||||
|
BEGIN
|
||||||
|
-- Determine the schema of the parent table
|
||||||
|
SELECT n.nspname INTO v_parent_schema
|
||||||
|
FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE c.relname = p_parent_table;
|
||||||
|
|
||||||
|
IF NOT FOUND THEN
|
||||||
|
RAISE EXCEPTION 'Parent table % not found', p_parent_table;
|
||||||
|
END IF;
|
||||||
|
-- (No changes needed for time here as passed params are already UTC-adjusted in caller)
|
||||||
|
v_start_ts := extract(epoch from p_start_time)::bigint;
|
||||||
|
v_end_ts := extract(epoch from p_end_time)::bigint;
|
||||||
|
|
||||||
|
IF p_period = 'month' THEN
|
||||||
|
v_suffix := to_char(p_start_time, 'YYYYMM');
|
||||||
|
ELSE
|
||||||
|
v_suffix := to_char(p_start_time, 'YYYYMMDD');
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
v_partition_name := p_parent_table || '_p' || v_suffix;
|
||||||
|
|
||||||
|
IF NOT partitions.partition_exists(v_partition_name) THEN
|
||||||
|
EXECUTE format(
|
||||||
|
'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)',
|
||||||
|
v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts
|
||||||
|
);
|
||||||
|
END IF;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
-- Function to drop old partitions
|
||||||
|
CREATE OR REPLACE PROCEDURE partitions.drop_old_partitions(
|
||||||
|
p_parent_table text,
|
||||||
|
p_retention interval,
|
||||||
|
p_period text
|
||||||
|
) LANGUAGE plpgsql AS $$
|
||||||
|
DECLARE
|
||||||
|
v_cutoff_ts bigint;
|
||||||
|
v_partition record;
|
||||||
|
v_partition_date timestamp with time zone;
|
||||||
|
v_suffix text;
|
||||||
|
v_partition_schema text;
|
||||||
|
BEGIN
|
||||||
|
-- Calculate cutoff timestamp
|
||||||
|
v_cutoff_ts := extract(epoch from (now() - p_retention))::bigint;
|
||||||
|
|
||||||
|
FOR v_partition IN
|
||||||
|
SELECT
|
||||||
|
child.relname AS partition_name,
|
||||||
|
n.nspname AS partition_schema
|
||||||
|
FROM pg_inherits
|
||||||
|
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
||||||
|
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
||||||
|
JOIN pg_namespace n ON child.relnamespace = n.oid
|
||||||
|
WHERE parent.relname = p_parent_table
|
||||||
|
LOOP
|
||||||
|
-- Parse partition suffix to determine age
|
||||||
|
-- Format: parent_pYYYYMM or parent_pYYYYMMDD
|
||||||
|
v_suffix := substring(v_partition.partition_name from length(p_parent_table) + 3);
|
||||||
|
|
||||||
|
BEGIN
|
||||||
|
IF length(v_suffix) = 6 THEN -- YYYYMM
|
||||||
|
v_partition_date := to_timestamp(v_suffix || '01', 'YYYYMMDD') AT TIME ZONE 'UTC';
|
||||||
|
-- For monthly, we check if the END of the month is older than retention?
|
||||||
|
-- Or just strict retention.
|
||||||
|
-- To be safe, adding 1 month to check vs cutoff.
|
||||||
|
IF extract(epoch from (v_partition_date + '1 month'::interval)) < v_cutoff_ts THEN
|
||||||
|
RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
|
||||||
|
EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name);
|
||||||
|
COMMIT; -- Release lock immediately
|
||||||
|
END IF;
|
||||||
|
ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD
|
||||||
|
v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC';
|
||||||
|
IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN
|
||||||
|
RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
|
||||||
|
EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name);
|
||||||
|
COMMIT; -- Release lock immediately
|
||||||
|
END IF;
|
||||||
|
END IF;
|
||||||
|
EXCEPTION WHEN OTHERS THEN
|
||||||
|
-- Ignore parsing errors for non-standard partitions
|
||||||
|
NULL;
|
||||||
|
END;
|
||||||
|
END LOOP;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
-- MAIN Procedure to maintain a single table
|
||||||
|
CREATE OR REPLACE PROCEDURE partitions.maintain_table(
|
||||||
|
p_table_name text,
|
||||||
|
p_period text,
|
||||||
|
p_keep_history interval,
|
||||||
|
p_future_partitions integer DEFAULT 5
|
||||||
|
) LANGUAGE plpgsql AS $$
|
||||||
|
DECLARE
|
||||||
|
v_start_time timestamp with time zone;
|
||||||
|
v_period_interval interval;
|
||||||
|
i integer;
|
||||||
|
v_past_iterations integer;
|
||||||
|
BEGIN
|
||||||
|
IF p_period = 'day' THEN
|
||||||
|
v_period_interval := '1 day'::interval;
|
||||||
|
v_start_time := date_trunc('day', now() AT TIME ZONE 'UTC');
|
||||||
|
-- Calculate how many past days cover the retention period (86400 seconds = 1 day)
|
||||||
|
v_past_iterations := ceil(extract(epoch from p_keep_history) / 86400)::integer;
|
||||||
|
|
||||||
|
ELSIF p_period = 'week' THEN
|
||||||
|
v_period_interval := '1 week'::interval;
|
||||||
|
v_start_time := date_trunc('week', now() AT TIME ZONE 'UTC');
|
||||||
|
-- 604800 seconds = 1 week
|
||||||
|
v_past_iterations := ceil(extract(epoch from p_keep_history) / 604800)::integer;
|
||||||
|
|
||||||
|
ELSIF p_period = 'month' THEN
|
||||||
|
v_period_interval := '1 month'::interval;
|
||||||
|
v_start_time := date_trunc('month', now() AT TIME ZONE 'UTC');
|
||||||
|
-- Approximate 30 days per month (2592000 seconds)
|
||||||
|
v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer;
|
||||||
|
ELSE
|
||||||
|
RETURN;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- 1. Create Future Partitions (Current + Buffer)
|
||||||
|
FOR i IN 0..p_future_partitions LOOP
|
||||||
|
CALL partitions.create_partition(
|
||||||
|
p_table_name,
|
||||||
|
v_start_time + (i * v_period_interval),
|
||||||
|
v_start_time + ((i + 1) * v_period_interval),
|
||||||
|
p_period
|
||||||
|
);
|
||||||
|
COMMIT; -- Release lock immediately
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
-- 2. Create Past Partitions (Covering retention period)
|
||||||
|
IF v_past_iterations > 0 THEN
|
||||||
|
FOR i IN 1..v_past_iterations LOOP
|
||||||
|
CALL partitions.create_partition(
|
||||||
|
p_table_name,
|
||||||
|
v_start_time - (i * v_period_interval),
|
||||||
|
v_start_time - ((i - 1) * v_period_interval),
|
||||||
|
p_period
|
||||||
|
);
|
||||||
|
COMMIT; -- Release lock immediately
|
||||||
|
END LOOP;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- 3. Drop Old Partitions
|
||||||
|
CALL partitions.drop_old_partitions(p_table_name, p_keep_history, p_period);
|
||||||
|
|
||||||
|
-- 4. Update Metadata
|
||||||
|
UPDATE partitions.config SET last_updated = now() WHERE table_name = p_table_name;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
-- Global Maintenance Procedure
|
||||||
|
CREATE OR REPLACE PROCEDURE partitions.run_maintenance()
|
||||||
|
LANGUAGE plpgsql AS $$
|
||||||
|
DECLARE
|
||||||
|
v_row record;
|
||||||
|
BEGIN
|
||||||
|
FOR v_row IN SELECT * FROM partitions.config LOOP
|
||||||
|
CALL partitions.maintain_table(v_row.table_name, v_row.period, v_row.keep_history, v_row.future_partitions);
|
||||||
|
END LOOP;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
56
postgresql/procedures/02_enable_partitioning.sql
Normal file
56
postgresql/procedures/02_enable_partitioning.sql
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
-- ============================================================================
|
||||||
|
-- Converts standard Zabbix tables to Partitioned tables.
|
||||||
|
-- WARNING: This renames existing tables to *_old.
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
v_row record;
|
||||||
|
v_table text;
|
||||||
|
v_old_table text;
|
||||||
|
v_pk_sql text;
|
||||||
|
v_schema text;
|
||||||
|
BEGIN
|
||||||
|
FOR v_row IN SELECT * FROM partitions.config LOOP
|
||||||
|
v_table := v_row.table_name;
|
||||||
|
v_old_table := v_table || '_old';
|
||||||
|
|
||||||
|
-- Determine schema
|
||||||
|
SELECT n.nspname INTO v_schema
|
||||||
|
FROM pg_class c
|
||||||
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||||
|
WHERE c.relname = v_table;
|
||||||
|
|
||||||
|
|
||||||
|
IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
|
||||||
|
RAISE NOTICE 'Converting table % to partitioned table...', v_table;
|
||||||
|
|
||||||
|
-- 1. Rename existing table
|
||||||
|
EXECUTE format('ALTER TABLE %I.%I RENAME TO %I', v_schema, v_table, v_old_table);
|
||||||
|
|
||||||
|
-- 2. Create new partitioned table (handling auditlog PK uniquely)
|
||||||
|
IF v_table = 'auditlog' THEN
|
||||||
|
EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING DEFAULTS INCLUDING COMMENTS) PARTITION BY RANGE (clock)', v_schema, v_table, v_schema, v_old_table);
|
||||||
|
EXECUTE format('ALTER TABLE %I.%I ADD PRIMARY KEY (auditid, clock)', v_schema, v_table);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_1 ON %I.%I (userid, clock)', v_schema, v_table);
|
||||||
|
EXECUTE format('CREATE INDEX IF NOT EXISTS auditlog_2 ON %I.%I (clock)', v_schema, v_table);
|
||||||
|
ELSE
|
||||||
|
EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING ALL) PARTITION BY RANGE (clock)', v_schema, v_table, v_schema, v_old_table);
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- 3. Create initial partitions
|
||||||
|
RAISE NOTICE 'Creating initial partitions for %...', v_table;
|
||||||
|
CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history, v_row.future_partitions);
|
||||||
|
|
||||||
|
-- Optional: Migrate existing data
|
||||||
|
-- EXECUTE format('INSERT INTO %I.%I SELECT * FROM %I.%I', v_schema, v_table, v_schema, v_old_table);
|
||||||
|
|
||||||
|
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN
|
||||||
|
RAISE NOTICE 'Table % is already partitioned. Skipping conversion.', v_table;
|
||||||
|
-- Just run maintenance to ensure partitions exist
|
||||||
|
CALL partitions.run_maintenance();
|
||||||
|
ELSE
|
||||||
|
RAISE WARNING 'Table % not found!', v_table;
|
||||||
|
END IF;
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
27
postgresql/procedures/03_monitoring_view.sql
Normal file
27
postgresql/procedures/03_monitoring_view.sql
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
-- ============================================================================
|
||||||
|
-- Creates a view to monitor partition status and sizes.
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW partitions.monitoring AS
|
||||||
|
SELECT
|
||||||
|
parent.relname AS parent_table,
|
||||||
|
c.table_name,
|
||||||
|
c.period,
|
||||||
|
c.keep_history,
|
||||||
|
count(child.relname) AS partition_count,
|
||||||
|
count(child.relname) FILTER (
|
||||||
|
WHERE
|
||||||
|
(c.period = 'day' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMMDD')))
|
||||||
|
OR
|
||||||
|
(c.period = 'month' AND child.relname > (parent.relname || '_p' || to_char(now(), 'YYYYMM')))
|
||||||
|
) AS future_partitions,
|
||||||
|
pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size,
|
||||||
|
min(child.relname) AS oldest_partition,
|
||||||
|
max(child.relname) AS newest_partition,
|
||||||
|
c.last_updated
|
||||||
|
FROM partitions.config c
|
||||||
|
JOIN pg_class parent ON parent.relname = c.table_name
|
||||||
|
LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid
|
||||||
|
LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
||||||
|
WHERE parent.relkind = 'p' -- Only partitioned tables
|
||||||
|
GROUP BY parent.relname, c.table_name, c.period, c.keep_history, c.last_updated;
|
||||||
@@ -2,6 +2,12 @@
|
|||||||
|
|
||||||
This is the declarative (PostgreSQL procedures based) partitioning implementation for Zabbix `history`, `trends`, and `auditlog` tables on PostgreSQL. This solution is intended to replace standard Zabbix housekeeping for the configured tables. Partitioning is very useful for large environments because it completely eliminates the housekeeper from the process. Instead of huge DELETE queries on several million rows, fast DDL queries (ALTER TABLE) are executed, which drop an entire partition.
|
This is the declarative (PostgreSQL procedures based) partitioning implementation for Zabbix `history`, `trends`, and `auditlog` tables on PostgreSQL. This solution is intended to replace standard Zabbix housekeeping for the configured tables. Partitioning is very useful for large environments because it completely eliminates the housekeeper from the process. Instead of huge DELETE queries on several million rows, fast DDL queries (ALTER TABLE) are executed, which drop an entire partition.
|
||||||
|
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> **High-Load Environments**:
|
||||||
|
> 1. **Data Visibility**: After enabling partitioning, old data remains in `*_old` tables and is **NOT visible** in Zabbix. You must migrate data manually if needed.
|
||||||
|
> 2. **Disable Housekeeping**: You **MUST** disable Zabbix Housekeeper for History and Trends in *Administration -> Housekeeping*. Failure to do so will cause massive `DELETE` loads.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
The solution uses PostgreSQL native declarative partitioning (`PARTITION BY RANGE`).
|
The solution uses PostgreSQL native declarative partitioning (`PARTITION BY RANGE`).
|
||||||
@@ -13,14 +19,44 @@ All procedures, information, statistics and configuration are stored in the `par
|
|||||||
3. **Monitoring View**: `partitions.monitoring` provides system state visibility.
|
3. **Monitoring View**: `partitions.monitoring` provides system state visibility.
|
||||||
4. **Version Table**: `partitions.version` provides information about installed version of the partitioning solution.
|
4. **Version Table**: `partitions.version` provides information about installed version of the partitioning solution.
|
||||||
|
|
||||||
|
## Prerequisites: Database & User Creation
|
||||||
|
If you are deploying Zabbix on a fresh database instance (like AWS RDS) rather than a local server, you must first create the `zabbix` user and database using your administrator account (e.g., `postgres`).
|
||||||
|
|
||||||
|
1. Connect to your DB instance as the administrator:
|
||||||
|
```bash
|
||||||
|
psql "host=YOUR_RDS_HOST port=5432 user=postgres dbname=postgres sslmode=require"
|
||||||
|
```
|
||||||
|
2. Create the user and database:
|
||||||
|
```sql
|
||||||
|
CREATE USER zabbix WITH PASSWORD 'your_secure_password';
|
||||||
|
-- On Cloud DBs like RDS, the master user must inherit the new role to grant ownership
|
||||||
|
GRANT zabbix TO postgres;
|
||||||
|
CREATE DATABASE zabbix OWNER zabbix;
|
||||||
|
```
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
The installation is performed by executing the SQL procedures in the following order:
|
The installation is performed by executing the SQL procedures in the following order:
|
||||||
1. Initialize schema (`00_partitions_init.sql`).
|
1. Initialize schema (`00_partitions_init.sql`).
|
||||||
2. Auditlog PK adjustment (`01_auditlog_prep.sql`).
|
2. Install maintenance procedures (`01_maintenance.sql`).
|
||||||
3. Install maintenance procedures (`02_maintenance.sql`).
|
3. Enable partitioning on tables (`02_enable_partitioning.sql`).
|
||||||
4. Enable partitioning on tables (`03_enable_partitioning.sql`).
|
4. Install monitoring views (`03_monitoring_view.sql`).
|
||||||
5. Install monitoring views (`04_monitoring_view.sql`).
|
|
||||||
|
**Command Example:**
|
||||||
|
You can deploy these scripts manually against your Zabbix database using `psql`. Navigate to the `procedures/` directory and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Connect as the zabbix database user
|
||||||
|
export PGPASSWORD="your_zabbix_password"
|
||||||
|
DB_HOST="localhost" # Or your RDS endpoint
|
||||||
|
DB_NAME="zabbix"
|
||||||
|
DB_USER="zabbix"
|
||||||
|
|
||||||
|
for script in 00_partitions_init.sql 01_maintenance.sql 02_enable_partitioning.sql 03_monitoring_view.sql; do
|
||||||
|
echo "Applying $script..."
|
||||||
|
psql -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
@@ -55,7 +91,7 @@ This procedure should be scheduled to run periodically (e.g., daily via `pg_cron
|
|||||||
```sql
|
```sql
|
||||||
CALL partitions.run_maintenance();
|
CALL partitions.run_maintenance();
|
||||||
```
|
```
|
||||||
### Automatic Maintenance (Cron)
|
### Scheduling Maintenance
|
||||||
|
|
||||||
To ensure partitions are created in advance and old data is cleaned up, the maintenance procedure should be scheduled to run automatically.
|
To ensure partitions are created in advance and old data is cleaned up, the maintenance procedure should be scheduled to run automatically.
|
||||||
|
|
||||||
@@ -63,6 +99,102 @@ It is recommended to run the maintenance **twice a day** (e.g., at 05:30 and 23:
|
|||||||
* **Primary Run**: Creates new future partitions and drops old ones.
|
* **Primary Run**: Creates new future partitions and drops old ones.
|
||||||
* **Secondary Run**: Acts as a safety check. Since the procedure is idempotent (safe to run multiple times), a second run ensures everything is consistent if the first run failed or was interrupted.
|
* **Secondary Run**: Acts as a safety check. Since the procedure is idempotent (safe to run multiple times), a second run ensures everything is consistent if the first run failed or was interrupted.
|
||||||
|
|
||||||
|
You can schedule this using one of the following methods:
|
||||||
|
|
||||||
|
#### Option 1: `pg_cron` (Recommended)
|
||||||
|
`pg_cron` is a cron-based job scheduler that runs directly inside the database as an extension.
|
||||||
|
|
||||||
|
> [!NOTE]
|
||||||
|
> **Cloud Managed Databases (AWS RDS, Aurora, Azure, GCP):**
|
||||||
|
> Managed databases generally have `pg_cron` pre-installed and handle the authentication/connections securely for you automatically. You do **not** need to install OS packages or configure a `.pgpass` file! Simply modify your RDS Parameter Group to include `shared_preload_libraries = 'pg_cron'` and `cron.database_name = 'zabbix'`, reboot the instance, and execute `CREATE EXTENSION pg_cron;`.
|
||||||
|
|
||||||
|
**Setup `pg_cron` (Self-Hosted):**
|
||||||
|
1. Install the package via your OS package manager (e.g., `postgresql-15-cron` on Debian/Ubuntu, or `pg_cron_15` on RHEL/CentOS).
|
||||||
|
2. Configure it modifying `postgresql.conf`:
|
||||||
|
```ini
|
||||||
|
shared_preload_libraries = 'pg_cron'
|
||||||
|
cron.database_name = 'zabbix' # Define the database where pg_cron will run
|
||||||
|
```
|
||||||
|
3. Restart PostgreSQL:
|
||||||
|
```bash
|
||||||
|
systemctl restart postgresql
|
||||||
|
```
|
||||||
|
4. Connect to your `zabbix` database as a superuser and create the extension:
|
||||||
|
```sql
|
||||||
|
CREATE EXTENSION pg_cron;
|
||||||
|
```
|
||||||
|
5. Schedule the job to run:
|
||||||
|
```sql
|
||||||
|
SELECT cron.schedule('zabbix_partition_maintenance', '30 5,23 * * *', 'CALL partitions.run_maintenance();');
|
||||||
|
```
|
||||||
|
6. **Manage your `pg_cron` jobs** (run as superuser):
|
||||||
|
- To **list all active schedules**: `SELECT * FROM cron.job;`
|
||||||
|
- To **view execution logs/history**: `SELECT * FROM cron.job_run_details;`
|
||||||
|
- To **remove/unschedule** the job: `SELECT cron.unschedule('zabbix_partition_maintenance');`
|
||||||
|
|
||||||
|
**⚠️ Troubleshooting `pg_cron` Connection Errors:**
|
||||||
|
If your cron jobs fail to execute and you see `FATAL: password authentication failed` in your PostgreSQL logs, it is because `pg_cron` attempts to connect via TCP (`localhost`) by default, which usually requires a password.
|
||||||
|
|
||||||
|
**Solution A: Use Local Unix Sockets (Easier)**
|
||||||
|
Edit your `postgresql.conf` to force `pg_cron` to use the local Unix socket (which uses passwordless `peer` authentication):
|
||||||
|
```ini
|
||||||
|
cron.host = '/var/run/postgresql' # Or '/tmp', depending on your OS
|
||||||
|
```
|
||||||
|
*(Restart PostgreSQL after making this change).*
|
||||||
|
|
||||||
|
**Solution B: Provide a Password (`.pgpass`)**
|
||||||
|
If you *must* connect via TCP with a specific database user and password, the `pg_cron` background worker needs a way to authenticate. You provide this by creating a `.pgpass` file for the OS `postgres` user.
|
||||||
|
1. Switch to the OS database user:
|
||||||
|
```bash
|
||||||
|
sudo su - postgres
|
||||||
|
```
|
||||||
|
2. Create or append your database credentials to `~/.pgpass` using the format `hostname:port:database:username:password`:
|
||||||
|
```bash
|
||||||
|
echo "localhost:5432:zabbix:zabbix:my_secure_password" >> ~/.pgpass
|
||||||
|
```
|
||||||
|
3. Set strict permissions (PostgreSQL will ignore the file if permissions are too loose):
|
||||||
|
```bash
|
||||||
|
chmod 0600 ~/.pgpass
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Option 2: Systemd Timers
|
||||||
|
Systemd timers provide better logging and error handling properties than standard cron.
|
||||||
|
|
||||||
|
1. Create a service file **`/etc/systemd/system/zabbix-partitions.service`**:
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=Zabbix PostgreSQL Partition Maintenance
|
||||||
|
After=network.target postgresql.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=postgres
|
||||||
|
ExecStart=/usr/bin/psql -d zabbix -c "CALL partitions.run_maintenance();"
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Create a timer file **`/etc/systemd/system/zabbix-partitions.timer`**:
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=Run Zabbix Partition Maintenance Twice Daily
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 05:30:00
|
||||||
|
OnCalendar=*-*-* 23:30:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Enable and start the timer:
|
||||||
|
```bash
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable --now zabbix-partitions.timer
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Option 3: System Cron (`crontab`)
|
||||||
|
Standard system cron is a simple fallback.
|
||||||
|
|
||||||
**Example Crontab Entry (`crontab -e`):**
|
**Example Crontab Entry (`crontab -e`):**
|
||||||
```bash
|
```bash
|
||||||
# Run Zabbix partition maintenance twice daily (5:30 AM and 5:30 PM)
|
# Run Zabbix partition maintenance twice daily (5:30 AM and 5:30 PM)
|
||||||
@@ -70,7 +202,7 @@ It is recommended to run the maintenance **twice a day** (e.g., at 05:30 and 23:
|
|||||||
```
|
```
|
||||||
|
|
||||||
**Docker Environment:**
|
**Docker Environment:**
|
||||||
If running in Docker, you can execute it via the container:
|
If running in Docker, you can execute it via the host's cron by targeting the container:
|
||||||
```bash
|
```bash
|
||||||
30 5,23 * * * docker exec zabbix-db-test psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();"
|
30 5,23 * * * docker exec zabbix-db-test psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();"
|
||||||
```
|
```
|
||||||
@@ -100,13 +232,14 @@ GRANT SELECT ON partitions.monitoring TO zbx_monitor;
|
|||||||
## Implementation Details
|
## Implementation Details
|
||||||
|
|
||||||
### `auditlog` Table
|
### `auditlog` Table
|
||||||
The standard `auditlog` table Primary Key is `(auditid)`. Partitioning by `clock` requires the partition key to be part of the Primary Key. The initialization script modifies the PK to `(auditid, clock)`.
|
The standard Zabbix `auditlog` table has a primary key on `(auditid)`. Partitioning by `clock` requires the partition key to be part of the primary key.
|
||||||
|
To prevent placing a heavy, blocking lock on a highly active `auditlog` table to alter its primary key, the enablement script (`02_enable_partitioning.sql`) detects it and handles it exactly like the history tables: it automatically renames the live, existing table to `auditlog_old`, and instantly creates a brand new, empty partitioned `auditlog` table pre-configured with the required `(auditid, clock)` composite primary key.
|
||||||
|
|
||||||
### Converting Existing Tables
|
### Converting Existing Tables
|
||||||
The enablement script renames the existing table to `table_name_old` and creates a new partitioned table with the same structure.
|
The enablement script guarantees practically zero downtime by automatically renaming the existing tables to `table_name_old` and creating new partitioned tables matching the exact schema.
|
||||||
* **Note**: Data from the old table is NOT automatically migrated to minimize downtime.
|
* **Note**: Data from the old tables is NOT automatically migrated to minimize downtime.
|
||||||
* New data flows into the new partitioned table immediately.
|
* New data flows into the new partitioned tables immediately.
|
||||||
* Old data remains accessible in `table_name_old` for manual query or migration if required.
|
* Old data remains accessible in `table_name_old` for manual lookup or migration if required.
|
||||||
|
|
||||||
## Upgrades
|
## Upgrades
|
||||||
|
|
||||||
@@ -114,3 +247,40 @@ When upgrading Zabbix:
|
|||||||
1. **Backup**: Ensure a full database backup exists.
|
1. **Backup**: Ensure a full database backup exists.
|
||||||
2. **Compatibility**: Zabbix upgrade scripts may attempt to `ALTER` tables. PostgreSQL supports `ALTER TABLE` on partitioned tables for adding columns, which propagates to partitions.
|
2. **Compatibility**: Zabbix upgrade scripts may attempt to `ALTER` tables. PostgreSQL supports `ALTER TABLE` on partitioned tables for adding columns, which propagates to partitions.
|
||||||
3. **Failure Scenarios**: If an upgrade script fails due to partitioning, the table may need to be temporarily reverted or the partition structure manually adjusted.
|
3. **Failure Scenarios**: If an upgrade script fails due to partitioning, the table may need to be temporarily reverted or the partition structure manually adjusted.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Appendix: Zabbix Server & Frontend RDS Configuration
|
||||||
|
|
||||||
|
If you are running Zabbix against an external Cloud database (like AWS RDS) via SSL (`verify-full`), you must explicitly configure both the Zabbix Server daemon and the Web Frontend to enforce SSL and locate the downloaded Root CA Certificate.
|
||||||
|
|
||||||
|
**Prerequisite:** Download your cloud provider's root certificate (e.g., `global-bundle.pem`) and place it in a secure location on your Zabbix Server (e.g., `/etc/zabbix/global-bundle.pem`).
|
||||||
|
|
||||||
|
### 1. Zabbix Server (`/etc/zabbix/zabbix_server.conf`)
|
||||||
|
Ensure the following database lines are active:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
DBHost=YOUR_RDS_ENDPOINT.amazonaws.com
|
||||||
|
DBPort=5432
|
||||||
|
DBName=zabbix
|
||||||
|
DBUser=zabbix
|
||||||
|
DBPassword=your_secure_password
|
||||||
|
DBTLSConnect=verify_full
|
||||||
|
DBTLSCAFile=/etc/zabbix/global-bundle.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Zabbix Frontend PHP (`/etc/zabbix/web/zabbix.conf.php`)
|
||||||
|
If you used the Web Setup Wizard, it might not configure the Root CA File correctly. Update your config array to enforce encryption and verify the host certificate:
|
||||||
|
|
||||||
|
```php
|
||||||
|
$DB['TYPE'] = 'POSTGRESQL';
|
||||||
|
$DB['SERVER'] = 'YOUR_RDS_ENDPOINT.amazonaws.com';
|
||||||
|
$DB['PORT'] = '5432';
|
||||||
|
$DB['DATABASE'] = 'zabbix';
|
||||||
|
$DB['USER'] = 'zabbix';
|
||||||
|
$DB['PASSWORD'] = 'your_secure_password';
|
||||||
|
$DB['SCHEMA'] = '';
|
||||||
|
$DB['ENCRYPTION'] = true;
|
||||||
|
$DB['VERIFY_HOST'] = true;
|
||||||
|
$DB['CA_FILE'] = '/etc/zabbix/global-bundle.pem';
|
||||||
|
```
|
||||||
|
|||||||
Reference in New Issue
Block a user