Zabbix/partitioning
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enterprise audit fixes (schema resolution, race conditions, documentation)

Browse Source
This commit is contained in:
Maksym Buz
2026-04-30 10:49:53 +00:00
parent 59b2780125
commit c401efea51
9 changed files with 232 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
postgresql/procedures/00_schema_create.sql
View File

@@ -32,9 +32,16 @@ INSERT INTO partitions.config (table_name, period, keep_history) VALUES
('history_uint', 'day', '30 days'),
('history_str', 'day', '30 days'),
('history_log', 'day', '30 days'),
('history_text', 'day', '30 days')
('history_text', 'day', '30 days'),
('history_bin', 'day', '30 days')
ON CONFLICT (table_name) DO NOTHING;
-- Zabbix 8.0+ only: Uncomment the following lines if running Zabbix 8.0 or later
-- INSERT INTO partitions.config (table_name, period, keep_history) VALUES
-- ('history_json', 'day', '30 days')
-- ON CONFLICT (table_name) DO NOTHING;
-- Trends tables: Monthly partitions, keep 12 months
INSERT INTO partitions.config (table_name, period, keep_history) VALUES
('trends', 'month', '12 months'),

21
postgresql/procedures/01_maintenance.sql
View File

@@ -2,14 +2,15 @@
-- Core functions for Zabbix partitioning (Create, Drop, Maintain).
-- ============================================================================
-- Function to check if a partition exists
CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text)
-- Function to check if a partition exists in a specific schema
CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text, p_schema text)
RETURNS boolean AS $$
BEGIN
RETURN EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = p_partition_name
AND n.nspname = p_schema
);
END;
$$ LANGUAGE plpgsql;
@@ -32,7 +33,7 @@ BEGIN
SELECT n.nspname INTO v_parent_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = p_parent_table;
WHERE c.relname = p_parent_table AND pg_table_is_visible(c.oid);
IF NOT FOUND THEN
RAISE EXCEPTION 'Parent table % not found', p_parent_table;
@@ -51,15 +52,19 @@ BEGIN
v_partition_name := p_parent_table || '_p' || v_suffix;
IF NOT partitions.partition_exists(v_partition_name) THEN
IF NOT partitions.partition_exists(v_partition_name, v_parent_schema) THEN
BEGIN
EXECUTE format(
'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)',
v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts
);
EXCEPTION WHEN invalid_object_definition THEN
-- Ignore overlap errors (e.g., when transitioning from daily to hourly partitioning)
RAISE NOTICE 'Partition % overlaps with an existing partition. Skipping.', v_partition_name;
EXCEPTION
WHEN invalid_object_definition THEN
-- Ignore overlap errors (e.g., when transitioning from daily to hourly partitioning)
RAISE NOTICE 'Partition % overlaps with an existing partition. Skipping.', v_partition_name;
WHEN duplicate_table THEN
-- Ignore race condition: another process created the partition concurrently
RAISE NOTICE 'Partition % already exists (concurrent creation). Skipping.', v_partition_name;
END;
END IF;
END;
@@ -89,7 +94,7 @@ BEGIN
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
JOIN pg_namespace n ON child.relnamespace = n.oid
WHERE parent.relname = p_parent_table
WHERE parent.relname = p_parent_table AND pg_table_is_visible(parent.oid)
LOOP
-- Parse partition suffix to determine age
-- Format: parent_pYYYYMM or parent_pYYYYMMDD

40
postgresql/procedures/02_enable_partitioning.sql
View File

@@ -19,10 +19,10 @@ BEGIN
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = v_table;
WHERE c.relname = v_table AND pg_table_is_visible(c.oid);
IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r' AND pg_table_is_visible(oid)) THEN
RAISE NOTICE 'Converting table % to partitioned table...', v_table;
-- 1. Rename existing table
@@ -48,19 +48,37 @@ BEGIN
-- Optional: Migrate existing data
-- EXECUTE format('INSERT INTO %I.%I SELECT * FROM %I.%I', v_schema, v_table, v_schema, v_old_table);
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p' AND pg_table_is_visible(oid)) THEN
RAISE NOTICE 'Table % is already partitioned. Skipping conversion.', v_table;
-- Just run maintenance to ensure partitions exist
CALL partitions.run_maintenance();
-- Just run maintenance for this specific table to ensure partitions exist
CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history, v_row.future_partitions);
ELSE
RAISE WARNING 'Table % not found!', v_table;
END IF;
END LOOP;
-- Attach trigger to housekeeper table to silently discard tasks for partitioned tables.
-- Dynamically determine the schema of the housekeeper table to support custom schemas.
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = 'housekeeper' AND pg_table_is_visible(c.oid);
IF v_schema IS NOT NULL THEN
EXECUTE format('DROP TRIGGER IF EXISTS housekeeper_filter ON %I.housekeeper', v_schema);
EXECUTE format('CREATE TRIGGER housekeeper_filter BEFORE INSERT ON %I.housekeeper FOR EACH ROW EXECUTE FUNCTION partitions.housekeeper_insert_trigger()', v_schema);
RAISE NOTICE 'Housekeeper intercept trigger installed on %.housekeeper', v_schema;
ELSE
RAISE WARNING 'housekeeper table not found — trigger NOT installed!';
END IF;
END $$;
-- Attach trigger to housekeeper table to silently discard tasks for partitioned tables
DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper;
CREATE TRIGGER housekeeper_filter
BEFORE INSERT ON housekeeper
FOR EACH ROW
EXECUTE FUNCTION partitions.housekeeper_insert_trigger();
-- ==========================================================================
-- IMPORTANT: If the Zabbix Server connects with a non-superuser (e.g., 'zabbix'),
-- that user MUST have access to the partitions schema for the housekeeper trigger
-- to work. Without these GRANTs, every INSERT into housekeeper will FAIL.
-- Uncomment and adjust the username below:
-- ==========================================================================
-- GRANT USAGE ON SCHEMA partitions TO zabbix;
-- GRANT SELECT ON partitions.config TO zabbix;

5
postgresql/procedures/03_monitoring_view.sql
View File

@@ -2,8 +2,7 @@
-- Creates a view to monitor partition status and sizes.
-- ============================================================================
DROP VIEW IF EXISTS partitions.monitoring;
CREATE VIEW partitions.monitoring AS
CREATE OR REPLACE VIEW partitions.monitoring AS
SELECT
parent.relname AS parent_table,
c.table_name,
@@ -27,7 +26,7 @@ SELECT
max(child.relname) AS newest_partition,
c.last_updated
FROM partitions.config c
JOIN pg_class parent ON parent.relname = c.table_name
JOIN pg_class parent ON parent.relname = c.table_name AND pg_table_is_visible(parent.oid)
LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid
LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid
WHERE parent.relkind = 'p' -- Only partitioned tables

18
postgresql/procedures/04_undo_partitioning.sql
View File

@@ -18,7 +18,7 @@ BEGIN
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = v_table AND c.relkind = 'p';
WHERE c.relname = v_table AND c.relkind = 'p' AND pg_table_is_visible(c.oid);
IF v_schema IS NOT NULL THEN
RAISE NOTICE 'Reverting partitioned table %...', v_table;
@@ -47,15 +47,25 @@ BEGIN
RAISE NOTICE 'SUCCESS: % reverted to default. Partitioned data stored in % (You can DROP TABLE % CASCADE; later).', v_table, v_part_table, v_part_table;
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN
ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r' AND pg_table_is_visible(oid)) THEN
RAISE NOTICE 'Table % is already a regular table. Skipping.', v_table;
ELSE
RAISE WARNING 'Partitioned table % not found!', v_table;
END IF;
END LOOP;
-- Drop the housekeeper intercept trigger
DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper;
-- Drop the housekeeper intercept trigger (dynamically determine schema for custom schema support)
SELECT n.nspname INTO v_schema
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = 'housekeeper' AND pg_table_is_visible(c.oid);
IF v_schema IS NOT NULL THEN
EXECUTE format('DROP TRIGGER IF EXISTS housekeeper_filter ON %I.housekeeper', v_schema);
RAISE NOTICE 'Housekeeper intercept trigger removed from %.housekeeper', v_schema;
ELSE
RAISE WARNING 'housekeeper table not found — trigger removal skipped.';
END IF;
RAISE NOTICE '================================================================================';
RAISE NOTICE 'Undo complete. Partitioned tables have been renamed to *_part.';

161
postgresql/procedures/MANUAL.md Normal file
View File

@@ -0,0 +1,161 @@
# Zabbix Partitioning Deployment Manual
This guide provides a step-by-step process for deploying the PostgreSQL partitioning solution for Zabbix.
**🚨 DANGER: CRITICAL WARNING 🚨**
**BEFORE YOU PROCEED, YOU ABSOLUTELY MUST TAKE A FULL BACKUP OF YOUR ZABBIX DATABASE.**
**DO NOT SKIP THIS STEP. Schema modifications are dangerous. If something goes wrong and you do not have a backup, your historical data will be lost permanently, and we take ZERO responsibility.**
---
## Step 1: Preparation & Safety
Because database migrations can take time (especially on large tables), **never** run these scripts directly in a standard SSH session that might disconnect.
1. Open a safe terminal session using `tmux` or `screen`:
```bash
tmux new -s zabbix_partitioning
# OR
screen -S zabbix_partitioning
```
2. Disable the Zabbix Housekeeper for History and Trends:
- Go to your Zabbix Web UI -> **Administration** -> **Housekeeping**.
- **Uncheck** "Enable internal housekeeping" for **History and Trends**.
- Click **Update**.
3. Stop your Zabbix Server to ensure no new data is being written during the schema migration:
```bash
sudo systemctl stop zabbix-server
```
---
## Step 2: Database Connection & Schema Selection
Connect to your PostgreSQL server as an administrator (e.g., `postgres` or the database owner).
```bash
psql -U postgres -h localhost
```
Once inside `psql`, connect to your Zabbix database (usually named `zabbix`):
```sql
\c zabbix
```
> [!IMPORTANT]
> **Custom Schemas:** By default, Zabbix installs into the `public` schema. If you installed Zabbix into a custom schema (e.g., `zabbix_schema`), you **must** set your `search_path` now before running the scripts, otherwise they will fail to find your tables:
> ```sql
> SET search_path TO zabbix_schema, public;
> ```
---
## Step 3: Execute Installation Scripts
Run the scripts in the following exact order. You can use the `\i` command in `psql` if you are in the `procedures` directory, or specify the full path.
**1. Create the partitioning schema and config tables:**
> [!NOTE]
> **Zabbix 8.0+ Users:** Zabbix 8.0 introduced a new `history_json` table. Before running the script below, open `00_schema_create.sql` in a text editor and uncomment the lines specifically marked for Zabbix 8.0 at the end of the history tables block.
```sql
\i 00_schema_create.sql
```
**2. Install the maintenance logic and functions:**
```sql
\i 01_maintenance.sql
```
**3. Enable Partitioning (MIGRATION STEP):**
*This step renames your existing large tables to `_old` and instantly creates new partitioned tables. This might take a few moments.*
```sql
\i 02_enable_partitioning.sql
```
**4. Create the Monitoring View:**
```sql
\i 03_monitoring_view.sql
```
---
## Step 4: Schedule Automated Maintenance
Partitioning requires a daily job to create new partitions for tomorrow and drop old partitions from last month.
If you are using **AWS RDS** or a managed database with `pg_cron` enabled, run this inside `psql`:
```sql
CREATE EXTENSION IF NOT EXISTS pg_cron;
SELECT cron.schedule('zabbix_partition_maintenance', '30 5,23 * * *', 'CALL partitions.run_maintenance();');
```
*(If you are self-hosting and don't have `pg_cron`, please refer to the `README.md` for instructions on setting up standard OS `cron` or systemd timers.)*
---
## Step 5: Start Zabbix Server
Now that the database is fully partitioned, you can safely start Zabbix Server again:
```bash
sudo systemctl start zabbix-server
```
*(Note: Your old history data remains in tables like `history_old`. It is no longer visible in the UI. If you need it, you must manually insert it into the new tables. See `README.md` for more details.)*
---
## Step 6: Configure Zabbix Agent Monitoring
To ensure your partitions don't run out, you must monitor them. We use Zabbix Agent 2 for this.
1. On your database server (where Zabbix Agent 2 is installed), create the SQL query file using this simple one-liner. Copy and paste the entire block below into your terminal:
```bash
cat << 'EOF' | sudo tee /etc/zabbix/zabbix_agent2.d/partitions.get_all.sql > /dev/null
SELECT
table_name,
period,
keep_history::text AS keep_history,
configured_future_partitions,
actual_future_partitions,
total_size_bytes,
EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds
FROM partitions.monitoring;
EOF
```
2. Configure the PostgreSQL Plugin by editing `/etc/zabbix/zabbix_agent2.d/plugins.d/postgresql.conf`. Ensure you have defined a session (e.g., `MY_DB`) and enabled custom queries:
```ini
Plugins.PostgreSQL.CustomQueriesPath=/etc/zabbix/zabbix_agent2.d/
Plugins.PostgreSQL.CustomQueriesEnabled=true
# Example Session (replace with your actual credentials)
Plugins.PostgreSQL.Sessions.MY_DB.Uri=tcp://localhost:5432
Plugins.PostgreSQL.Sessions.MY_DB.User=zbx_monitor
Plugins.PostgreSQL.Sessions.MY_DB.Password=your_password
```
3. Restart the Zabbix Agent 2:
```bash
sudo systemctl restart zabbix-agent2
```
---
## Step 7: Import Template in Zabbix
1. Log into your Zabbix Web UI.
2. Go to **Data collection** -> **Templates** and click **Import**.
3. Upload the `template/zbx_pg_partitions_monitor_agent2.yaml` file from this repository.
4. Go to your Database Host in Zabbix, and link the newly imported template: `PostgreSQL Partitioning by Zabbix Agent 2`.
5. On the Host configuration, go to the **Macros** tab.
6. You will see a macro named `{$PG.CONNSTRING.AGENT2}` with the value `<replace_me>`.
7. Change `<replace_me>` to the name of the session you configured in Step 6 (e.g., `MY_DB`).
8. Click **Update**.
**Congratulations!** Your Zabbix database is now fully partitioned, optimized, and monitored.

29
postgresql/procedures/README.md
View File

@@ -38,33 +38,10 @@ All procedures, information, statistics and configuration are stored in the `par
## Installation
The installation is performed by executing the SQL procedures in the following order:
1. Initialize schema (`00_schema_create.sql`).
2. Install maintenance procedures (`01_maintenance.sql`).
3. Enable partitioning on tables (`02_enable_partitioning.sql`).
4. Install monitoring views (`03_monitoring_view.sql`).
> [!IMPORTANT]
> **Please refer to the [MANUAL.md](MANUAL.md) for the complete, step-by-step, foolproof installation instructions.**
> The manual contains critical safety procedures, backup warnings, and copy-pasteable commands for a safe deployment.
**Command Example:**
You can deploy these scripts manually against your Zabbix database using `psql`. Navigate to the `procedures/` directory and run:
```bash
# Connect as the zabbix database user
export PGPASSWORD="your_zabbix_password"
DB_HOST="localhost" # Or your DB endpoint
DB_NAME="zabbix"
DB_USER="zbxpart_admin"
for script in 00_schema_create.sql 01_maintenance.sql 02_enable_partitioning.sql 03_monitoring_view.sql; do
echo "Applying $script..."
# -v ON_ERROR_STOP=1 forces psql to exit immediately with an error code if any statement fails
if ! psql -v ON_ERROR_STOP=1 -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script"; then
echo -e "\nERROR: Failed to apply $script."
read -p "Press [Enter] to forcefully continue anyway, or Ctrl+C to abort... "
else
echo -e "Successfully applied $script.\n----------------------------------------"
fi
done
```
## Configuration

2
postgresql/template/zbx_pg_partitions_monitor_agent2.yaml
View File

@@ -131,7 +131,7 @@ zabbix_export:
value: '2'
description: 'The minimum number of partitions that must exist in the future'
- macro: '{$PG.CONNSTRING.AGENT2}'
value: AWS_RDS
value: '<replace_me>'
description: 'Session name or URI of the PostgreSQL instance'
- macro: '{$PG.DBNAME}'
value: zabbix

1
postgresql/tests/docker/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
init_scripts/