diff --git a/postgresql/procedures/00_schema_create.sql b/postgresql/procedures/00_schema_create.sql index d75a9c4..46761b1 100644 --- a/postgresql/procedures/00_schema_create.sql +++ b/postgresql/procedures/00_schema_create.sql @@ -32,9 +32,16 @@ INSERT INTO partitions.config (table_name, period, keep_history) VALUES ('history_uint', 'day', '30 days'), ('history_str', 'day', '30 days'), ('history_log', 'day', '30 days'), -('history_text', 'day', '30 days') +('history_text', 'day', '30 days'), +('history_bin', 'day', '30 days') ON CONFLICT (table_name) DO NOTHING; +-- Zabbix 8.0+ only: Uncomment the following lines if running Zabbix 8.0 or later +-- INSERT INTO partitions.config (table_name, period, keep_history) VALUES +-- ('history_json', 'day', '30 days') +-- ON CONFLICT (table_name) DO NOTHING; + + -- Trends tables: Monthly partitions, keep 12 months INSERT INTO partitions.config (table_name, period, keep_history) VALUES ('trends', 'month', '12 months'), diff --git a/postgresql/procedures/01_maintenance.sql b/postgresql/procedures/01_maintenance.sql index 56b4a17..af60ce3 100644 --- a/postgresql/procedures/01_maintenance.sql +++ b/postgresql/procedures/01_maintenance.sql @@ -2,14 +2,15 @@ -- Core functions for Zabbix partitioning (Create, Drop, Maintain). -- ============================================================================ --- Function to check if a partition exists -CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text) +-- Function to check if a partition exists in a specific schema +CREATE OR REPLACE FUNCTION partitions.partition_exists(p_partition_name text, p_schema text) RETURNS boolean AS $$ BEGIN RETURN EXISTS ( SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = p_partition_name + AND n.nspname = p_schema ); END; $$ LANGUAGE plpgsql; @@ -32,7 +33,7 @@ BEGIN SELECT n.nspname INTO v_parent_schema FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname = p_parent_table; + WHERE c.relname = p_parent_table AND pg_table_is_visible(c.oid); IF NOT FOUND THEN RAISE EXCEPTION 'Parent table % not found', p_parent_table; @@ -51,15 +52,19 @@ BEGIN v_partition_name := p_parent_table || '_p' || v_suffix; - IF NOT partitions.partition_exists(v_partition_name) THEN + IF NOT partitions.partition_exists(v_partition_name, v_parent_schema) THEN BEGIN EXECUTE format( 'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)', v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts ); - EXCEPTION WHEN invalid_object_definition THEN - -- Ignore overlap errors (e.g., when transitioning from daily to hourly partitioning) - RAISE NOTICE 'Partition % overlaps with an existing partition. Skipping.', v_partition_name; + EXCEPTION + WHEN invalid_object_definition THEN + -- Ignore overlap errors (e.g., when transitioning from daily to hourly partitioning) + RAISE NOTICE 'Partition % overlaps with an existing partition. Skipping.', v_partition_name; + WHEN duplicate_table THEN + -- Ignore race condition: another process created the partition concurrently + RAISE NOTICE 'Partition % already exists (concurrent creation). Skipping.', v_partition_name; END; END IF; END; @@ -89,7 +94,7 @@ BEGIN JOIN pg_class parent ON pg_inherits.inhparent = parent.oid JOIN pg_class child ON pg_inherits.inhrelid = child.oid JOIN pg_namespace n ON child.relnamespace = n.oid - WHERE parent.relname = p_parent_table + WHERE parent.relname = p_parent_table AND pg_table_is_visible(parent.oid) LOOP -- Parse partition suffix to determine age -- Format: parent_pYYYYMM or parent_pYYYYMMDD diff --git a/postgresql/procedures/02_enable_partitioning.sql b/postgresql/procedures/02_enable_partitioning.sql index d6cb961..b256c1d 100644 --- a/postgresql/procedures/02_enable_partitioning.sql +++ b/postgresql/procedures/02_enable_partitioning.sql @@ -19,10 +19,10 @@ BEGIN SELECT n.nspname INTO v_schema FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname = v_table; + WHERE c.relname = v_table AND pg_table_is_visible(c.oid); - IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN + IF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r' AND pg_table_is_visible(oid)) THEN RAISE NOTICE 'Converting table % to partitioned table...', v_table; -- 1. Rename existing table @@ -48,19 +48,37 @@ BEGIN -- Optional: Migrate existing data -- EXECUTE format('INSERT INTO %I.%I SELECT * FROM %I.%I', v_schema, v_table, v_schema, v_old_table); - ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p') THEN + ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'p' AND pg_table_is_visible(oid)) THEN RAISE NOTICE 'Table % is already partitioned. Skipping conversion.', v_table; - -- Just run maintenance to ensure partitions exist - CALL partitions.run_maintenance(); + -- Just run maintenance for this specific table to ensure partitions exist + CALL partitions.maintain_table(v_table, v_row.period, v_row.keep_history, v_row.future_partitions); ELSE RAISE WARNING 'Table % not found!', v_table; END IF; END LOOP; + + -- Attach trigger to housekeeper table to silently discard tasks for partitioned tables. + -- Dynamically determine the schema of the housekeeper table to support custom schemas. + SELECT n.nspname INTO v_schema + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = 'housekeeper' AND pg_table_is_visible(c.oid); + + IF v_schema IS NOT NULL THEN + EXECUTE format('DROP TRIGGER IF EXISTS housekeeper_filter ON %I.housekeeper', v_schema); + EXECUTE format('CREATE TRIGGER housekeeper_filter BEFORE INSERT ON %I.housekeeper FOR EACH ROW EXECUTE FUNCTION partitions.housekeeper_insert_trigger()', v_schema); + RAISE NOTICE 'Housekeeper intercept trigger installed on %.housekeeper', v_schema; + ELSE + RAISE WARNING 'housekeeper table not found — trigger NOT installed!'; + END IF; END $$; --- Attach trigger to housekeeper table to silently discard tasks for partitioned tables -DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper; -CREATE TRIGGER housekeeper_filter -BEFORE INSERT ON housekeeper -FOR EACH ROW -EXECUTE FUNCTION partitions.housekeeper_insert_trigger(); +-- ========================================================================== +-- IMPORTANT: If the Zabbix Server connects with a non-superuser (e.g., 'zabbix'), +-- that user MUST have access to the partitions schema for the housekeeper trigger +-- to work. Without these GRANTs, every INSERT into housekeeper will FAIL. +-- Uncomment and adjust the username below: +-- ========================================================================== +-- GRANT USAGE ON SCHEMA partitions TO zabbix; +-- GRANT SELECT ON partitions.config TO zabbix; + diff --git a/postgresql/procedures/03_monitoring_view.sql b/postgresql/procedures/03_monitoring_view.sql index eb77ade..705bd95 100644 --- a/postgresql/procedures/03_monitoring_view.sql +++ b/postgresql/procedures/03_monitoring_view.sql @@ -2,8 +2,7 @@ -- Creates a view to monitor partition status and sizes. -- ============================================================================ -DROP VIEW IF EXISTS partitions.monitoring; -CREATE VIEW partitions.monitoring AS +CREATE OR REPLACE VIEW partitions.monitoring AS SELECT parent.relname AS parent_table, c.table_name, @@ -27,7 +26,7 @@ SELECT max(child.relname) AS newest_partition, c.last_updated FROM partitions.config c -JOIN pg_class parent ON parent.relname = c.table_name +JOIN pg_class parent ON parent.relname = c.table_name AND pg_table_is_visible(parent.oid) LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid WHERE parent.relkind = 'p' -- Only partitioned tables diff --git a/postgresql/procedures/04_undo_partitioning.sql b/postgresql/procedures/04_undo_partitioning.sql index 449033c..7a25fbf 100644 --- a/postgresql/procedures/04_undo_partitioning.sql +++ b/postgresql/procedures/04_undo_partitioning.sql @@ -18,7 +18,7 @@ BEGIN SELECT n.nspname INTO v_schema FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname = v_table AND c.relkind = 'p'; + WHERE c.relname = v_table AND c.relkind = 'p' AND pg_table_is_visible(c.oid); IF v_schema IS NOT NULL THEN RAISE NOTICE 'Reverting partitioned table %...', v_table; @@ -47,15 +47,25 @@ BEGIN RAISE NOTICE 'SUCCESS: % reverted to default. Partitioned data stored in % (You can DROP TABLE % CASCADE; later).', v_table, v_part_table, v_part_table; - ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r') THEN + ELSIF EXISTS (SELECT 1 FROM pg_class WHERE relname = v_table AND relkind = 'r' AND pg_table_is_visible(oid)) THEN RAISE NOTICE 'Table % is already a regular table. Skipping.', v_table; ELSE RAISE WARNING 'Partitioned table % not found!', v_table; END IF; END LOOP; - -- Drop the housekeeper intercept trigger - DROP TRIGGER IF EXISTS housekeeper_filter ON housekeeper; + -- Drop the housekeeper intercept trigger (dynamically determine schema for custom schema support) + SELECT n.nspname INTO v_schema + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = 'housekeeper' AND pg_table_is_visible(c.oid); + + IF v_schema IS NOT NULL THEN + EXECUTE format('DROP TRIGGER IF EXISTS housekeeper_filter ON %I.housekeeper', v_schema); + RAISE NOTICE 'Housekeeper intercept trigger removed from %.housekeeper', v_schema; + ELSE + RAISE WARNING 'housekeeper table not found — trigger removal skipped.'; + END IF; RAISE NOTICE '================================================================================'; RAISE NOTICE 'Undo complete. Partitioned tables have been renamed to *_part.'; diff --git a/postgresql/procedures/MANUAL.md b/postgresql/procedures/MANUAL.md new file mode 100644 index 0000000..e6e29be --- /dev/null +++ b/postgresql/procedures/MANUAL.md @@ -0,0 +1,161 @@ +# Zabbix Partitioning Deployment Manual + +This guide provides a step-by-step process for deploying the PostgreSQL partitioning solution for Zabbix. + +**🚨 DANGER: CRITICAL WARNING 🚨** +**BEFORE YOU PROCEED, YOU ABSOLUTELY MUST TAKE A FULL BACKUP OF YOUR ZABBIX DATABASE.** +**DO NOT SKIP THIS STEP. Schema modifications are dangerous. If something goes wrong and you do not have a backup, your historical data will be lost permanently, and we take ZERO responsibility.** + +--- + +## Step 1: Preparation & Safety + +Because database migrations can take time (especially on large tables), **never** run these scripts directly in a standard SSH session that might disconnect. + +1. Open a safe terminal session using `tmux` or `screen`: + ```bash + tmux new -s zabbix_partitioning + # OR + screen -S zabbix_partitioning + ``` +2. Disable the Zabbix Housekeeper for History and Trends: + - Go to your Zabbix Web UI -> **Administration** -> **Housekeeping**. + - **Uncheck** "Enable internal housekeeping" for **History and Trends**. + - Click **Update**. +3. Stop your Zabbix Server to ensure no new data is being written during the schema migration: + ```bash + sudo systemctl stop zabbix-server + ``` + +--- + +## Step 2: Database Connection & Schema Selection + +Connect to your PostgreSQL server as an administrator (e.g., `postgres` or the database owner). + +```bash +psql -U postgres -h localhost +``` + +Once inside `psql`, connect to your Zabbix database (usually named `zabbix`): + +```sql +\c zabbix +``` + +> [!IMPORTANT] +> **Custom Schemas:** By default, Zabbix installs into the `public` schema. If you installed Zabbix into a custom schema (e.g., `zabbix_schema`), you **must** set your `search_path` now before running the scripts, otherwise they will fail to find your tables: +> ```sql +> SET search_path TO zabbix_schema, public; +> ``` + +--- + +## Step 3: Execute Installation Scripts + +Run the scripts in the following exact order. You can use the `\i` command in `psql` if you are in the `procedures` directory, or specify the full path. + +**1. Create the partitioning schema and config tables:** +> [!NOTE] +> **Zabbix 8.0+ Users:** Zabbix 8.0 introduced a new `history_json` table. Before running the script below, open `00_schema_create.sql` in a text editor and uncomment the lines specifically marked for Zabbix 8.0 at the end of the history tables block. + +```sql +\i 00_schema_create.sql +``` + +**2. Install the maintenance logic and functions:** +```sql +\i 01_maintenance.sql +``` + +**3. Enable Partitioning (MIGRATION STEP):** +*This step renames your existing large tables to `_old` and instantly creates new partitioned tables. This might take a few moments.* +```sql +\i 02_enable_partitioning.sql +``` + +**4. Create the Monitoring View:** +```sql +\i 03_monitoring_view.sql +``` + +--- + +## Step 4: Schedule Automated Maintenance + +Partitioning requires a daily job to create new partitions for tomorrow and drop old partitions from last month. + +If you are using **AWS RDS** or a managed database with `pg_cron` enabled, run this inside `psql`: + +```sql +CREATE EXTENSION IF NOT EXISTS pg_cron; +SELECT cron.schedule('zabbix_partition_maintenance', '30 5,23 * * *', 'CALL partitions.run_maintenance();'); +``` + +*(If you are self-hosting and don't have `pg_cron`, please refer to the `README.md` for instructions on setting up standard OS `cron` or systemd timers.)* + +--- + +## Step 5: Start Zabbix Server + +Now that the database is fully partitioned, you can safely start Zabbix Server again: + +```bash +sudo systemctl start zabbix-server +``` + +*(Note: Your old history data remains in tables like `history_old`. It is no longer visible in the UI. If you need it, you must manually insert it into the new tables. See `README.md` for more details.)* + +--- + +## Step 6: Configure Zabbix Agent Monitoring + +To ensure your partitions don't run out, you must monitor them. We use Zabbix Agent 2 for this. + +1. On your database server (where Zabbix Agent 2 is installed), create the SQL query file using this simple one-liner. Copy and paste the entire block below into your terminal: + +```bash +cat << 'EOF' | sudo tee /etc/zabbix/zabbix_agent2.d/partitions.get_all.sql > /dev/null +SELECT + table_name, + period, + keep_history::text AS keep_history, + configured_future_partitions, + actual_future_partitions, + total_size_bytes, + EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds +FROM partitions.monitoring; +EOF +``` + +2. Configure the PostgreSQL Plugin by editing `/etc/zabbix/zabbix_agent2.d/plugins.d/postgresql.conf`. Ensure you have defined a session (e.g., `MY_DB`) and enabled custom queries: + +```ini +Plugins.PostgreSQL.CustomQueriesPath=/etc/zabbix/zabbix_agent2.d/ +Plugins.PostgreSQL.CustomQueriesEnabled=true + +# Example Session (replace with your actual credentials) +Plugins.PostgreSQL.Sessions.MY_DB.Uri=tcp://localhost:5432 +Plugins.PostgreSQL.Sessions.MY_DB.User=zbx_monitor +Plugins.PostgreSQL.Sessions.MY_DB.Password=your_password +``` + +3. Restart the Zabbix Agent 2: +```bash +sudo systemctl restart zabbix-agent2 +``` + +--- + +## Step 7: Import Template in Zabbix + +1. Log into your Zabbix Web UI. +2. Go to **Data collection** -> **Templates** and click **Import**. +3. Upload the `template/zbx_pg_partitions_monitor_agent2.yaml` file from this repository. +4. Go to your Database Host in Zabbix, and link the newly imported template: `PostgreSQL Partitioning by Zabbix Agent 2`. +5. On the Host configuration, go to the **Macros** tab. +6. You will see a macro named `{$PG.CONNSTRING.AGENT2}` with the value ``. +7. Change `` to the name of the session you configured in Step 6 (e.g., `MY_DB`). +8. Click **Update**. + +**Congratulations!** Your Zabbix database is now fully partitioned, optimized, and monitored. diff --git a/postgresql/procedures/README.md b/postgresql/procedures/README.md index b8fc281..44a5abc 100644 --- a/postgresql/procedures/README.md +++ b/postgresql/procedures/README.md @@ -38,33 +38,10 @@ All procedures, information, statistics and configuration are stored in the `par ## Installation -The installation is performed by executing the SQL procedures in the following order: -1. Initialize schema (`00_schema_create.sql`). -2. Install maintenance procedures (`01_maintenance.sql`). -3. Enable partitioning on tables (`02_enable_partitioning.sql`). -4. Install monitoring views (`03_monitoring_view.sql`). +> [!IMPORTANT] +> **Please refer to the [MANUAL.md](MANUAL.md) for the complete, step-by-step, foolproof installation instructions.** +> The manual contains critical safety procedures, backup warnings, and copy-pasteable commands for a safe deployment. -**Command Example:** -You can deploy these scripts manually against your Zabbix database using `psql`. Navigate to the `procedures/` directory and run: - -```bash -# Connect as the zabbix database user -export PGPASSWORD="your_zabbix_password" -DB_HOST="localhost" # Or your DB endpoint -DB_NAME="zabbix" -DB_USER="zbxpart_admin" - -for script in 00_schema_create.sql 01_maintenance.sql 02_enable_partitioning.sql 03_monitoring_view.sql; do - echo "Applying $script..." - # -v ON_ERROR_STOP=1 forces psql to exit immediately with an error code if any statement fails - if ! psql -v ON_ERROR_STOP=1 -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script"; then - echo -e "\nERROR: Failed to apply $script." - read -p "Press [Enter] to forcefully continue anyway, or Ctrl+C to abort... " - else - echo -e "Successfully applied $script.\n----------------------------------------" - fi -done -``` ## Configuration diff --git a/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml b/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml index bcb52e6..e3af438 100644 --- a/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml +++ b/postgresql/template/zbx_pg_partitions_monitor_agent2.yaml @@ -131,7 +131,7 @@ zabbix_export: value: '2' description: 'The minimum number of partitions that must exist in the future' - macro: '{$PG.CONNSTRING.AGENT2}' - value: AWS_RDS + value: '' description: 'Session name or URI of the PostgreSQL instance' - macro: '{$PG.DBNAME}' value: zabbix diff --git a/postgresql/tests/docker/.gitignore b/postgresql/tests/docker/.gitignore new file mode 100644 index 0000000..7f0a76a --- /dev/null +++ b/postgresql/tests/docker/.gitignore @@ -0,0 +1 @@ +init_scripts/