Compare commits

..

3 Commits

5 changed files with 51 additions and 20 deletions

View File

@@ -52,10 +52,15 @@ BEGIN
v_partition_name := p_parent_table || '_p' || v_suffix; v_partition_name := p_parent_table || '_p' || v_suffix;
IF NOT partitions.partition_exists(v_partition_name) THEN IF NOT partitions.partition_exists(v_partition_name) THEN
BEGIN
EXECUTE format( EXECUTE format(
'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)', 'CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%s) TO (%s)',
v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts v_parent_schema, v_partition_name, v_parent_schema, p_parent_table, v_start_ts, v_end_ts
); );
EXCEPTION WHEN invalid_object_definition THEN
-- Ignore overlap errors (e.g., when transitioning from daily to hourly partitioning)
RAISE NOTICE 'Partition % overlaps with an existing partition. Skipping.', v_partition_name;
END;
END IF; END IF;
END; END;
$$; $$;
@@ -92,11 +97,11 @@ BEGIN
BEGIN BEGIN
IF length(v_suffix) = 6 THEN -- YYYYMM IF length(v_suffix) = 6 THEN -- YYYYMM
v_partition_date := to_timestamp(v_suffix || '01', 'YYYYMMDD') AT TIME ZONE 'UTC'; v_partition_date := timezone('UTC', to_timestamp(v_suffix || '01', 'YYYYMMDD')::timestamp without time zone);
ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD
v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC'; v_partition_date := timezone('UTC', to_timestamp(v_suffix, 'YYYYMMDD')::timestamp without time zone);
ELSIF length(v_suffix) = 10 THEN -- YYYYMMDDHH ELSIF length(v_suffix) = 10 THEN -- YYYYMMDDHH
v_partition_date := to_timestamp(v_suffix, 'YYYYMMDDHH24') AT TIME ZONE 'UTC'; v_partition_date := timezone('UTC', to_timestamp(v_suffix, 'YYYYMMDDHH24')::timestamp without time zone);
ELSE ELSE
CONTINUE; -- Ignore non-matching suffix lengths CONTINUE; -- Ignore non-matching suffix lengths
END IF; END IF;
@@ -153,25 +158,25 @@ DECLARE
BEGIN BEGIN
IF p_period = 'day' THEN IF p_period = 'day' THEN
v_period_interval := '1 day'::interval; v_period_interval := '1 day'::interval;
v_start_time := date_trunc('day', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('day', now(), 'UTC');
-- Calculate how many past days cover the retention period (86400 seconds = 1 day) -- Calculate how many past days cover the retention period (86400 seconds = 1 day)
v_past_iterations := ceil(extract(epoch from p_keep_history) / 86400)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 86400)::integer;
ELSIF p_period = 'week' THEN ELSIF p_period = 'week' THEN
v_period_interval := '1 week'::interval; v_period_interval := '1 week'::interval;
v_start_time := date_trunc('week', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('week', now(), 'UTC');
-- 604800 seconds = 1 week -- 604800 seconds = 1 week
v_past_iterations := ceil(extract(epoch from p_keep_history) / 604800)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 604800)::integer;
ELSIF p_period = 'month' THEN ELSIF p_period = 'month' THEN
v_period_interval := '1 month'::interval; v_period_interval := '1 month'::interval;
v_start_time := date_trunc('month', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('month', now(), 'UTC');
-- Approximate 30 days per month (2592000 seconds) -- Approximate 30 days per month (2592000 seconds)
v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer;
ELSIF p_period LIKE '%hour%' THEN ELSIF p_period LIKE '%hour%' THEN
v_period_interval := p_period::interval; v_period_interval := p_period::interval;
v_start_time := date_trunc('hour', now() AT TIME ZONE 'UTC'); v_start_time := to_timestamp(floor(extract(epoch from now()) / extract(epoch from v_period_interval)) * extract(epoch from v_period_interval));
v_past_iterations := ceil(extract(epoch from p_keep_history) / extract(epoch from v_period_interval))::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / extract(epoch from v_period_interval))::integer;
ELSE ELSE

View File

@@ -9,6 +9,7 @@ SELECT
c.table_name, c.table_name,
c.period, c.period,
c.keep_history, c.keep_history,
c.future_partitions AS configured_future_partitions,
count(child.relname) AS partition_count, count(child.relname) AS partition_count,
count(child.relname) FILTER ( count(child.relname) FILTER (
WHERE WHERE
@@ -19,7 +20,7 @@ SELECT
(c.period = 'week' AND child.relname > (parent.relname || '_p' || to_char(date_trunc('week', now() AT TIME ZONE 'UTC'), 'YYYYMMDD'))) (c.period = 'week' AND child.relname > (parent.relname || '_p' || to_char(date_trunc('week', now() AT TIME ZONE 'UTC'), 'YYYYMMDD')))
OR OR
(c.period LIKE '%hour%' AND child.relname > (parent.relname || '_p' || to_char(now() AT TIME ZONE 'UTC', 'YYYYMMDDHH24'))) (c.period LIKE '%hour%' AND child.relname > (parent.relname || '_p' || to_char(now() AT TIME ZONE 'UTC', 'YYYYMMDDHH24')))
) AS future_partitions, ) AS actual_future_partitions,
sum(pg_total_relation_size(child.oid)) AS total_size_bytes, sum(pg_total_relation_size(child.oid)) AS total_size_bytes,
pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size, pg_size_pretty(sum(pg_total_relation_size(child.oid))) AS total_size,
min(child.relname) AS oldest_partition, min(child.relname) AS oldest_partition,
@@ -30,4 +31,4 @@ JOIN pg_class parent ON parent.relname = c.table_name
LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid LEFT JOIN pg_inherits ON pg_inherits.inhparent = parent.oid
LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid LEFT JOIN pg_class child ON pg_inherits.inhrelid = child.oid
WHERE parent.relkind = 'p' -- Only partitioned tables WHERE parent.relkind = 'p' -- Only partitioned tables
GROUP BY parent.relname, c.table_name, c.period, c.keep_history, c.last_updated; GROUP BY parent.relname, c.table_name, c.period, c.keep_history, c.future_partitions, c.last_updated;

View File

@@ -54,7 +54,13 @@ DB_USER="zbxpart_admin"
for script in 00_schema_create.sql 01_maintenance.sql 02_enable_partitioning.sql 03_monitoring_view.sql; do for script in 00_schema_create.sql 01_maintenance.sql 02_enable_partitioning.sql 03_monitoring_view.sql; do
echo "Applying $script..." echo "Applying $script..."
psql -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script" # -v ON_ERROR_STOP=1 forces psql to exit immediately with an error code if any statement fails
if ! psql -v ON_ERROR_STOP=1 -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script"; then
echo -e "\nERROR: Failed to apply $script."
read -p "Press [Enter] to forcefully continue anyway, or Ctrl+C to abort... "
else
echo -e "Successfully applied $script.\n----------------------------------------"
fi
done done
``` ```
@@ -213,6 +219,24 @@ System state can be monitored via the `partitions.monitoring` view. It includes
SELECT * FROM partitions.monitoring; SELECT * FROM partitions.monitoring;
``` ```
### Zabbix Agent Integration
To monitor the state of the partitions directly from Zabbix, you need to provide the Zabbix Agent with the SQL query used to fetch this data. You can automatically generate the required `partitions.get_all.sql` file on your agent using this one-liner:
```bash
cat << 'EOF' | sudo tee /etc/zabbix/zabbix_agent2.d/partitions.get_all.sql > /dev/null
SELECT
table_name,
period,
keep_history::text AS keep_history,
configured_future_partitions,
actual_future_partitions,
total_size_bytes,
EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds
FROM partitions.monitoring;
EOF
```
*(Make sure to adjust the destination path according to your Zabbix Agent template directory)*
### Versioning ### Versioning
To check the installed version of the partitioning solution: To check the installed version of the partitioning solution:
```sql ```sql

View File

@@ -2,7 +2,8 @@ SELECT
table_name, table_name,
period, period,
keep_history::text AS keep_history, keep_history::text AS keep_history,
future_partitions, configured_future_partitions,
actual_future_partitions,
total_size_bytes, total_size_bytes,
EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds EXTRACT(EPOCH FROM (now() - last_updated)) AS age_seconds
FROM partitions.monitoring; FROM partitions.monitoring;

View File

@@ -90,7 +90,7 @@ BEGIN
BEGIN BEGIN
IF length(v_suffix) = 6 THEN -- YYYYMM IF length(v_suffix) = 6 THEN -- YYYYMM
v_partition_date := to_timestamp(v_suffix || '01', 'YYYYMMDD') AT TIME ZONE 'UTC'; v_partition_date := timezone('UTC', to_timestamp(v_suffix || '01', 'YYYYMMDD')::timestamp without time zone);
-- For monthly, we check if the END of the month is older than retention? -- For monthly, we check if the END of the month is older than retention?
-- Or just strict retention. -- Or just strict retention.
-- To be safe, adding 1 month to check vs cutoff. -- To be safe, adding 1 month to check vs cutoff.
@@ -100,7 +100,7 @@ BEGIN
COMMIT; -- Release lock immediately COMMIT; -- Release lock immediately
END IF; END IF;
ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD ELSIF length(v_suffix) = 8 THEN -- YYYYMMDD
v_partition_date := to_timestamp(v_suffix, 'YYYYMMDD') AT TIME ZONE 'UTC'; v_partition_date := timezone('UTC', to_timestamp(v_suffix, 'YYYYMMDD')::timestamp without time zone);
IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN IF extract(epoch from (v_partition_date + '1 day'::interval)) < v_cutoff_ts THEN
RAISE NOTICE 'Dropping old partition %', v_partition.partition_name; RAISE NOTICE 'Dropping old partition %', v_partition.partition_name;
EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name); EXECUTE format('DROP TABLE %I.%I', v_partition.partition_schema, v_partition.partition_name);
@@ -130,19 +130,19 @@ DECLARE
BEGIN BEGIN
IF p_period = 'day' THEN IF p_period = 'day' THEN
v_period_interval := '1 day'::interval; v_period_interval := '1 day'::interval;
v_start_time := date_trunc('day', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('day', now(), 'UTC');
-- Calculate how many past days cover the retention period (86400 seconds = 1 day) -- Calculate how many past days cover the retention period (86400 seconds = 1 day)
v_past_iterations := ceil(extract(epoch from p_keep_history) / 86400)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 86400)::integer;
ELSIF p_period = 'week' THEN ELSIF p_period = 'week' THEN
v_period_interval := '1 week'::interval; v_period_interval := '1 week'::interval;
v_start_time := date_trunc('week', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('week', now(), 'UTC');
-- 604800 seconds = 1 week -- 604800 seconds = 1 week
v_past_iterations := ceil(extract(epoch from p_keep_history) / 604800)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 604800)::integer;
ELSIF p_period = 'month' THEN ELSIF p_period = 'month' THEN
v_period_interval := '1 month'::interval; v_period_interval := '1 month'::interval;
v_start_time := date_trunc('month', now() AT TIME ZONE 'UTC'); v_start_time := date_trunc('month', now(), 'UTC');
-- Approximate 30 days per month (2592000 seconds) -- Approximate 30 days per month (2592000 seconds)
v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer; v_past_iterations := ceil(extract(epoch from p_keep_history) / 2592000)::integer;
ELSE ELSE