diff --git a/.gitignore b/.gitignore index e69de29..255ce87 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,2 @@ +db_credentials +global-bundle.pem \ No newline at end of file diff --git a/QUICKSTART.md b/QUICKSTART.md index 4889bd4..9ad8948 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -25,28 +25,66 @@ docker compose down -v ``` ## Partitioning -See [PARTITIONING.md](../PARTITIONING.md) for details on the implemented declarative partitioning. +See [ARCHITECTURE.md](../ARCHITECTURE.md) for details on the implemented declarative partitioning. -## 🐳 Docker Deployment (Production) -The `run_test_env.sh` script automatically populates `init_scripts` for the test environment. To deploy this in your own Docker setup: +## AWS RDS / External Database Testing -1. **Mount Scripts**: Map the SQL procedures to `/docker-entrypoint-initdb.d/` in your PostgreSQL container. -2. **Order Matters**: Scripts execute alphabetically. Ensure they run **after** the Zabbix schema import. +You can run these partitioning tests against a real AWS RDS (or any external PostgreSQL instance). -**Example `docker-compose.yml` snippet:** -```yaml -services: - postgres-server: - image: postgres:16 - volumes: - # Mount Zabbix Schema first (e.g., as 01_schema.sql) - - ./zabbix_schema.sql:/docker-entrypoint-initdb.d/01_schema.sql - - # Mount Partitioning Procedures (Prefix to run AFTER schema) - - ../postgresql/procedures/00_partitions_init.sql:/docker-entrypoint-initdb.d/02_00_part_init.sql - - ../postgresql/procedures/01_auditlog_prep.sql:/docker-entrypoint-initdb.d/02_01_audit_prep.sql - - ../postgresql/procedures/02_maintenance.sql:/docker-entrypoint-initdb.d/02_02_maintenance.sql - - ../postgresql/procedures/03_enable_partitioning.sql:/docker-entrypoint-initdb.d/02_03_enable.sql - - ../postgresql/procedures/04_monitoring_view.sql:/docker-entrypoint-initdb.d/02_04_monitor.sql -``` -The container will automatically execute these scripts on first startup, partitioning the tables. +### 1. Configure Credentials +First, create a `db_credentials` file in the `postgresql/` directory. (This file is ignored by Git to keep your passwords safe). +Example `postgresql/db_credentials`: +```bash +# Admin credentials +export DB_HOST="your-rds-endpoint.rds.amazonaws.com" +export DB_PORT="5432" +export DB_NAME="postgres" +export DB_USER="postgres" +export DB_PASSWORD="your_admin_password" + +# SSL Configuration +export DB_SSL_MODE="verify-full" +export DB_PEM_URL="https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem" +export DB_SSL_ROOT_CERT="./global-bundle.pem" + +# Zabbix credentials to be created +export ZBX_DB_NAME="zabbix" +export ZBX_DB_USER="zabbix" +export ZBX_DB_PASSWORD="zabbix_password" +``` + +### 2. Automated Testing +You can run the same automated deployment script, but instruct it to deploy directly to your RDS instance instead of a local Docker container: + +```bash +cd postgresql/docker +./run_test_env.sh --pg 16 --zabbix 7.0 --rds +``` + +If you want to completely clean up the RDS database and start fresh (terminating existing connections and dropping all data), use the `--rds-drop` flag. You will be prompted to type `yes` to safely confirm the deletion: +```bash +./run_test_env.sh --pg 16 --zabbix 7.0 --rds-drop +``` + +### 3. Manual Setup & Zabbix Integration +If you want to prepare the real database for your Production Zabbix Server manually, you can just run the initialization script directly: + +```bash +cd postgresql +./setup_rds.sh +# To drop an existing database and start fresh, use: +# ./setup_rds.sh --drop +``` + +The script will automatically connect as the `postgres` user, conditionally download the SSL certificates if needed, and set up the `zabbix` user and database. +Upon success, the script will output the exact block you need to copy into your `zabbix_server.conf`, e.g.: + +```ini +DBHost=your-rds-endpoint.rds.amazonaws.com +DBName=zabbix +DBUser=zabbix +DBPassword=zabbix_password +DBPort=5432 +DBTLSConnect=verify_full +DBTLSCAFile=/full/path/to/global-bundle.pem +``` \ No newline at end of file diff --git a/postgresql/docker/run_test_env.sh b/postgresql/docker/run_test_env.sh index ee6f041..5744b14 100755 --- a/postgresql/docker/run_test_env.sh +++ b/postgresql/docker/run_test_env.sh @@ -10,16 +10,20 @@ RED='\033[0;31m' NC='\033[0m' # No Color usage() { - echo "Usage: $0 --pg <16|17|18> --zabbix <7.0|7.4>" - echo "Example: $0 --pg 16 --zabbix 7.0" + echo "Usage: $0 --pg <16|17|18> --zabbix <7.0|7.4> [--rds] [--rds-drop]" + echo "Example: $0 --pg 16 --zabbix 7.0 [--rds-drop]" exit 1 } # Parse arguments +USE_RDS=false +DROP_RDS=false while [[ "$#" -gt 0 ]]; do case $1 in --pg) PG_VERSION="$2"; shift ;; --zabbix) ZABBIX_VERSION="$2"; shift ;; + --rds) USE_RDS=true ;; + --rds-drop) USE_RDS=true; DROP_RDS=true ;; *) echo "Unknown parameter: $1"; usage ;; esac shift @@ -113,23 +117,51 @@ fi # Export variable for Docker Compose export PG_VERSION=$PG_VERSION -# Run Docker Compose -echo -e "${GREEN}Starting PostgreSQL container...${NC}" -docker compose up -d - -echo -e "${GREEN}Waiting for database to be ready...${NC}" -# Simple wait loop -for i in {1..30}; do - if docker exec zabbix-db-test pg_isready -U zabbix > /dev/null 2>&1; then - echo -e "${GREEN}Database is ready!${NC}" - break +if [ "$USE_RDS" = "true" ]; then + echo -e "${GREEN}Deploying directly to RDS environment...${NC}" + if [ ! -f "../db_credentials" ]; then + echo -e "${RED}Error: ../db_credentials file not found. Please create it first.${NC}" + exit 1 fi - echo -n "." - sleep 1 -done + + # Initialize RDS (create/drop user and db) + if [ "$DROP_RDS" = "true" ]; then + echo "Initializing Zabbix RDS user and database (with DROP requested)..." + bash ../setup_rds.sh --drop + else + echo "Initializing Zabbix RDS user and database..." + bash ../setup_rds.sh + fi + + source ../db_credentials + export PGPASSWORD="$ZBX_DB_PASSWORD" + + echo "Applying scripts from init_scripts/ to RDS..." + for sql_file in $(ls ./init_scripts/*.sql | sort); do + echo "Executing $sql_file..." + psql "host=$DB_HOST port=$DB_PORT dbname=$ZBX_DB_NAME user=$ZBX_DB_USER sslmode=$DB_SSL_MODE sslrootcert=../$DB_SSL_ROOT_CERT" -f "$sql_file" -v ON_ERROR_STOP=1 + done + + echo -e "${GREEN}RDS Environment ready.${NC}" + echo "Connect: psql \"host=$DB_HOST port=$DB_PORT dbname=$ZBX_DB_NAME user=$ZBX_DB_USER sslmode=$DB_SSL_MODE sslrootcert=../$DB_SSL_ROOT_CERT\"" +else + # Run Docker Compose + echo -e "${GREEN}Starting PostgreSQL container...${NC}" + docker compose up -d -# Check if data generation finished (it runs as part of init, which might take a bit longer than just port open) -# We can check logs -echo "To follow initialization logs, run: docker logs -f zabbix-db-test" -echo -e "${GREEN}Environment ready.${NC}" -echo "Connect: psql -h localhost -p 5432 -U zabbix -d zabbix" + echo -e "${GREEN}Waiting for database to be ready...${NC}" + # Simple wait loop + for i in {1..30}; do + if docker exec zabbix-db-test pg_isready -U zabbix > /dev/null 2>&1; then + echo -e "${GREEN}Database is ready!${NC}" + break + fi + echo -n "." + sleep 1 + done + + # Check if data generation finished + echo "To follow initialization logs, run: docker logs -f zabbix-db-test" + echo -e "${GREEN}Environment ready.${NC}" + echo "Connect: psql -h localhost -p 5432 -U zabbix -d zabbix" +fi diff --git a/postgresql/procedures/README.md b/postgresql/procedures/README.md index 0a554ef..2784e2d 100644 --- a/postgresql/procedures/README.md +++ b/postgresql/procedures/README.md @@ -28,6 +28,22 @@ The installation is performed by executing the SQL procedures in the following o 4. Enable partitioning on tables (`03_enable_partitioning.sql`). 5. Install monitoring views (`04_monitoring_view.sql`). +**Command Example:** +You can deploy these scripts manually against your Zabbix database using `psql`. Navigate to the `procedures/` directory and run: + +```bash +# Connect as the zabbix database user +export PGPASSWORD="your_zabbix_password" +DB_HOST="localhost" # Or your RDS endpoint +DB_NAME="zabbix" +DB_USER="zabbix" + +for script in 00_partitions_init.sql 01_auditlog_prep.sql 02_maintenance.sql 03_enable_partitioning.sql 04_monitoring_view.sql; do + echo "Applying $script..." + psql -h $DB_HOST -U $DB_USER -d $DB_NAME -f "$script" +done +``` + ## Configuration Partitioning policies are defined in the `partitions.config` table. @@ -61,7 +77,7 @@ This procedure should be scheduled to run periodically (e.g., daily via `pg_cron ```sql CALL partitions.run_maintenance(); ``` -### Automatic Maintenance (Cron) +### Scheduling Maintenance To ensure partitions are created in advance and old data is cleaned up, the maintenance procedure should be scheduled to run automatically. @@ -69,6 +85,73 @@ It is recommended to run the maintenance **twice a day** (e.g., at 05:30 and 23: * **Primary Run**: Creates new future partitions and drops old ones. * **Secondary Run**: Acts as a safety check. Since the procedure is idempotent (safe to run multiple times), a second run ensures everything is consistent if the first run failed or was interrupted. +You can schedule this using one of the following methods: + +#### Option 1: `pg_cron` (Recommended) +`pg_cron` is a cron-based job scheduler that runs directly inside the database as an extension. + +**Setup `pg_cron`:** +1. Install the package via your OS package manager (e.g., `postgresql-15-cron` on Debian/Ubuntu, or `pg_cron_15` on RHEL/CentOS). +2. Configure it modifying `postgresql.conf`: + ```ini + shared_preload_libraries = 'pg_cron' + cron.database_name = 'zabbix' # Define the database where pg_cron will run + ``` +3. Restart PostgreSQL: + ```bash + systemctl restart postgresql + ``` +4. Connect to your `zabbix` database as a superuser and create the extension: + ```sql + CREATE EXTENSION pg_cron; + ``` +5. Schedule the job to run: + ```sql + SELECT cron.schedule('zabbix_partition_maintenance', '30 5,23 * * *', 'CALL partitions.run_maintenance();'); + ``` +6. **Manage your `pg_cron` jobs** (run as superuser): + - To **list all active schedules**: `SELECT * FROM cron.job;` + - To **view execution logs/history**: `SELECT * FROM cron.job_run_details;` + - To **remove/unschedule** the job: `SELECT cron.unschedule('zabbix_partition_maintenance');` + +#### Option 2: Systemd Timers +Systemd timers provide better logging and error handling properties than standard cron. + +1. Create a service file **`/etc/systemd/system/zabbix-partitions.service`**: + ```ini + [Unit] + Description=Zabbix PostgreSQL Partition Maintenance + After=network.target postgresql.service + + [Service] + Type=oneshot + User=postgres + ExecStart=/usr/bin/psql -d zabbix -c "CALL partitions.run_maintenance();" + ``` + +2. Create a timer file **`/etc/systemd/system/zabbix-partitions.timer`**: + ```ini + [Unit] + Description=Run Zabbix Partition Maintenance Twice Daily + + [Timer] + OnCalendar=*-*-* 05:30:00 + OnCalendar=*-*-* 23:30:00 + Persistent=true + + [Install] + WantedBy=timers.target + ``` + +3. Enable and start the timer: + ```bash + systemctl daemon-reload + systemctl enable --now zabbix-partitions.timer + ``` + +#### Option 3: System Cron (`crontab`) +Standard system cron is a simple fallback. + **Example Crontab Entry (`crontab -e`):** ```bash # Run Zabbix partition maintenance twice daily (5:30 AM and 5:30 PM) @@ -76,7 +159,7 @@ It is recommended to run the maintenance **twice a day** (e.g., at 05:30 and 23: ``` **Docker Environment:** -If running in Docker, you can execute it via the container: +If running in Docker, you can execute it via the host's cron by targeting the container: ```bash 30 5,23 * * * docker exec zabbix-db-test psql -U zabbix -d zabbix -c "CALL partitions.run_maintenance();" ``` diff --git a/postgresql/setup_rds.sh b/postgresql/setup_rds.sh new file mode 100755 index 0000000..db57817 --- /dev/null +++ b/postgresql/setup_rds.sh @@ -0,0 +1,101 @@ +#!/bin/bash +set -e + +# Change directory to script's location +cd "$(dirname "$0")" + +DROP_DB=false +while [[ "$#" -gt 0 ]]; do + case $1 in + --drop) DROP_DB=true ;; + esac + shift +done + +# Source credentials from db_credentials file +if [ -f "./db_credentials" ]; then + echo "Loading credentials from db_credentials..." + source ./db_credentials +else + echo "Error: db_credentials file not found in $(pwd)" + exit 1 +fi + +# 1. Provide the PEM key for AWS RDS if not exists +if [ -n "$DB_PEM_URL" ] && [ ! -f "$DB_SSL_ROOT_CERT" ]; then + echo "Downloading SSL root certificate from AWS..." + wget -qO "$DB_SSL_ROOT_CERT" "$DB_PEM_URL" +fi + +# Ensure PEM has right permissions if it exists +if [ -f "$DB_SSL_ROOT_CERT" ]; then + chmod 600 "$DB_SSL_ROOT_CERT" +fi + +# 2. Login as the RDS admin user (postgres) to create the zabbix user/database +echo "Connecting to PostgreSQL to create Zabbix user and database..." + +export PGPASSWORD="$DB_PASSWORD" + +# Create the zabbix user if it doesn't already exist +psql "host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USER sslmode=$DB_SSL_MODE sslrootcert=$DB_SSL_ROOT_CERT" -v ON_ERROR_STOP=1 < pg_backend_pid();" + psql "host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USER sslmode=$DB_SSL_MODE sslrootcert=$DB_SSL_ROOT_CERT" -c "DROP DATABASE $ZBX_DB_NAME;" + DB_EXISTS="" +fi + +if [ "$DB_EXISTS" != "1" ]; then + echo "Database '$ZBX_DB_NAME' does not exist. Creating..." + psql "host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USER sslmode=$DB_SSL_MODE sslrootcert=$DB_SSL_ROOT_CERT" -c "CREATE DATABASE $ZBX_DB_NAME OWNER $ZBX_DB_USER;" +else + echo "Database '$ZBX_DB_NAME' already exists." +fi + +# Grant necessary permissions +psql "host=$DB_HOST port=$DB_PORT dbname=$DB_NAME user=$DB_USER sslmode=$DB_SSL_MODE sslrootcert=$DB_SSL_ROOT_CERT" -c "GRANT ALL PRIVILEGES ON DATABASE $ZBX_DB_NAME TO $ZBX_DB_USER;" + +echo "" +echo "================================================================================" +echo "✅ Initialization Successful!" +echo "================================================================================" +echo "You can now use these settings in your Zabbix server configuration:" +echo "--------------------------------------------------------------------------------" +echo "DBHost=$DB_HOST" +echo "DBName=$ZBX_DB_NAME" +echo "DBUser=$ZBX_DB_USER" +echo "DBPassword=$ZBX_DB_PASSWORD" +echo "DBPort=$DB_PORT" +echo "DBTLSConnect=verify_full" +echo "DBTLSCAFile=$(realpath $DB_SSL_ROOT_CERT)" +echo "================================================================================" +echo "" +echo "To connect manually for testing directly to the Zabbix DB:" +echo "export PGPASSWORD=\"$ZBX_DB_PASSWORD\"" +echo "psql \"host=$DB_HOST port=$DB_PORT dbname=$ZBX_DB_NAME user=$ZBX_DB_USER sslmode=$DB_SSL_MODE sslrootcert=$DB_SSL_ROOT_CERT\"" +echo ""