Feature: Added setup wizard. Changed scan-skip logic.
Some checks failed
Zabbix APK Builder / check-version (push) Successful in 14s
Zabbix APK Builder / update-version (push) Has been skipped
Zabbix APK Builder / build-packages (push) Failing after 21s
Zabbix APK Builder / deploy-test (push) Has been skipped

This commit is contained in:
2025-12-16 22:11:56 +01:00
parent bcd2bd627e
commit 08ba77cdf0
4 changed files with 266 additions and 21 deletions

View File

@@ -105,6 +105,13 @@ def main():
sys.exit(1)
cmd.append('--check-days')
cmd.append(target)
elif run_mode == 'stats':
target = os.getenv('CHECK_TARGET')
if not target:
print("Error: CHECK_TARGET env var required for stats mode")
sys.exit(1)
cmd.append('--stats')
cmd.append(target)
print(f"Executing: {' '.join(cmd)}")
result = subprocess.run(cmd)

View File

@@ -26,9 +26,8 @@ partitions:
# weekly: Partitions created weekly
weekly:
# - auditlog: 180d
# Note: auditlog is not partitionable by default in Zabbix 7.0 and 7.4 (PK missing clock).
# Note: auditlog is not partitionable by default in Zabbix 7.0 (PK missing clock).
# To partition, the Primary Key must be altered to include 'clock'.
# https://www.zabbix.com/documentation/current/en/manual/appendix/install/auditlog_primary_keys
# monthly: Partitions created monthly
monthly:
- trends: 1y
@@ -40,14 +39,6 @@ logging: syslog
# premake: Number of partitions to create in advance
premake: 10
# initial_partitioning_start: Strategy for the first partition during initialization (--init).
# Options:
# db_min: (Default) Queries SELECT MIN(clock) to ensure ALL data is covered. Slow on huge tables consistently.
# retention: Starts partitioning from (Now - Retention Period).
# Creates a 'p_archive' partition for all data older than retention.
# Much faster as it skips the MIN(clock) query. (Recommended for large DBs)
initial_partitioning_start: db_min
# replicate_sql: False - Disable binary logging. Partitioning changes are NOT replicated to slaves (use for independent maintenance).
# replicate_sql: True - Enable binary logging. Partitioning changes ARE replicated to slaves (use for consistent cluster schema).
replicate_sql: False

View File

@@ -35,9 +35,10 @@ class DatabaseError(Exception):
pass
class ZabbixPartitioner:
def __init__(self, config: Dict[str, Any], dry_run: bool = False):
def __init__(self, config: Dict[str, Any], dry_run: bool = False, fast_init: bool = False):
self.config = config
self.dry_run = dry_run
self.fast_init = fast_init
self.conn = None
self.logger = logging.getLogger('zabbix_partitioning')
@@ -385,12 +386,29 @@ class ZabbixPartitioner:
self.logger.info(f"Table {table} is already partitioned.")
return
init_strategy = self.config.get('initial_partitioning_start', 'db_min')
# init_strategy = self.config.get('initial_partitioning_start', 'db_min') # Removed in favor of flag
# but flag needs to be passed to this method or accessed from somewhere.
# Since I can't easily change signature without affecting calls, I'll pass it in kwargs or check self.fast_init if I add it to class.
pass
def initialize_partitioning(self, table: str, period: str, premake: int, retention_str: str, fast_init: bool = False):
"""Initial partitioning for a table (convert regular table to partitioned)."""
self.logger.info(f"Initializing partitioning for {table}")
if self.has_incompatible_primary_key(table):
self.logger.error(f"Cannot partition {table}: Primary Key does not include 'clock' column.")
return
# If already partitioned, skip
if self.get_existing_partitions(table):
self.logger.info(f"Table {table} is already partitioned.")
return
start_dt = None
p_archive_ts = None
if init_strategy == 'retention':
self.logger.info(f"Strategy 'retention': Calculating start date from retention ({retention_str})")
if fast_init:
self.logger.info(f"Strategy 'fast-init': Calculating start date from retention ({retention_str})")
retention_date = self.get_lookback_date(retention_str)
# Start granular partitions from the retention date
start_dt = self.truncate_date(retention_date, period)
@@ -481,6 +499,51 @@ class ZabbixPartitioner:
diff = end_dt - now
return max(0, diff.days)
def get_table_stats(self, table: str) -> Dict[str, Any]:
"""
Get detailed statistics for a table:
- size_bytes (data + index)
- partition_count
- days_left (coverage)
"""
# 1. Get Size
size_query = """
SELECT (DATA_LENGTH + INDEX_LENGTH)
FROM information_schema.TABLES
WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
"""
size_bytes = self.execute_query(size_query, (self.db_name, table), fetch='one')
# 2. Get Partition Count
count_query = """
SELECT COUNT(*)
FROM information_schema.PARTITIONS
WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND PARTITION_NAME IS NOT NULL
"""
p_count = self.execute_query(count_query, (self.db_name, table), fetch='one')
# 3. Get Days Left
# We need the period first.
partitions_conf = self.config.get('partitions', {})
found_period = None
for period, tables in partitions_conf.items():
for item in tables:
if list(item.keys())[0] == table:
found_period = period
break
if found_period: break
days_left = -1
if found_period:
days_left = self.check_partitions_coverage(table, found_period)
return {
"table": table,
"size_bytes": int(size_bytes) if size_bytes is not None else 0,
"partition_count": int(p_count) if p_count is not None else 0,
"days_left": days_left
}
def run(self, mode: str, target_table: str = None):
"""Main execution loop."""
with self.connect_db():
@@ -517,6 +580,15 @@ class ZabbixPartitioner:
print(days_left)
return
# --- Stats Mode ---
if mode == 'stats':
if not target_table:
raise ConfigurationError("Target table required for stats mode")
stats = self.get_table_stats(target_table)
print(json.dumps(stats))
return
# --- Normal Mode (Init/Maintain) ---
self.check_compatibility()
premake = self.config.get('premake', 10)
@@ -530,7 +602,7 @@ class ZabbixPartitioner:
retention = item[table]
if mode == 'init':
self.initialize_partitioning(table, period, premake, retention)
self.initialize_partitioning(table, period, premake, retention, fast_init=self.fast_init)
else:
# Maintenance mode (Add new, remove old)
self.create_future_partitions(table, period, premake)
@@ -543,6 +615,106 @@ class ZabbixPartitioner:
if mode != 'init' and not self.dry_run:
pass
def run_wizard():
print("Welcome to Zabbix Partitioning Wizard")
print("-------------------------------------")
config = {
'database': {'type': 'mysql'},
'partitions': {'daily': [], 'weekly': [], 'monthly': []},
'logging': 'console',
'premake': 10,
'replicate_sql': False
}
# 1. Connection
print("\n[Database Connection]")
use_socket = input("Use Socket (s) or Address (a)? [s/a]: ").lower().strip() == 's'
if use_socket:
sock = input("Socket path [/var/run/mysqld/mysqld.sock]: ").strip() or '/var/run/mysqld/mysqld.sock'
config['database']['socket'] = sock
config['database']['host'] = 'localhost' # Fallback
config['database']['port'] = 3306
else:
host = input("Database Host [localhost]: ").strip() or 'localhost'
port_str = input("Database Port [3306]: ").strip() or '3306'
config['database']['host'] = host
config['database']['port'] = int(port_str)
config['database']['user'] = input("Database User [zabbix]: ").strip() or 'zabbix'
config['database']['passwd'] = input("Database Password: ").strip()
config['database']['db'] = input("Database Name [zabbix]: ").strip() or 'zabbix'
# 2. Auditlog
print("\n[Auditlog]")
print("Note: To partition 'auditlog', ensure its Primary Key includes the 'clock' column.")
if input("Partition 'auditlog' table? [y/N]: ").lower().strip() == 'y':
ret = input("Auditlog retention (e.g. 365d) [365d]: ").strip() or '365d'
config['partitions']['weekly'].append({'auditlog': ret})
# 3. History Tables
# History tables list
history_tables = ['history', 'history_uint', 'history_str', 'history_log', 'history_text', 'history_bin']
print("\n[History Tables]")
# Separate logic as requested
if input("Set SAME retention for all history tables? [Y/n]: ").lower().strip() != 'n':
ret = input("Retention for all history tables (e.g. 30d) [30d]: ").strip() or '30d'
for t in history_tables:
config['partitions']['daily'].append({t: ret})
else:
for t in history_tables:
ret = input(f"Retention for '{t}' (e.g. 30d, skip to ignore): ").strip()
if ret:
config['partitions']['daily'].append({t: ret})
# 4. Trends Tables
trends_tables = ['trends', 'trends_uint']
print("\n[Trends Tables]")
if input("Set SAME retention for all trends tables? [Y/n]: ").lower().strip() != 'n':
ret = input("Retention for all trends tables (e.g. 365d) [365d]: ").strip() or '365d'
for t in trends_tables:
config['partitions']['monthly'].append({t: ret})
else:
for t in trends_tables:
ret = input(f"Retention for '{t}' (e.g. 365d, skip to ignore): ").strip()
if ret:
config['partitions']['monthly'].append({t: ret})
# 5. Replication
print("\n[Replication]")
config['replicate_sql'] = input("Enable binary logging for replication? [y/N]: ").lower().strip() == 'y'
# 6. Premake
print("\n[Premake]")
pm = input("How many future partitions to create? [10]: ").strip()
config['premake'] = int(pm) if pm.isdigit() else 10
# 7. Logging
print("\n[Logging]")
config['logging'] = 'syslog' if input("Log to syslog? [y/N]: ").lower().strip() == 'y' else 'console'
# Save
print("\n[Output]")
path = input("Save config to [/etc/zabbix/zabbix_partitioning.conf]: ").strip() or '/etc/zabbix/zabbix_partitioning.conf'
try:
# Create dir if not exists
folder = os.path.dirname(path)
if folder and not os.path.exists(folder):
try:
os.makedirs(folder)
except OSError:
print(f"Warning: Could not create directory {folder}. Saving to current directory.")
path = 'zabbix_partitioning.conf'
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
print(f"\nConfiguration saved to {path}")
except Exception as e:
print(f"Error saving config: {e}")
print(yaml.dump(config)) # dump to stdout if fails
def setup_logging(config_log_type: str, verbose: bool = False):
logger = logging.getLogger('zabbix_partitioning')
logger.setLevel(logging.DEBUG if verbose else logging.INFO)
@@ -568,6 +740,12 @@ def parse_args():
# Monitoring args
parser.add_argument('--discovery', action='store_true', help='Output Zabbix LLD JSON')
parser.add_argument('--check-days', type=str, help='Check days of future partitions left for table', metavar='TABLE')
parser.add_argument('--stats', type=str, help='Output detailed table statistics in JSON', metavar='TABLE')
# Wizard & Flags
parser.add_argument('--wizard', action='store_true', help='Launch interactive configuration wizard')
parser.add_argument('--fast-init', action='store_true', help='Skip MIN(clock) check during init, start from retention')
parser.add_argument('-V', '--version', action='version', version=f'%(prog)s {VERSION}', help='Show version and exit')
return parser.parse_args()
@@ -584,6 +762,10 @@ def main():
args = parse_args()
try:
if args.wizard:
run_wizard()
return
conf_path = load_config(args.config)
with open(conf_path, 'r') as f:
config = yaml.safe_load(f)
@@ -602,10 +784,13 @@ def main():
elif args.check_days:
mode = 'check'
target = args.check_days
elif args.stats:
mode = 'stats'
target = args.stats
elif args.init: mode = 'init'
# Setup logging
if mode in ['discovery', 'check']:
if mode in ['discovery', 'check', 'stats']:
logging.basicConfig(level=logging.ERROR) # Only show critical errors
else:
setup_logging(config.get('logging', 'console'), verbose=args.verbose)
@@ -616,7 +801,7 @@ def main():
logger.info("Starting in DRY-RUN mode")
# ZabbixPartitioner expects dict config
app = ZabbixPartitioner(config, dry_run=args.dry_run)
app = ZabbixPartitioner(config, dry_run=args.dry_run, fast_init=args.fast_init)
app.run(mode, target)
except Exception as e:

View File

@@ -13,7 +13,9 @@ zabbix_export:
1. Install zabbix_partitioning.py on the Zabbix Server/Proxy.
2. Configure userparameter for automatic discovery:
UserParameter=zabbix.partitioning.discovery[*], /usr/local/bin/zabbix_partitioning.py -c $1 --discovery
UserParameter=zabbix.partitioning.check[*], /usr/local/bin/zabbix_partitioning.py -c $1 --check-days $2
UserParameter=zabbix.partitioning.discovery[*], /usr/local/bin/zabbix_partitioning.py -c $1 --discovery
UserParameter=zabbix.partitioning.stats[*], /usr/local/bin/zabbix_partitioning.py -c $1 --stats $2
# Legacy check removed in favor of stats
Or use Docker wrapper scripts.
groups:
@@ -43,11 +45,32 @@ zabbix_export:
description: 'Discover partitioned tables'
item_prototypes:
- uuid: 1fbff85191c244dca956be7a94bf08a3
name: 'Days remaining: {#TABLE}'
key: 'zabbix.partitioning.check[{$PATH.TO.CONFIG}, {#TABLE}]'
name: 'Partitioning Stats: {#TABLE}'
key: 'zabbix.partitioning.stats[{$PATH.TO.CONFIG}, {#TABLE}]'
delay: 12h
history: '0'
trends: '0'
value_type: TEXT
description: 'JSON statistics for table {#TABLE}'
tags:
- tag: component
value: partitioning
- tag: table
value: '{#TABLE}'
- uuid: a8371234567890abcdef1234567890ab
name: 'Days remaining: {#TABLE}'
type: DEPENDENT
key: 'zabbix.partitioning.days_left[{#TABLE}]'
delay: 0
history: 7d
description: 'Days until the last partition runs out for {#TABLE}'
master_item:
key: 'zabbix.partitioning.stats[{$PATH.TO.CONFIG}, {#TABLE}]'
preprocessing:
- type: JSONPATH
parameters:
- $.days_left
tags:
- tag: component
value: partitioning
@@ -55,11 +78,50 @@ zabbix_export:
value: '{#TABLE}'
trigger_prototypes:
- uuid: da23fae76a41455c86c58267d6d9f86d
expression: 'last(/Zabbix Partitioning Monitor/zabbix.partitioning.check[{$PATH.TO.CONFIG}, {#TABLE}])<={$PARTITION.DAYS}'
expression: 'last(/Zabbix Partitioning Monitor/zabbix.partitioning.days_left[{#TABLE}])<={$PARTITION.DAYS}'
name: 'Partitioning critical: {#TABLE} has less than {$PARTITION.DAYS} days in partition'
opdata: 'Days till Zabbix server will crash: {ITEM.LASTVALUE}'
priority: DISASTER
description: 'New partitions are not being created. Check the script logs.'
- uuid: b9482345678901bcdef23456789012cd
name: 'Partition Count: {#TABLE}'
type: DEPENDENT
key: 'zabbix.partitioning.count[{#TABLE}]'
delay: 0
history: 7d
description: 'Total number of partitions for {#TABLE}'
master_item:
key: 'zabbix.partitioning.stats[{$PATH.TO.CONFIG}, {#TABLE}]'
preprocessing:
- type: JSONPATH
parameters:
- $.partition_count
tags:
- tag: component
value: partitioning
- tag: table
value: '{#TABLE}'
- uuid: c0593456789012cdef345678901234de
name: 'Table Size: {#TABLE}'
type: DEPENDENT
key: 'zabbix.partitioning.size[{#TABLE}]'
delay: 0
history: 7d
units: B
description: 'Total size (Data+Index) of {#TABLE}'
master_item:
key: 'zabbix.partitioning.stats[{$PATH.TO.CONFIG}, {#TABLE}]'
preprocessing:
- type: JSONPATH
parameters:
- $.size_bytes
tags:
- tag: component
value: partitioning
- tag: table
value: '{#TABLE}'
macros:
- macro: '{$PARTITION.DAYS}'
value: '3'