diff --git a/.github/workflows/plugin-ci-workflow.yml b/.github/workflows/plugin-ci-workflow.yml index 5e4f3db6..e17554e8 100644 --- a/.github/workflows/plugin-ci-workflow.yml +++ b/.github/workflows/plugin-ci-workflow.yml @@ -35,21 +35,12 @@ jobs: integration-test: runs-on: ${{ matrix.os }} - # A failure against the pinned release is a real failure. The develop entry - # is advisory: it is how a core regression becomes visible here, but it must - # not turn the plugin's own pull requests red. - continue-on-error: ${{ matrix.cacti != 'release/1.2.31' }} - strategy: fail-fast: false matrix: php: ['8.1', '8.2', '8.3', '8.4'] os: [ubuntu-latest] cacti: ['release/1.2.31'] - include: - - php: '8.4' - os: ubuntu-latest - cacti: 'develop' services: mariadb: @@ -95,7 +86,24 @@ jobs: echo "PHP_BINARY=$(command -v php)" >> "$GITHUB_ENV" - name: Run apt-get update - run: sudo apt-get update + run: | + for attempt in 1 2 3; do + if sudo timeout 3m apt-get \ + -o Dpkg::Lock::Timeout=60 \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + update; then + exit 0 + fi + + if [ "$attempt" -lt 3 ]; then + sleep 10 + fi + done + + echo 'apt-get update failed after three bounded attempts.' >&2 + exit 1 - name: Install System Dependencies run: sudo apt-get install -y apache2 snmp snmpd rrdtool fping diff --git a/tests/Helpers/CactiStubs.php b/tests/Helpers/CactiStubs.php index 59d084cf..b7515c9e 100644 --- a/tests/Helpers/CactiStubs.php +++ b/tests/Helpers/CactiStubs.php @@ -31,7 +31,7 @@ final class CactiStubs { /** * Every Cacti function call the plugin made, in order. * - * @var array}> + * @var array}> */ public static $calls = []; @@ -107,7 +107,7 @@ public static function reset() { * * @param string $fn Cacti function name. * @param string $sql SQL text, or '' for non-query calls. - * @param array $params Bound parameters, if any. + * @param array $params Bound parameters, if any. * * @return void */ @@ -197,7 +197,7 @@ public static function nextReturn($fn, $default, $sql = '') { * * @param string $fn Cacti function name. * - * @return array}> + * @return array}> */ public static function callsTo($fn) { return array_values(array_filter(self::$calls, function ($call) use ($fn) { diff --git a/tests/Helpers/ThresholdOutcome.php b/tests/Helpers/ThresholdOutcome.php new file mode 100644 index 00000000..4642467c --- /dev/null +++ b/tests/Helpers/ThresholdOutcome.php @@ -0,0 +1,205 @@ + + */ + public $thold; + + /** + * @param array $thold + */ + public function __construct(array $thold) { + $this->thold = $thold; + } + + /** + * Subject lines of the mail that was sent, in order. + * + * @return array + */ + public function subjects() { + return array_column(CactiStubs::$mail, 'subject'); + } + + /** + * Recipients of the mail that was sent, in order. + * + * @return array + */ + public function recipients() { + return array_column(CactiStubs::$mail, 'to'); + } + + /** + * @return int + */ + public function mailCount() { + return count(CactiStubs::$mail); + } + + /** + * Status codes written to plugin_thold_log, in order. + * + * The log row goes through sql_save(), so the status is available as data + * rather than having to be parsed back out of a query. + * + * @return array + */ + public function logStatuses() { + $statuses = []; + + foreach (CactiStubs::callsTo('sql_save') as $call) { + if ($call['sql'] === 'plugin_thold_log' && isset($call['params']['status'])) { + $statuses[] = (int) $call['params']['status']; + } + } + + return $statuses; + } + + /** + * @return int + */ + public function trapCount() { + return count(CactiStubs::callsTo('cacti_snmp_send')); + } + + /** + * Whether the run marked the threshold as having changed state. + * + * @return bool + */ + public function touchedLastChanged() { + foreach (CactiStubs::callsTo('db_execute_prepared') as $call) { + if (strpos($call['sql'], 'lastchanged = NOW()') !== false) { + return true; + } + } + + return false; + } + + /** + * Whether the run set the acknowledgment flag. + * + * @return bool + */ + public function acknowledged() { + foreach (CactiStubs::callsTo('db_execute_prepared') as $call) { + if (strpos($call['sql'], 'acknowledgment = "on"') !== false) { + return true; + } + } + + return false; + } + + /** + * Columns the run wrote to thold_data, resolved to their values. + * + * The statements mix placeholders and literals in the same SET clause, so + * the clause is parsed and each "?" resolved against the bound parameters + * in order. Returns the merge of every such statement, later writes last. + * + * @return array + */ + public function persistedColumns() { + $columns = []; + + foreach (CactiStubs::callsTo('db_execute_prepared') as $call) { + if (strpos($call['sql'], 'UPDATE thold_data') === false) { + continue; + } + + if (!preg_match('/SET\s+(.*?)\s+WHERE/s', $call['sql'], $clause)) { + continue; + } + + $position = 0; + + foreach (explode(',', $clause[1]) as $assignment) { + $parts = explode('=', $assignment, 2); + + if (count($parts) !== 2) { + continue; + } + + $name = trim($parts[0]); + $value = trim($parts[1]); + + if ($value === '?') { + $value = isset($call['params'][$position]) ? (string) $call['params'][$position] : ''; + $position++; + } + + $columns[$name] = trim($value, '"\''); + } + } + + return $columns; + } + + /** + * The alert state the run persisted, or null when it wrote none. + * + * @return int|null + */ + public function persistedAlertState() { + $columns = $this->persistedColumns(); + + return isset($columns['thold_alert']) ? (int) $columns['thold_alert'] : null; + } + + /** + * The fail counts the run persisted, or null when it wrote neither. + * + * @return array{alert: int|null, warning: int|null}|null + */ + public function persistedFailCounts() { + $columns = $this->persistedColumns(); + + if (!isset($columns['thold_fail_count']) && !isset($columns['thold_warning_fail_count'])) { + return null; + } + + return [ + 'alert' => isset($columns['thold_fail_count']) ? (int) $columns['thold_fail_count'] : null, + 'warning' => isset($columns['thold_warning_fail_count']) ? (int) $columns['thold_warning_fail_count'] : null, + ]; + } + + /** + * Whether the run did nothing at all beyond reading. + * + * @return bool + */ + public function isSilent() { + return $this->mailCount() === 0 + && $this->logStatuses() === [] + && $this->trapCount() === 0 + && CactiStubs::callsTo('thold_command_execution') === []; + } +} diff --git a/tests/Helpers/ThresholdScenario.php b/tests/Helpers/ThresholdScenario.php new file mode 100644 index 00000000..de3fc6d4 --- /dev/null +++ b/tests/Helpers/ThresholdScenario.php @@ -0,0 +1,292 @@ + + */ + private $thold; + + /** + * A threshold row with every column the function reads, set to values that + * on their own produce no breach and no notification. + * + * @param array $overrides Columns to change. + */ + private function __construct(array $overrides) { + $this->thold = $overrides + [ + 'id' => 1, + 'name' => 'CPU utilisation', + 'name_cache' => 'CPU utilisation', + 'host_id' => 2, + 'local_data_id' => 4, + 'local_graph_id' => 7, + 'data_template_rrd_id' => 9, + 'data_source_name' => 'traffic_in', + 'thold_type' => 0, + 'data_type' => 0, + 'lastread' => 50, + 'oldvalue' => 50, + 'lasttime' => 0, + 'rrd_step' => 300, + + 'thold_hi' => '', + 'thold_low' => '', + 'thold_warning_hi' => '', + 'thold_warning_low' => '', + 'thold_fail_trigger' => 1, + 'thold_warning_fail_trigger' => 1, + 'thold_fail_count' => 0, + 'thold_warning_fail_count' => 0, + 'thold_alert' => 0, + 'repeat_alert' => 0, + + 'time_hi' => '', + 'time_low' => '', + 'time_warning_hi' => '', + 'time_warning_low' => '', + 'time_fail_trigger' => 1, + 'time_warning_fail_trigger' => 1, + 'time_fail_length' => 300, + 'time_warning_fail_length' => 300, + + 'bl_fail_count' => 0, + 'bl_alert' => 0, + 'bl_pct_down' => '', + 'bl_pct_up' => '', + 'bl_fail_trigger' => 1, + 'bl_ref_time_range' => 3600, + 'bl_type' => 0, + 'bl_cf' => 'AVG', + 'bl_thold_valid' => 0, + 'cdef' => 0, + + 'notify_warning' => 0, + 'notify_alert' => 0, + 'notify_extra' => '', + 'notify_warning_extra' => '', + 'persist_ack' => '', + 'reset_ack' => '', + 'acknowledgment' => '', + 'exempt' => '', + + 'syslog_enabled' => '', + 'syslog_priority' => 5, + 'syslog_facility' => 1, + 'snmp_event_severity' => 3, + 'snmp_event_description' => '', + 'snmp_engine_id' => '', + + 'trigger_cmd_high' => '', + 'trigger_cmd_low' => '', + 'trigger_cmd_norm' => '', + + 'notes' => '', + 'dnotes' => '', + 'external_id' => '', + 'email_subject' => '', + 'email_subject_warn' => '', + 'email_subject_restoral' => '', + 'restored_alert' => '', + 'graph_timespan' => 7, + 'show_units' => '', + 'units_suffix' => '', + 'decimals' => 2, + 'format_file' => '', + 'thold_enabled' => 'on', + 'thold_daemon_id' => 0, + ]; + } + + /** + * @param array $overrides + * + * @return self + */ + public static function threshold(array $overrides = []) { + $scenario = new self($overrides); + + $scenario->device(); + + /* + * The time-based arm multiplies this into a window bound; an empty + * value is a fatal on PHP 8 rather than a missing step. + */ + CactiStubs::willReturnFor('db_fetch_cell_prepared', 'SELECT rrd_step', 300); + + // Counts of prior log rows; the arms add these together arithmetically. + CactiStubs::willReturnFor('db_fetch_cell_prepared', 'SELECT COUNT(id)', 0); + + return $scenario; + } + + /** + * Program the device row the function loads for the threshold's host. + * + * @param array $overrides + * + * @return self + */ + public function device(array $overrides = []) { + CactiStubs::willReturnFor('db_fetch_row_prepared', 'FROM host WHERE id = ?', $overrides + [ + 'id' => 2, + 'description' => 'core-switch-1', + 'hostname' => '10.0.0.1', + 'location' => 'rack 4', + 'site_id' => 1, + 'status' => 3, + 'status_fail_date' => '2026-01-01 00:00:00', + 'status_rec_date' => '2026-01-02 00:00:00', + 'status_last_error' => '', + 'snmp_engine_id' => '', + 'notes' => '', + ]); + + return $this; + } + + /** + * Give the threshold a legacy alert contact, which is what makes the alert + * recipient list non-empty. + * + * @param string $address + * + * @return self + */ + public function alertRecipient($address) { + CactiStubs::willReturnFor('db_fetch_assoc_prepared', 'FROM plugin_thold_contacts', [['data' => $address]]); + + return $this; + } + + /** + * @param string $name + * @param mixed $value + * + * @return self + */ + public function option($name, $value) { + CactiStubs::$configOptions[$name] = $value; + + return $this; + } + + /** + * Give the RRD a set of reference statistics for the baseline arm. + * + * thold reaches these through three rrdtool calls: file_exists, info to + * discover the data sources and consolidation functions, then a graph + * command whose PRINT output is decoded by position. The doubles below + * answer all three in the shapes that decode expects. + * + * @param float|int $average + * @param float|int $max + * @param float|int $min + * @param float|int $last + * @param string $dsname + * + * @return self + */ + public function referenceStatistics($average, $max, $min, $last, $dsname = 'traffic_in') { + /* + * With a storage location set, thold asks rrdtool whether the file + * exists rather than touching the filesystem, which keeps the fixture + * off disk. + */ + CactiStubs::$configOptions['storage_location'] = 1; + + CactiStubs::willReturnFor('db_fetch_cell_prepared', 'SELECT rrd_path', '/var/lib/cacti/rra/test.rrd'); + CactiStubs::willReturnFor('rrdtool_execute', 'file_exists', true); + + /* + * One rra line per consolidation function: the info parser sets a + * single flag per line, and the number of flags set has to match the + * number of values the graph command below prints. + */ + CactiStubs::willReturnFor('rrdtool_execute', 'info ', implode("\n", [ + 'ds[' . $dsname . '].type = "COUNTER"', + 'rra[0].cf = "AVERAGE"', + 'rra[1].cf = "MAX"', + 'rra[2].cf = "MIN"', + 'rra[3].cf = "LAST"', + 'step = 300', + ])); + + /* + * First line is the graph size and is skipped; then one value per + * PRINT in AVG, MAX, MIN, LAST order; then the timing line. + */ + CactiStubs::willReturnFor('rrdtool_execute', 'graph x --start', implode("\n", [ + '0x0', + (string) $average, + (string) $max, + (string) $min, + (string) $last, + 'OK u:0.01 s:0.00 r:0.01', + ])); + + return $this; + } + + /** + * Put the device into a maintenance window. + * + * @return self + */ + public function inMaintenance() { + // Asked more than once per poll, so a queued value would run out. + CactiStubs::willAlwaysReturn('api_plugin_is_enabled', true); + CactiStubs::willAlwaysReturn('plugin_maint_check_cacti_host', true); + + /* + * thold include_once()s the maint plugin when it reports enabled. The + * fixture supplies an empty file so the include succeeds; the function + * it would define is already stubbed. + */ + $maint = dirname(__DIR__, 3) . '/maint'; + + if (!is_dir($maint)) { + mkdir($maint, 0755, true); + } + + if (!file_exists($maint . '/functions.php')) { + file_put_contents($maint . '/functions.php', "thold; + + thold_check_threshold($thold); + + return new ThresholdOutcome($thold); + } +} diff --git a/tests/Unit/TholdEvaluationContextTest.php b/tests/Unit/TholdEvaluationContextTest.php new file mode 100644 index 00000000..758ee811 --- /dev/null +++ b/tests/Unit/TholdEvaluationContextTest.php @@ -0,0 +1,206 @@ + $overrides + * + * @return array + */ + private function context(array $overrides = []) { + return thold_evaluation_context($overrides + [ + 'id' => 1, + 'name' => 'CPU', + 'name_cache' => 'CPU', + 'local_graph_id' => 7, + 'local_data_id' => 4, + 'data_source_name' => 'traffic_in', + 'lastread' => 50, + 'thold_alert' => 0, + 'thold_fail_trigger' => 3, + 'thold_warning_fail_trigger' => 2, + 'notify_alert' => 0, + 'notify_warning' => 0, + 'notify_extra' => '', + 'notify_warning_extra' => '', + 'syslog_enabled' => '', + 'syslog_priority' => 5, + 'syslog_facility' => 1, + ]); + } + + /** + * @return void + */ + public function testThresholdTriggersAreUsedWhenSet(): void { + $context = $this->context(); + + $this->assertSame(3, $context['trigger']); + $this->assertSame(2, $context['warning_trigger']); + } + + /** + * @return void + */ + public function testUnsetTriggersFallBackToTheGlobalDefault(): void { + CactiStubs::$configOptions['alert_trigger'] = 5; + + $context = $this->context([ + 'thold_fail_trigger' => '', + 'thold_warning_fail_trigger' => '', + ]); + + $this->assertSame(5, $context['trigger']); + $this->assertSame(5, $context['warning_trigger']); + } + + /** + * @return void + */ + public function testSyslogIsOffUnlessTheThresholdEnablesIt(): void { + $this->assertFalse($this->context()['syslog']); + $this->assertTrue($this->context(['syslog_enabled' => 'on'])['syslog']); + } + + /** + * @return array + */ + public static function trapSettingProvider() { + return [ + 'alert' => ['thold_alert_snmp', 'thold_snmp_traps'], + 'warning' => ['thold_alert_snmp_warning', 'thold_snmp_warning_traps'], + 'normal' => ['thold_alert_snmp_normal', 'thold_snmp_normal_traps'], + ]; + } + + /** + * @dataProvider trapSettingProvider + * + * @param string $option + * @param string $key + * + * @return void + */ + public function testEachTrapClassFollowsItsOwnSetting($option, $key): void { + $this->assertFalse($this->context()[$key]); + + CactiStubs::$configOptions[$option] = 'on'; + + $this->assertTrue($this->context()[$key]); + } + + /** + * Alerts go to the warning recipients as well only when the two lists + * differ and the operator has asked for it. + * + * @return void + */ + public function testWarningRecipientsAreAddedOnlyWhenConfiguredAndDistinct(): void { + CactiStubs::$configOptions['thold_notify_alerts_to_warning_recipients'] = 'on'; + + $this->assertTrue($this->context(['notify_alert' => 1, 'notify_warning' => 2])['notify_different']); + $this->assertFalse($this->context(['notify_alert' => 2, 'notify_warning' => 2])['notify_different']); + $this->assertFalse($this->context(['notify_alert' => 1, 'notify_warning' => 0])['notify_different']); + } + + /** + * @return void + */ + public function testWarningRecipientsAreNotAddedWhenTheOptionIsOff(): void { + $this->assertFalse($this->context(['notify_alert' => 1, 'notify_warning' => 2])['notify_different']); + } + + /** + * @return void + */ + public function testAGraphIsAttachedByDefault(): void { + $file_array = $this->context()['file_array']; + + $this->assertSame(7, $file_array['local_graph_id']); + $this->assertSame('image/png', $file_array['mimetype']); + } + + /** + * @return void + */ + public function testTextOnlyNotificationsAttachNoGraph(): void { + CactiStubs::$configOptions['thold_send_text_only'] = 'on'; + + $this->assertSame([], $this->context()['file_array']); + } + + /** + * @return void + */ + public function testNoGraphIsAttachedWhenTheThresholdHasNone(): void { + $this->assertSame([], $this->context(['local_graph_id' => 0])['file_array']); + } + + /** + * @return void + */ + public function testTheGraphUrlPointsAtTheThresholdsGraph(): void { + CactiStubs::$configOptions['base_url'] = 'http://cacti.example.org'; + + $this->assertSame( + 'http://cacti.example.org/graph.php?local_graph_id=7&rra_id=all', + $this->context()['url'] + ); + } + + /** + * @return void + */ + public function testRecipientsAreResolvedForBothClasses(): void { + CactiStubs::willReturnFor('db_fetch_assoc_prepared', 'FROM plugin_thold_contacts', [['data' => 'ops@example.org']]); + + $context = $this->context(); + + $this->assertStringContainsString('ops@example.org', $context['alert_emails']); + $this->assertArrayHasKey('warning_emails', $context); + $this->assertArrayHasKey('alert_bcc_emails', $context); + $this->assertArrayHasKey('warning_bcc_emails', $context); + } + + /** + * @return void + */ + public function testTheCurrentReadingIsCarriedThrough(): void { + $this->assertSame(95, $this->context(['lastread' => 95])['lastread']); + } + + /** + * @return void + */ + public function testThePreviousAlertStateIsCarriedThrough(): void { + $this->assertSame(2, $this->context(['thold_alert' => 2])['alertstat']); + } +} diff --git a/tests/Unit/ThresholdBaselineCharacterizationTest.php b/tests/Unit/ThresholdBaselineCharacterizationTest.php new file mode 100644 index 00000000..a510eaf0 --- /dev/null +++ b/tests/Unit/ThresholdBaselineCharacterizationTest.php @@ -0,0 +1,186 @@ + $overrides + * + * @return ThresholdScenario + */ + private function baseline(array $overrides = []) { + return ThresholdScenario::threshold($overrides + [ + 'thold_type' => 1, + 'bl_type' => 0, + 'bl_pct_up' => 10, + 'bl_pct_down' => 10, + 'bl_ref_time_range' => 3600, + 'bl_fail_trigger' => 1, + ]) + ->alertRecipient('ops@example.org') + ->referenceStatistics(100, 100, 100, 100); + } + + /** + * @return void + */ + public function testReadingInsideTheBandEmitsNothing(): void { + $outcome = $this->baseline(['lastread' => 100])->poll(); + + $this->assertTrue($outcome->isSilent()); + $this->assertSame(0, $outcome->thold['bl_alert']); + } + + /** + * @return void + */ + public function testReadingAboveTheBandAlerts(): void { + $outcome = $this->baseline(['lastread' => 500])->poll(); + + $this->assertSame(STAT_HI, $outcome->thold['bl_alert']); + $this->assertSame([ST_NOTIFYAL], $outcome->logStatuses()); + $this->assertSame(1, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testReadingBelowTheBandAlerts(): void { + $outcome = $this->baseline(['lastread' => 1])->poll(); + + $this->assertSame(STAT_LO, $outcome->thold['bl_alert']); + $this->assertSame([ST_NOTIFYAL], $outcome->logStatuses()); + $this->assertSame(1, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testReturningToTheBandNotifiesTheRestoral(): void { + $outcome = $this->baseline([ + 'lastread' => 100, + 'bl_alert' => STAT_HI, + 'bl_fail_count' => 3, + ])->poll(); + + $this->assertSame(0, $outcome->thold['bl_alert']); + $this->assertSame([ST_RESTORAL], $outcome->logStatuses()); + $this->assertSame(1, $outcome->mailCount()); + } + + /** + * When rrdtool returns no reference statistics the arm cannot decide + * anything, so it reports -1 and leaves the threshold alone. This is the + * state a newly created baseline threshold sits in until its reference + * window has filled. + * + * @return void + */ + public function testMissingReferenceStatisticsEmitsNothing(): void { + $outcome = ThresholdScenario::threshold([ + 'thold_type' => 1, + 'bl_type' => 0, + 'bl_pct_up' => 10, + 'bl_pct_down' => 10, + 'lastread' => 50, + ])->alertRecipient('ops@example.org')->poll(); + + $this->assertSame(-1, $outcome->thold['bl_alert']); + $this->assertTrue($outcome->isSilent()); + } + + /** + * @return void + */ + public function testBreachBelowTheTriggerDoesNotNotify(): void { + $outcome = $this->baseline([ + 'lastread' => 500, + 'bl_fail_trigger' => 3, + 'bl_fail_count' => 0, + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testMaintenanceWindowSuppressesNotification(): void { + $outcome = $this->baseline(['lastread' => 500])->inMaintenance()->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testAcknowledgedThresholdDoesNotMailOnBreach(): void { + $outcome = $this->baseline([ + 'lastread' => 500, + 'acknowledgment' => 'on', + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * An absolute-deviation baseline adds the configured amount to the + * reference rather than a percentage of it, so 100 with a band of 10 gives + * the same 90 to 110 range for a very different configuration. + * + * @return void + */ + public function testAbsoluteDeviationUsesTheBandAsAnAmount(): void { + $inside = $this->baseline([ + 'bl_type' => 2, + 'lastread' => 105, + 'bl_pct_up' => 10, + 'bl_pct_down' => 10, + ])->poll(); + + $this->assertSame(0, $inside->thold['bl_alert']); + } + + /** + * @return void + */ + public function testAbsoluteDeviationAlertsOutsideTheAmount(): void { + $outcome = $this->baseline([ + 'bl_type' => 2, + 'lastread' => 200, + 'bl_pct_up' => 10, + 'bl_pct_down' => 10, + ])->poll(); + + $this->assertSame(STAT_HI, $outcome->thold['bl_alert']); + } +} diff --git a/tests/Unit/ThresholdHiLowCharacterizationTest.php b/tests/Unit/ThresholdHiLowCharacterizationTest.php new file mode 100644 index 00000000..85eb1004 --- /dev/null +++ b/tests/Unit/ThresholdHiLowCharacterizationTest.php @@ -0,0 +1,269 @@ + $overrides + * + * @return ThresholdScenario + */ + private function bounded(array $overrides = []) { + return ThresholdScenario::threshold($overrides + [ + 'thold_hi' => 90, + 'thold_low' => 10, + 'thold_warning_hi' => 80, + 'thold_warning_low' => 20, + ])->alertRecipient('ops@example.org'); + } + + /** + * @return void + */ + public function testReadingInsideBothBoundsEmitsNothing(): void { + $outcome = $this->bounded(['lastread' => 50])->poll(); + + $this->assertTrue($outcome->isSilent()); + } + + /** + * @return void + */ + public function testBreachAtTriggerNotifiesAndLogsTheAlert(): void { + $outcome = $this->bounded(['lastread' => 95, 'thold_fail_trigger' => 1])->poll(); + + $this->assertSame(1, $outcome->mailCount()); + $this->assertSame([ST_NOTIFYAL], $outcome->logStatuses()); + $this->assertTrue($outcome->touchedLastChanged()); + + /* + * The legacy contact list is joined with the global address and the + * device address whether or not those are set, so the To header carries + * trailing empty entries. Recorded, not endorsed. + */ + $this->assertSame(['ops@example.org,,'], $outcome->recipients()); + } + + /** + * @return void + */ + public function testBreachBelowTriggerCountsButDoesNotNotify(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'thold_fail_trigger' => 3, + 'thold_fail_count' => 0, + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + + /* + * No log row either. ST_TRIGGERA exists for this case but is only + * written by the time-based arm, so a hi/low threshold counting up to + * its trigger leaves no trace in the log. + */ + $this->assertSame([], $outcome->logStatuses()); + + /* + * An alert breach also zeroes the warning counter, so a threshold that + * crosses the warning bound on its way up loses that progress. + */ + $this->assertSame(['alert' => 1, 'warning' => 0], $outcome->persistedFailCounts()); + } + + /** + * The alert state is recorded on the first breaching poll, before the + * trigger count is met, so the interface shows a threshold in alert that + * has not notified and may never do so. + * + * @return void + */ + public function testAlertStateIsRecordedBeforeTheTriggerIsMet(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'thold_fail_trigger' => 3, + ])->poll(); + + $this->assertSame(STAT_HI, $outcome->persistedAlertState()); + } + + /** + * @return void + */ + public function testBreachBelowTheLowerBoundRecordsTheLowState(): void { + $outcome = $this->bounded(['lastread' => 5])->poll(); + + $this->assertSame(STAT_LO, $outcome->persistedAlertState()); + $this->assertSame([ST_NOTIFYAL], $outcome->logStatuses()); + } + + /** + * @return void + */ + public function testReadingBetweenWarningAndAlertBoundsNotifiesTheWarning(): void { + $outcome = $this->bounded(['lastread' => 85])->poll(); + + $this->assertSame([ST_NOTIFYWA], $outcome->logStatuses()); + } + + /** + * @return void + */ + public function testRestoralFromAlertNotifiesAndClearsTheState(): void { + $outcome = $this->bounded([ + 'lastread' => 50, + 'thold_alert' => STAT_HI, + 'thold_fail_count' => 3, + ])->poll(); + + $this->assertSame([ST_NOTIFYRS], $outcome->logStatuses()); + $this->assertSame(STAT_NORMAL, $outcome->persistedAlertState()); + $this->assertSame(1, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testRestoralResetsBothFailCounts(): void { + $outcome = $this->bounded([ + 'lastread' => 50, + 'thold_alert' => STAT_HI, + 'thold_fail_count' => 3, + 'thold_warning_fail_count' => 2, + ])->poll(); + + $this->assertSame(['alert' => 0, 'warning' => 0], $outcome->persistedFailCounts()); + } + + /** + * A re-alert fires when the fail count passes the trigger and lands on a + * multiple of repeat_alert. + * + * @return void + */ + public function testRepeatAlertNotifiesAgainOnTheConfiguredInterval(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'thold_fail_trigger' => 1, + 'thold_fail_count' => 3, + 'repeat_alert' => 2, + ])->poll(); + + $this->assertSame([ST_NOTIFYRA], $outcome->logStatuses()); + } + + /** + * @return void + */ + public function testRepeatAlertStaysQuietBetweenIntervals(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'thold_fail_trigger' => 1, + 'thold_fail_count' => 1, + 'repeat_alert' => 3, + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testAcknowledgedThresholdDoesNotMailOnBreach(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'acknowledgment' => 'on', + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testPersistAckSetsTheAcknowledgmentOnFirstNotification(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'persist_ack' => 'on', + ])->poll(); + + $this->assertTrue($outcome->acknowledged()); + } + + /** + * A device in a maintenance window still evaluates, but must not notify + * and must not advance the fail count. + * + * @return void + */ + public function testMaintenanceWindowSuppressesNotification(): void { + $outcome = $this->bounded(['lastread' => 95])->inMaintenance()->poll(); + + $this->assertSame(0, $outcome->mailCount()); + $this->assertSame([], $outcome->logStatuses()); + } + + /** + * @return void + */ + public function testUnknownReadingEmitsNoAlert(): void { + $outcome = $this->bounded(['lastread' => 'U'])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * With no bounds configured the threshold can never breach, whatever the + * reading. + * + * @return void + */ + public function testThresholdWithNoBoundsNeverBreaches(): void { + $outcome = ThresholdScenario::threshold(['lastread' => 99999]) + ->alertRecipient('ops@example.org') + ->poll(); + + $this->assertTrue($outcome->isSilent()); + } + + /** + * @return void + */ + public function testDisabledGloballyStopsBeforeAnyEvaluation(): void { + $outcome = $this->bounded(['lastread' => 95]) + ->option('thold_disable_all', 'on') + ->poll(); + + $this->assertTrue($outcome->isSilent()); + } +} diff --git a/tests/Unit/ThresholdTimeBasedCharacterizationTest.php b/tests/Unit/ThresholdTimeBasedCharacterizationTest.php new file mode 100644 index 00000000..e5a87fd5 --- /dev/null +++ b/tests/Unit/ThresholdTimeBasedCharacterizationTest.php @@ -0,0 +1,152 @@ + $overrides + * + * @return ThresholdScenario + */ + private function bounded(array $overrides = []) { + return ThresholdScenario::threshold($overrides + [ + 'thold_type' => 2, + 'time_hi' => 90, + 'time_low' => 10, + ])->alertRecipient('ops@example.org'); + } + + /** + * @return void + */ + public function testReadingInsideBothBoundsEmitsNothing(): void { + $outcome = $this->bounded(['lastread' => 50])->poll(); + + $this->assertTrue($outcome->isSilent()); + } + + /** + * @return void + */ + public function testBreachAtTriggerNotifiesAndLogsTheAlert(): void { + $outcome = $this->bounded(['lastread' => 95, 'time_fail_trigger' => 1])->poll(); + + $this->assertSame(1, $outcome->mailCount()); + $this->assertSame([ST_NOTIFYAL], $outcome->logStatuses()); + $this->assertSame(STAT_HI, $outcome->persistedAlertState()); + } + + /** + * @return void + */ + public function testBreachBelowTheLowerBoundRecordsTheLowState(): void { + $outcome = $this->bounded(['lastread' => 5, 'time_fail_trigger' => 1])->poll(); + + $this->assertSame(STAT_LO, $outcome->persistedAlertState()); + } + + /** + * The hi/low arm mails on restoral. This one writes the restoral to the log + * and clears the state, but sends nothing, so an operator watching a + * time-based threshold sees the alert and never the all-clear. + * + * @return void + */ + public function testRestoralLogsButDoesNotMail(): void { + $outcome = $this->bounded([ + 'lastread' => 50, + 'thold_alert' => STAT_HI, + 'thold_fail_count' => 3, + ])->poll(); + + $this->assertSame([ST_NOTIFYRS], $outcome->logStatuses()); + $this->assertSame(STAT_NORMAL, $outcome->persistedAlertState()); + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testRestoralResetsTheFailCounts(): void { + $outcome = $this->bounded([ + 'lastread' => 50, + 'thold_alert' => STAT_HI, + 'thold_fail_count' => 3, + 'thold_warning_fail_count' => 2, + ])->poll(); + + $this->assertSame(['alert' => 0, 'warning' => 0], $outcome->persistedFailCounts()); + } + + /** + * @return void + */ + public function testMaintenanceWindowSuppressesNotification(): void { + $outcome = $this->bounded(['lastread' => 95, 'time_fail_trigger' => 1]) + ->inMaintenance() + ->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testAcknowledgedThresholdDoesNotMailOnBreach(): void { + $outcome = $this->bounded([ + 'lastread' => 95, + 'time_fail_trigger' => 1, + 'acknowledgment' => 'on', + ])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testUnknownReadingEmitsNoAlert(): void { + $outcome = $this->bounded(['lastread' => 'U'])->poll(); + + $this->assertSame(0, $outcome->mailCount()); + } + + /** + * @return void + */ + public function testThresholdWithNoBoundsNeverBreaches(): void { + $outcome = ThresholdScenario::threshold(['thold_type' => 2, 'lastread' => 99999]) + ->alertRecipient('ops@example.org') + ->poll(); + + $this->assertTrue($outcome->isSilent()); + } +} diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index ce2b2615..6c4a8e4f 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -62,6 +62,8 @@ require_once $autoload; require_once __DIR__ . '/Helpers/CactiStubs.php'; require_once __DIR__ . '/TestCase.php'; +require_once __DIR__ . '/Helpers/ThresholdOutcome.php'; +require_once __DIR__ . '/Helpers/ThresholdScenario.php'; /* * base_path has to point at the Cacti root two levels above this plugin: @@ -157,7 +159,7 @@ function db_fetch_cell_prepared($sql, $params = [], $col_name = '', $log = true, } if (!function_exists('db_qstr')) { - function db_qstr($string) { + function db_qstr($string, $db_conn = false) { return "'" . str_replace("'", "''", (string) $string) . "'"; } } @@ -482,7 +484,7 @@ function raise_message($message_id, $message = '', $level = 0) { function rrdtool_execute($command, $log_to_stdout = false, $output_flag = 1, $rrdtool_pipe = false, $logopt = 'WEBLOG') { CactiStubs::record('rrdtool_execute', $command); - return CactiStubs::nextReturn('rrdtool_execute', ''); + return CactiStubs::nextReturn('rrdtool_execute', '', $command); } } @@ -521,8 +523,13 @@ function number_format_i18n($number, $decimals = 0, $baseu = 1000) { } } -if (!defined('RRDTOOL_OUTPUT_STDOUT')) { - define('RRDTOOL_OUTPUT_STDOUT', 1); +// rrdtool output modes, from Cacti include/global_constants.php. +foreach (['RRDTOOL_OUTPUT_NULL' => 0, 'RRDTOOL_OUTPUT_STDOUT' => 1, 'RRDTOOL_OUTPUT_STDERR' => 2, + 'RRDTOOL_OUTPUT_GRAPH_DATA' => 3, 'RRDTOOL_OUTPUT_BOOLEAN' => 4, + 'RRDTOOL_OUTPUT_RETURN_STDERR' => 5] as $name => $value) { + if (!defined($name)) { + define($name, $value); + } } if (!defined('CACTI_DATE_TIME_FORMAT')) { diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 00000000..518f7321 --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,29 @@ +# Test runner for the Thold plugin. +# +# Pinned to PHP 8.1 because that is the oldest interpreter the CI matrix +# covers; what passes here passes on 8.2-8.4. pcov rather than Xdebug: line +# coverage is the only debug feature the suite needs and pcov is far cheaper. +FROM php:8.1-cli-alpine@sha256:7949370448b0b4d9787776dc5968e0fd8d48763292344b5fbf21539441228a98 + +# git is needed by the changed-line coverage gate, which diffs against the +# base branch. +RUN apk add --no-cache git gmp-dev \ + && docker-php-ext-install gmp \ + && apk add --no-cache --virtual .build-deps $PHPIZE_DEPS \ + && pecl install pcov \ + && docker-php-ext-enable pcov \ + && apk del .build-deps + +COPY --from=composer:2@sha256:4d71c3c2109c61d5415544264b59ad4087e4c5b7244481723664138fd36d5040 /usr/bin/composer /usr/bin/composer + +# The plugin lives where Cacti would put it, because thold_functions.php +# resolves its own includes through $config['base_path'] . '/plugins/thold'. +# No network or database is involved; the Cacti framework functions themselves +# are stubbed in tests/bootstrap.php. +WORKDIR /cacti/plugins/thold + +ENV COMPOSER_ALLOW_SUPERUSER=1 \ + COMPOSER_NO_INTERACTION=1 \ + COMPOSER_CACHE_DIR=/tmp/composer-cache + +CMD ["sh", "-c", "composer install --no-progress --no-ansi && composer test"] diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml new file mode 100644 index 00000000..99d38b47 --- /dev/null +++ b/tests/docker/docker-compose.yml @@ -0,0 +1,15 @@ +# Local mirror of the unit-test CI job. `docker compose -f +# tests/docker/docker-compose.yml run --rm phpunit` runs exactly what CI runs. +services: + phpunit: + build: + context: . + dockerfile: Dockerfile + image: cacti-thold-test:php8.1 + working_dir: /cacti/plugins/thold + volumes: + - ../..:/cacti/plugins/thold + - composer-cache:/tmp/composer-cache + +volumes: + composer-cache: diff --git a/thold_functions.php b/thold_functions.php index 018cf99b..2911e8ce 100644 --- a/thold_functions.php +++ b/thold_functions.php @@ -2171,6 +2171,71 @@ function thold_datasource_required($name, $data_source) { return true; } +/** + * Gather the settings a threshold evaluation reads, in one place. + * + * Everything here is derived from the threshold row and the Cacti settings; it + * does not decide anything and has no side effects, which is what lets it move + * out of thold_check_threshold() without changing behaviour. + * + * @param array $thold_data Threshold row. + * + * @return array + */ +function thold_evaluation_context(array $thold_data) { + $alert_trigger = read_config_option('alert_trigger'); + $httpurl = read_config_option('base_url'); + $thold_send_text_only = read_config_option('thold_send_text_only'); + + // see if we have two notification lists or one + $notify_different = $thold_data['notify_warning'] > 0 + && $thold_data['notify_warning'] != $thold_data['notify_alert'] + && read_config_option('thold_notify_alerts_to_warning_recipients') == 'on'; + + $file_array = []; + + if ($thold_send_text_only != 'on' && !empty($thold_data['local_graph_id'])) { + $file_array = [ + 'local_graph_id' => $thold_data['local_graph_id'], + 'local_data_id' => $thold_data['local_data_id'], + 'rra_id' => 0, + 'file' => "$httpurl/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', + 'mimetype' => 'image/png', + 'filename' => clean_up_name(thold_get_cached_name($thold_data)) + ]; + } + + return [ + // Settings for syslogging + 'syslog' => $thold_data['syslog_enabled'] == 'on', + 'syslog_priority' => $thold_data['syslog_priority'], + 'syslog_facility' => $thold_data['syslog_facility'], + + 'realert' => read_config_option('alert_repeat'), + 'alert_bl_trigger' => read_config_option('alert_bl_trigger'), + + 'thold_snmp_traps' => read_config_option('thold_alert_snmp') == 'on', + 'thold_snmp_warning_traps' => read_config_option('thold_alert_snmp_warning') == 'on', + 'thold_snmp_normal_traps' => read_config_option('thold_alert_snmp_normal') == 'on', + 'cacti_polling_interval' => read_config_option('poller_interval'), + + // An unset trigger on the threshold falls back to the global default. + 'trigger' => $thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger'], + 'warning_trigger' => $thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger'], + 'alertstat' => $thold_data['thold_alert'], + + 'notify_different' => $notify_different, + 'file_array' => $file_array, + 'url' => $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all', + 'lastread' => $thold_data['lastread'], + + 'alert_emails' => get_thold_emails($thold_data, 'alert', 'to'), + 'alert_bcc_emails' => get_thold_emails($thold_data, 'alert', 'bcc'), + 'warning_emails' => get_thold_emails($thold_data, 'warning', 'to'), + 'warning_bcc_emails' => get_thold_emails($thold_data, 'warning', 'bcc'), + ]; +} + function thold_check_threshold(&$thold_data) { global $config, $plugins, $debug; @@ -2250,81 +2315,28 @@ function thold_check_threshold(&$thold_data) { // ensure that Cacti will make of individual defined SNMP Engine IDs $overwrite['snmp_engine_id'] = $h['snmp_engine_id']; - // pull a few default settings - $global_alert_address = read_config_option('alert_email'); - - // Settings for syslogging - $syslog = $thold_data['syslog_enabled'] == 'on' ? true : false; - $syslog_priority = $thold_data['syslog_priority']; - $syslog_facility = $thold_data['syslog_facility']; - - $realert = read_config_option('alert_repeat'); - $alert_trigger = read_config_option('alert_trigger'); - $alert_bl_trigger = read_config_option('alert_bl_trigger'); - $httpurl = read_config_option('base_url'); - $thold_send_text_only = read_config_option('thold_send_text_only'); - - $thold_snmp_traps = (read_config_option('thold_alert_snmp') == 'on'); - $thold_snmp_warning_traps = (read_config_option('thold_alert_snmp_warning') == 'on'); - $thold_snmp_normal_traps = (read_config_option('thold_alert_snmp_normal') == 'on'); - $cacti_polling_interval = read_config_option('poller_interval'); - - // remove this after adding an option for it - $show_datasource = thold_datasource_required(thold_get_cached_name($thold_data), $thold_data['data_source_name']); - - $trigger = ($thold_data['thold_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_fail_trigger']); - $warning_trigger = ($thold_data['thold_warning_fail_trigger'] == '' ? $alert_trigger : $thold_data['thold_warning_fail_trigger']); - $alertstat = $thold_data['thold_alert']; - - // see if we have two notification lists or one - $notify_different = false; - - if ($thold_data['notify_warning'] > 0) { - if ($thold_data['notify_warning'] != $thold_data['notify_alert']) { - if (read_config_option('thold_notify_alerts_to_warning_recipients') == 'on') { - $notify_different = true; - } - } - } - - // setup base units - $baseu = db_fetch_cell_prepared('SELECT base_value - FROM graph_templates_graph - WHERE local_graph_id = ?', - [$thold_data['local_graph_id']]); - - if ($thold_data['data_type'] == 2) { - $suffix = false; - } else { - $suffix = true; - } - - $show_units = ($thold_data['show_units'] ? true : false); - $units_suffix = $thold_data['units_suffix']; - $decimals = $thold_data['decimals'] >= 0 ? $thold_data['decimals'] : 2; - - $file_array = []; - - if ($thold_send_text_only != 'on') { - if (!empty($thold_data['local_graph_id'])) { - $file_array = [ - 'local_graph_id' => $thold_data['local_graph_id'], - 'local_data_id' => $thold_data['local_data_id'], - 'rra_id' => 0, - 'file' => "$httpurl/graph_image.php?local_graph_id=" . $thold_data['local_graph_id'] . '&rra_id=0&view_type=tree', - 'mimetype' => 'image/png', - 'filename' => clean_up_name(thold_get_cached_name($thold_data)) - ]; - } - } - - $url = $httpurl . '/graph.php?local_graph_id=' . $thold_data['local_graph_id'] . '&rra_id=all'; - $lastread = $thold_data['lastread']; - - $alert_emails = get_thold_emails($thold_data, 'alert', 'to'); - $alert_bcc_emails = get_thold_emails($thold_data, 'alert', 'bcc'); - $warning_emails = get_thold_emails($thold_data, 'warning', 'to'); - $warning_bcc_emails = get_thold_emails($thold_data, 'warning', 'bcc'); + $context = thold_evaluation_context($thold_data); + + $syslog = $context['syslog']; + $syslog_priority = $context['syslog_priority']; + $syslog_facility = $context['syslog_facility']; + $realert = $context['realert']; + $alert_bl_trigger = $context['alert_bl_trigger']; + $thold_snmp_traps = $context['thold_snmp_traps']; + $thold_snmp_warning_traps = $context['thold_snmp_warning_traps']; + $thold_snmp_normal_traps = $context['thold_snmp_normal_traps']; + $cacti_polling_interval = $context['cacti_polling_interval']; + $trigger = $context['trigger']; + $warning_trigger = $context['warning_trigger']; + $alertstat = $context['alertstat']; + $notify_different = $context['notify_different']; + $file_array = $context['file_array']; + $url = $context['url']; + $lastread = $context['lastread']; + $alert_emails = $context['alert_emails']; + $alert_bcc_emails = $context['alert_bcc_emails']; + $warning_emails = $context['warning_emails']; + $warning_bcc_emails = $context['warning_bcc_emails']; switch ($thold_data['thold_type']) { case 0: // hi/low