Retry failed queued email notifications

CHANGELOG.md

@@ -7,6 +7,7 @@
 * issue#710: Fixing Typo in thold_daemons.service File
 * issue#714: Increase the Name column to 255 characters
 * issue#719: Plugin Disabled due to mix of string and int
+* issue#784: Retry failed queued email notifications with bounded exponential backoff
 * issue#812: Recover stale notification claims, scope worker drains, and deprecate the ignored notification --thread option
 * issue: All Columns checkd on Thresholds page
 * issue: Special character previous value handling broken on data query indexes with special characters

INFO

@@ -21,7 +21,7 @@
 
 [info]
 name = thold
-version = 1.8.2
+version = 1.8.3
 longname = Thresholds
 author = The Cacti Group
 email = 

README.md

@@ -27,6 +27,12 @@ and become familiar with its settings.  From there, you can provide overall
 control of thold, and set defaults for things like Email bodies, weekend
 exemptions, alert log retention, logging, etc.
 
+When the notification queue is enabled, transient email failures are retried
+up to five times with exponential backoff from one to eight minutes. The
+Notification Queue page shows the attempt count and next eligible retry time.
+After the fifth failed attempt the row becomes a terminal error so a permanent
+SMTP or address failure cannot retry forever.
+
 Notification workers claim queue rows with their process ID and drain only
 that claim. Unfinished rows are released when a worker stops or notifications
 are suspended, while claims left by a hard-killed worker are recovered after

includes/database.php

@@ -1755,6 +1755,27 @@ function thold_upgrade_database($force = false) {
 		db_execute('UPDATE plugin_notification_lists SET enabled = "on"');
 	}
 
+	if (cacti_version_compare($oldv, '1.8.3', '<')) {
+		db_add_column('notification_queue', [
+			'name'     => 'attempt_count',
+			'type'     => 'int',
+			'unsigned' => true,
+			'NULL'     => false,
+			'default'  => '0',
+			'after'    => 'error_message']
+		);
+
+		db_add_column('notification_queue', [
+			'name'    => 'next_attempt',
+			'type'    => 'timestamp',
+			'NULL'    => true,
+			'default' => null,
+			'after'   => 'attempt_count']
+		);
+
+		db_add_index('notification_queue', 'INDEX', 'retry_ready', ['event_processed', 'process_id', 'next_attempt']);
+	}
+
 	db_add_column('thold_data', [
 		'name'     => 'external_id',
 		'type'     => 'varchar(20)',
@@ -2123,12 +2144,15 @@ function thold_setup_database() {
 	$data['columns'][] = ['name' => 'event_data', 'type' => 'longblob', 'NULL' => false, 'default' => ''];
 	$data['columns'][] = ['name' => 'error_code', 'type' => 'int', 'NULL' => false, 'default' => '0'];
 	$data['columns'][] = ['name' => 'error_message', 'type' => 'varchar(128)', 'NULL' => false, 'default' => ''];
+	$data['columns'][] = ['name' => 'attempt_count', 'type' => 'int', 'unsigned' => true, 'NULL' => false, 'default' => '0'];
+	$data['columns'][] = ['name' => 'next_attempt', 'type' => 'timestamp', 'NULL' => true, 'default' => null];
 	$data['columns'][] = ['name' => 'process_id', 'type' => 'int', 'unsigned' => true, 'NULL' => false, 'default' => '0'];
 	$data['columns'][] = ['name' => 'event_processed', 'type' => 'tinyint', 'unsigned' => true, 'NULL' => false, 'default' => '0'];
 	$data['columns'][] = ['name' => 'event_processed_time', 'type' => 'timestamp', 'NULL' => false, 'default' => '0000-00-00'];
 	$data['columns'][] = ['name' => 'event_processed_runtime', 'type' => 'double', 'unsigned' => true, 'NULL' => false, 'default' => '0'];
 	$data['primary']   = 'id';
 	$data['keys'][]    = ['name' => 'topic_processed', 'columns' => 'topic`, `event_processed'];
+	$data['keys'][]    = ['name' => 'retry_ready', 'columns' => 'event_processed`, `process_id`, `next_attempt'];
 	$data['keys'][]    = ['name' => 'process_id', 'columns' => 'process_id'];
 	$data['keys'][]    = ['name' => 'object_id', 'columns' => 'object_id'];
 	$data['keys'][]    = ['name' => 'host_id', 'columns' => 'host_id'];

notify_queue.php

@@ -463,6 +463,18 @@ function clearFilter() {
 			'sort'    => 'DESC',
 			'tip'     => __('Did this notification result in an error.  Hover on the error column for details.', 'thold')
 		],
+		'attempt_count' => [
+			'display' => __('Attempts', 'thold'),
+			'align'   => 'right',
+			'sort'    => 'DESC',
+			'tip'     => __('The number of delivery attempts made for this notification.', 'thold')
+		],
+		'next_attempt' => [
+			'display' => __('Next Attempt', 'thold'),
+			'align'   => 'right',
+			'sort'    => 'DESC',
+			'tip'     => __('When a failed notification is eligible for its next retry.', 'thold')
+		],
 		'event_processed_runtime' => [
 			'display' => __('Run Time', 'thold'),
 			'align'   => 'right',
@@ -491,14 +503,9 @@ function clearFilter() {
 
 			form_selectable_cell($n['id'], $n['id'], '', 'right');
 			form_selectable_cell($n['event_time'], $n['id'], '', 'right');
-			form_selectable_cell($n['event_processed'] == 0 ? __('Pending', 'thold') : __('Done', 'thold'), $n['id'], '', 'right');
 
-			if ($n['event_processed'] > 0) {
-				form_selectable_cell($n['error_code'] > 0 ? __('Errored', 'thold') : __('Success', 'thold'), $n['id'], '', 'right');
-				form_selectable_cell(number_format_i18n($n['event_processed_runtime'], 2), $n['id'], '', 'right');
-			} else {
-				form_selectable_cell(__('N/A', 'thold'), $n['id'], '', 'right');
-				form_selectable_cell(__('N/A', 'thold'), $n['id'], '', 'right');
+			foreach (thold_notification_queue_status_cells($n) as $cell) {
+				form_selectable_cell($cell, $n['id'], '', 'right');
 			}
 
 			form_checkbox_cell($n['object_name'], $n['id']);

tests/Unit/NotificationQueueClaimTest.php

@@ -130,6 +130,7 @@ public function testAClaimRecoversOrphansThenTakesOnlyUnheldRows(): void {
 		$this->assertStringContainsString('LEFT JOIN processes', $calls[0]['sql']);
 		$this->assertStringContainsString('p.pid IS NULL', $calls[0]['sql']);
 		$this->assertSame(['thold_notify', 'child'], $calls[0]['params']);
+		$this->assertStringContainsString('(next_attempt IS NULL OR next_attempt <= NOW())', $calls[1]['sql']);
 		$this->assertStringContainsString('AND process_id = 0', $calls[1]['sql']);
 		$this->assertSame([4242], $calls[1]['params']);
 	}
@@ -681,7 +682,9 @@ public function testDeviceCommandAndGroupedMailComplete(): void {
 		putenv('THOLD_DEVICE_TEST');
 
 		$updates = array_values(array_filter(CactiStubs::$calls, static function ($call) {
-			return $call['fn'] === 'db_execute_prepared' && strpos($call['sql'], 'event_processed = 1') !== false;
+			return $call['fn'] === 'db_execute_prepared'
+				&& (strpos($call['sql'], 'event_processed = 1') !== false
+					|| strpos($call['sql'], 'attempt_count = CASE id') !== false);
 		}));
 
 		$this->assertCount(2, $updates);
@@ -690,7 +693,7 @@ public function testDeviceCommandAndGroupedMailComplete(): void {
 		$this->assertSame(128, strlen($updates[0]['params'][1]));
 		$this->assertSame(1, $updates[1]['params'][0]);
 		$this->assertSame(128, mb_strlen($updates[1]['params'][1], 'UTF-8'));
-		$this->assertSame(77, $updates[1]['params'][3]);
+		$this->assertSame([102, 77], array_slice($updates[1]['params'], -2));
 		$this->assertStringContainsString('AND process_id = ?', $updates[0]['sql']);
 		$this->assertStringContainsString('AND process_id = ?', $updates[1]['sql']);
 		$this->assertSame(3, $heartbeats);
@@ -723,8 +726,7 @@ public function testIndividualDeviceMailCompletionRequiresItsOwner(): void {
 
 		$this->assertStringContainsString('AND process_id = ?', $call['sql']);
 		$this->assertSame(128, mb_strlen($call['params'][1], 'UTF-8'));
-		$this->assertSame(104, $call['params'][3]);
-		$this->assertSame(77, $call['params'][4]);
+		$this->assertSame([104, 77], array_slice($call['params'], -2));
 	}
 
 	/**
@@ -780,8 +782,7 @@ public function testNonDeviceMailCompletionRequiresItsOwner(): void {
 
 		$this->assertStringContainsString('AND process_id = ?', $call['sql']);
 		$this->assertSame(128, mb_strlen($call['params'][1], 'UTF-8'));
-		$this->assertSame(105, $call['params'][3]);
-		$this->assertSame(77, $call['params'][4]);
+		$this->assertSame([105, 77], array_slice($call['params'], -2));
 	}
 
 	/**