Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 82 additions & 11 deletions Helpers/BrowserDrivers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Edge;
Expand Down Expand Up @@ -85,6 +86,21 @@ public static class BrowserDrivers
private static readonly TimeSpan _sessionTimeout =
TimeSpan.FromSeconds(SessionTimeoutSeconds);

/// <summary>
/// Attempts a browser session is given to start. A driver that
/// started the browser but could not reach it answers in about a
/// second and leaves nothing behind, and the next attempt usually
/// starts, so one such answer does not fail a test.
/// </summary>
public const int SessionStartAttempts = 3;

/// <summary>
/// Seconds between one failed start of a session and the next
/// attempt, which leaves the machine a moment before it is asked
/// again.
/// </summary>
public const int SessionRetryDelaySeconds = 2;

/// <summary>
/// Arguments every Chrome is given. --no-sandbox turns off Chrome's
/// sandbox, without which Chrome exits as it starts on some runners,
Expand Down Expand Up @@ -169,12 +185,12 @@ public static WebDriver CreateChrome(ChromeOptions options)
ChromeService(driverPath), options, timeout)));
if (driverPath != null)
{
return new ChromeDriver(
ChromeService(driverPath), options, _sessionTimeout);
return Retried("Chrome", () => new ChromeDriver(
ChromeService(driverPath), options, _sessionTimeout));
}
return Fetched(
() => new ChromeDriver(
ChromeService(null), options, _sessionTimeout),
() => Retried("Chrome", () => new ChromeDriver(
ChromeService(null), options, _sessionTimeout)),
"Chrome", ChromeDriverVariable, ChromeDriverNames);
}

Expand Down Expand Up @@ -264,12 +280,12 @@ public static WebDriver CreateFirefox(FirefoxOptions options)
FirefoxService(driverPath), options, timeout)));
if (driverPath != null)
{
return new FirefoxDriver(
FirefoxService(driverPath), options, _sessionTimeout);
return Retried("Firefox", () => new FirefoxDriver(
FirefoxService(driverPath), options, _sessionTimeout));
}
return Fetched(
() => new FirefoxDriver(
FirefoxService(null), options, _sessionTimeout),
() => Retried("Firefox", () => new FirefoxDriver(
FirefoxService(null), options, _sessionTimeout)),
"Firefox", GeckoDriverVariable, GeckoDriverNames);
}

Expand Down Expand Up @@ -304,12 +320,12 @@ public static WebDriver CreateEdge(EdgeOptions options)
SetBinaryLocation(options, EdgeBinaryVariable, _edgeBinaryNames);
if (driverPath != null)
{
return new EdgeDriver(
return Retried("Edge", () => new EdgeDriver(
EdgeDriverService.CreateDefaultService(driverPath),
options);
options));
}
return Fetched(
() => new EdgeDriver(options),
() => Retried("Edge", () => new EdgeDriver(options)),
"Edge", EdgeDriverVariable, EdgeDriverNames);
}

Expand All @@ -328,6 +344,61 @@ public static IReadOnlyDictionary<string, string> FirstStarts
}
}

/// <summary>
/// Runs <paramref name="create"/> and, when the browser or its
/// driver did not start, runs it again up to
/// <see cref="SessionStartAttempts"/> times, waiting
/// <see cref="SessionRetryDelaySeconds"/> seconds between attempts.
/// What stops the last attempt travels on as it is, so a test
/// reports the error it would report without this, and each earlier
/// failure is written to the test's output.
/// </summary>
/// <typeparam name="T">What is created.</typeparam>
/// <param name="browser">Name of the browser, for the output.</param>
/// <param name="create">Creates the session.</param>
/// <returns>What <paramref name="create"/> gave.</returns>
public static T Retried<T>(string browser, Func<T> create)
{
if (browser == null)
{
throw new ArgumentNullException(nameof(browser));
}
if (create == null)
{
throw new ArgumentNullException(nameof(create));
}
for (var attempt = 1; ; attempt++)
{
try
{
return create();
}
catch (Exception e) when (
attempt < SessionStartAttempts && IsAStartFailure(e))
{
Console.WriteLine(
$"{browser} did not start on attempt {attempt} of "
+ $"{SessionStartAttempts}, so it is asked again in "
+ $"{SessionRetryDelaySeconds} s. "
+ FirstLine(e.Message));
Thread.Sleep(
TimeSpan.FromSeconds(SessionRetryDelaySeconds));
}
}
}

/// <summary>
/// True when what stopped an attempt says the browser or its driver
/// did not start. Selenium reports a session it could not create as
/// an InvalidOperationException and a driver service that would not
/// run as a WebDriverException, and the one call this wraps raises
/// neither for any other reason.
/// </summary>
/// <param name="e">What stopped the attempt.</param>
/// <returns>True when another attempt is worth making.</returns>
private static bool IsAStartFailure(Exception e) =>
e is WebDriverException || e is InvalidOperationException;

/// <summary>
/// Runs <paramref name="start"/> the first time a browser is asked
/// for in this run, and never again, so that the browser's slow first
Expand Down
142 changes: 142 additions & 0 deletions Helpers/SessionRetryTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
using System;
using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using OpenQA.Selenium;

namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests;

/// <summary>
/// Checks that a browser session which fails to start is started again.
/// Nothing here starts a browser, so these run anywhere.
/// </summary>
[TestClass]
public class SessionRetryTests
{
/// <summary>
/// The message Chrome's driver gives when the browser process starts but
/// its renderer never answers, which is the failure these tests exist
/// for. Selenium reports it as an InvalidOperationException.
/// </summary>
private const string RendererMessage =
"session not created from disconnected: unable to connect to "
+ "renderer (SessionNotCreated)";

/// <summary>
/// A session that starts is handed back and started once.
/// </summary>
[TestMethod]
public void Retried_ReturnsWhatTheFirstAttemptGives()
{
var attempts = 0;
var session = BrowserDrivers.Retried("Chrome", () =>
{
attempts++;
return "a session";
});
Assert.AreEqual("a session", session);
Assert.AreEqual(1, attempts);
}

/// <summary>
/// A session the driver could not create is started again, which is the
/// renderer failure seen on the Windows runners.
/// </summary>
[TestMethod]
public void Retried_StartsAgainWhenTheSessionWasNotCreated()
{
var attempts = 0;
var session = BrowserDrivers.Retried("Chrome", () =>
{
attempts++;
if (attempts == 1)
{
throw new InvalidOperationException(RendererMessage);
}
return "a session";
});
Assert.AreEqual("a session", session);
Assert.AreEqual(2, attempts);
}

/// <summary>
/// A driver service that would not run is started again. Selenium
/// reports that one as a WebDriverException rather than an
/// InvalidOperationException.
/// </summary>
[TestMethod]
public void Retried_StartsAgainWhenTheDriverServiceWouldNotRun()
{
var attempts = 0;
var session = BrowserDrivers.Retried("Firefox", () =>
{
attempts++;
if (attempts == 1)
{
throw new WebDriverException(
"Driver service process exited unexpectedly before "
+ "initialization completed");
}
return "a session";
});
Assert.AreEqual("a session", session);
Assert.AreEqual(2, attempts);
}

/// <summary>
/// Every attempt is made, and the failure of the last one is thrown as
/// it stands, so a test reports the same error it reports without this.
/// </summary>
[TestMethod]
public void Retried_ThrowsTheLastFailureWhenNoAttemptStarts()
{
var attempts = 0;
var failure = Assert.ThrowsExactly<InvalidOperationException>(
() => BrowserDrivers.Retried<string>("Chrome", () =>
{
attempts++;
throw new InvalidOperationException(
RendererMessage + " on attempt " + attempts);
}));
Assert.AreEqual(BrowserDrivers.SessionStartAttempts, attempts);
Assert.AreEqual(
RendererMessage + " on attempt "
+ BrowserDrivers.SessionStartAttempts,
failure.Message);
}

/// <summary>
/// A fault in the test itself, rather than in starting the browser, is
/// reported at once and not started again.
/// </summary>
[TestMethod]
public void Retried_DoesNotStartAgainAfterAFaultInTheCaller()
{
var attempts = 0;
Assert.ThrowsExactly<ArgumentException>(
() => BrowserDrivers.Retried<string>("Chrome", () =>
{
attempts++;
throw new ArgumentException("the options are wrong");
}));
Assert.AreEqual(1, attempts);
}

/// <summary>
/// More than one attempt is allowed, and the wait between them is short
/// enough that a browser that will not start does not hold the run.
/// </summary>
[TestMethod]
public void SessionStartAttempts_AllowsMoreThanOneAttempt()
{
Assert.IsTrue(

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / build-and-test

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / build-and-test

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-24.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-24.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-22.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-22.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-latest

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 131 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-latest

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)
BrowserDrivers.SessionStartAttempts > 1,
"A session must be allowed more than one attempt, or nothing "
+ "is started again.");
Assert.IsTrue(

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / build-and-test

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / build-and-test

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-24.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-24.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-22.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-22.04-arm

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-latest

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)

Check warning on line 135 in Helpers/SessionRetryTests.cs

View workflow job for this annotation

GitHub Actions / Browsers on ubuntu-latest

Review or remove the assertion as its condition is known to be always true (https://learn.microsoft.com/dotnet/core/testing/mstest-analyzers/mstest0032)
BrowserDrivers.SessionStartAttempts
* BrowserDrivers.SessionRetryDelaySeconds < 30,
"The attempts and the wait between them must add up to less "
+ "than 30 s, so a browser that will not start fails the test "
+ "rather than holding the run.");
}
}
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@ Select a subset with `--filter TestCategory=Contract` or
start included, gives each command up to 300 seconds, because a start on a
fresh or busy runner can take longer than the driver's default of 60
seconds. A session on a Selenium grid gets the same 300 seconds. The first
start's time is written to that test's output. Every Chrome is given
start's time is written to that test's output. A session that the browser
or its driver did not start is started again, up to three attempts with two
seconds between them, and each failed attempt is written to that test's
output, because a driver that starts the browser and then cannot reach it
answers in about a second and the next attempt usually starts. Every Chrome
is given
`--no-sandbox` and `--disable-dev-shm-usage`, so a test does not add them
itself, and the Chrome and Firefox drivers start quietly, with no start up
banner, no verbose log and no console window. The 51Did tests are the
Expand Down
Loading