diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs index b800830..cb4005d 100644 --- a/Helpers/BrowserDrivers.cs +++ b/Helpers/BrowserDrivers.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; +using System.Threading; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Edge; @@ -85,6 +86,21 @@ public static class BrowserDrivers private static readonly TimeSpan _sessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds); + /// + /// Attempts a browser session is given to start. A driver that + /// started the browser but could not reach it answers in about a + /// second and leaves nothing behind, and the next attempt usually + /// starts, so one such answer does not fail a test. + /// + public const int SessionStartAttempts = 3; + + /// + /// Seconds between one failed start of a session and the next + /// attempt, which leaves the machine a moment before it is asked + /// again. + /// + public const int SessionRetryDelaySeconds = 2; + /// /// Arguments every Chrome is given. --no-sandbox turns off Chrome's /// sandbox, without which Chrome exits as it starts on some runners, @@ -169,12 +185,12 @@ public static WebDriver CreateChrome(ChromeOptions options) ChromeService(driverPath), options, timeout))); if (driverPath != null) { - return new ChromeDriver( - ChromeService(driverPath), options, _sessionTimeout); + return Retried("Chrome", () => new ChromeDriver( + ChromeService(driverPath), options, _sessionTimeout)); } return Fetched( - () => new ChromeDriver( - ChromeService(null), options, _sessionTimeout), + () => Retried("Chrome", () => new ChromeDriver( + ChromeService(null), options, _sessionTimeout)), "Chrome", ChromeDriverVariable, ChromeDriverNames); } @@ -264,12 +280,12 @@ public static WebDriver CreateFirefox(FirefoxOptions options) FirefoxService(driverPath), options, timeout))); if (driverPath != null) { - return new FirefoxDriver( - FirefoxService(driverPath), options, _sessionTimeout); + return Retried("Firefox", () => new FirefoxDriver( + FirefoxService(driverPath), options, _sessionTimeout)); } return Fetched( - () => new FirefoxDriver( - FirefoxService(null), options, _sessionTimeout), + () => Retried("Firefox", () => new FirefoxDriver( + FirefoxService(null), options, _sessionTimeout)), "Firefox", GeckoDriverVariable, GeckoDriverNames); } @@ -304,12 +320,12 @@ public static WebDriver CreateEdge(EdgeOptions options) SetBinaryLocation(options, EdgeBinaryVariable, _edgeBinaryNames); if (driverPath != null) { - return new EdgeDriver( + return Retried("Edge", () => new EdgeDriver( EdgeDriverService.CreateDefaultService(driverPath), - options); + options)); } return Fetched( - () => new EdgeDriver(options), + () => Retried("Edge", () => new EdgeDriver(options)), "Edge", EdgeDriverVariable, EdgeDriverNames); } @@ -328,6 +344,61 @@ public static IReadOnlyDictionary FirstStarts } } + /// + /// Runs and, when the browser or its + /// driver did not start, runs it again up to + /// times, waiting + /// seconds between attempts. + /// What stops the last attempt travels on as it is, so a test + /// reports the error it would report without this, and each earlier + /// failure is written to the test's output. + /// + /// What is created. + /// Name of the browser, for the output. + /// Creates the session. + /// What gave. + public static T Retried(string browser, Func create) + { + if (browser == null) + { + throw new ArgumentNullException(nameof(browser)); + } + if (create == null) + { + throw new ArgumentNullException(nameof(create)); + } + for (var attempt = 1; ; attempt++) + { + try + { + return create(); + } + catch (Exception e) when ( + attempt < SessionStartAttempts && IsAStartFailure(e)) + { + Console.WriteLine( + $"{browser} did not start on attempt {attempt} of " + + $"{SessionStartAttempts}, so it is asked again in " + + $"{SessionRetryDelaySeconds} s. " + + FirstLine(e.Message)); + Thread.Sleep( + TimeSpan.FromSeconds(SessionRetryDelaySeconds)); + } + } + } + + /// + /// True when what stopped an attempt says the browser or its driver + /// did not start. Selenium reports a session it could not create as + /// an InvalidOperationException and a driver service that would not + /// run as a WebDriverException, and the one call this wraps raises + /// neither for any other reason. + /// + /// What stopped the attempt. + /// True when another attempt is worth making. + private static bool IsAStartFailure(Exception e) => + e is WebDriverException || e is InvalidOperationException; + /// /// Runs the first time a browser is asked /// for in this run, and never again, so that the browser's slow first diff --git a/Helpers/SessionRetryTests.cs b/Helpers/SessionRetryTests.cs new file mode 100644 index 0000000..495ac78 --- /dev/null +++ b/Helpers/SessionRetryTests.cs @@ -0,0 +1,142 @@ +using System; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Tests; + +/// +/// Checks that a browser session which fails to start is started again. +/// Nothing here starts a browser, so these run anywhere. +/// +[TestClass] +public class SessionRetryTests +{ + /// + /// The message Chrome's driver gives when the browser process starts but + /// its renderer never answers, which is the failure these tests exist + /// for. Selenium reports it as an InvalidOperationException. + /// + private const string RendererMessage = + "session not created from disconnected: unable to connect to " + + "renderer (SessionNotCreated)"; + + /// + /// A session that starts is handed back and started once. + /// + [TestMethod] + public void Retried_ReturnsWhatTheFirstAttemptGives() + { + var attempts = 0; + var session = BrowserDrivers.Retried("Chrome", () => + { + attempts++; + return "a session"; + }); + Assert.AreEqual("a session", session); + Assert.AreEqual(1, attempts); + } + + /// + /// A session the driver could not create is started again, which is the + /// renderer failure seen on the Windows runners. + /// + [TestMethod] + public void Retried_StartsAgainWhenTheSessionWasNotCreated() + { + var attempts = 0; + var session = BrowserDrivers.Retried("Chrome", () => + { + attempts++; + if (attempts == 1) + { + throw new InvalidOperationException(RendererMessage); + } + return "a session"; + }); + Assert.AreEqual("a session", session); + Assert.AreEqual(2, attempts); + } + + /// + /// A driver service that would not run is started again. Selenium + /// reports that one as a WebDriverException rather than an + /// InvalidOperationException. + /// + [TestMethod] + public void Retried_StartsAgainWhenTheDriverServiceWouldNotRun() + { + var attempts = 0; + var session = BrowserDrivers.Retried("Firefox", () => + { + attempts++; + if (attempts == 1) + { + throw new WebDriverException( + "Driver service process exited unexpectedly before " + + "initialization completed"); + } + return "a session"; + }); + Assert.AreEqual("a session", session); + Assert.AreEqual(2, attempts); + } + + /// + /// Every attempt is made, and the failure of the last one is thrown as + /// it stands, so a test reports the same error it reports without this. + /// + [TestMethod] + public void Retried_ThrowsTheLastFailureWhenNoAttemptStarts() + { + var attempts = 0; + var failure = Assert.ThrowsExactly( + () => BrowserDrivers.Retried("Chrome", () => + { + attempts++; + throw new InvalidOperationException( + RendererMessage + " on attempt " + attempts); + })); + Assert.AreEqual(BrowserDrivers.SessionStartAttempts, attempts); + Assert.AreEqual( + RendererMessage + " on attempt " + + BrowserDrivers.SessionStartAttempts, + failure.Message); + } + + /// + /// A fault in the test itself, rather than in starting the browser, is + /// reported at once and not started again. + /// + [TestMethod] + public void Retried_DoesNotStartAgainAfterAFaultInTheCaller() + { + var attempts = 0; + Assert.ThrowsExactly( + () => BrowserDrivers.Retried("Chrome", () => + { + attempts++; + throw new ArgumentException("the options are wrong"); + })); + Assert.AreEqual(1, attempts); + } + + /// + /// More than one attempt is allowed, and the wait between them is short + /// enough that a browser that will not start does not hold the run. + /// + [TestMethod] + public void SessionStartAttempts_AllowsMoreThanOneAttempt() + { + Assert.IsTrue( + BrowserDrivers.SessionStartAttempts > 1, + "A session must be allowed more than one attempt, or nothing " + + "is started again."); + Assert.IsTrue( + BrowserDrivers.SessionStartAttempts + * BrowserDrivers.SessionRetryDelaySeconds < 30, + "The attempts and the wait between them must add up to less " + + "than 30 s, so a browser that will not start fails the test " + + "rather than holding the run."); + } +} diff --git a/README.md b/README.md index 964b72b..b97c67f 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,12 @@ Select a subset with `--filter TestCategory=Contract` or start included, gives each command up to 300 seconds, because a start on a fresh or busy runner can take longer than the driver's default of 60 seconds. A session on a Selenium grid gets the same 300 seconds. The first - start's time is written to that test's output. Every Chrome is given + start's time is written to that test's output. A session that the browser + or its driver did not start is started again, up to three attempts with two + seconds between them, and each failed attempt is written to that test's + output, because a driver that starts the browser and then cannot reach it + answers in about a second and the next attempt usually starts. Every Chrome + is given `--no-sandbox` and `--disable-dev-shm-usage`, so a test does not add them itself, and the Chrome and Firefox drivers start quietly, with no start up banner, no verbose log and no console window. The 51Did tests are the