diff --git a/examples/using.autodoc b/examples/using.autodoc index 8bf9ea4..9c559c7 100644 --- a/examples/using.autodoc +++ b/examples/using.autodoc @@ -75,7 +75,7 @@ true @EndExampleSession -In some cases we might want to return an explanation of why a test failed. Rather than returning `false` from `QC_Equal`, you can instead return a string. Returning a string is treated as a failure, and the string is printed out to the user. +In some cases we might want to return an explanation of why a test failed. Rather than returning `false` from a function tested by `QC_Check`, you can instead return a string. Returning a string is treated as a failure, and the string is printed out to the user. @Section Valid argument types for QuickCheck diff --git a/gap/digraphs.g b/gap/digraphs.g index c895bd9..0fb79d3 100644 --- a/gap/digraphs.g +++ b/gap/digraphs.g @@ -1 +1,7 @@ -QC_RegisterFilterGen(IsDigraph, {rg, limit} -> RandomDigraph(Random([1..limit]))); +# RandomDigraph cannot take a random source, so we build the edges from rg +QC_RegisterFilterGen(IsDigraph, function(rg, limit) + local n, p; + n := Random(rg, [1..limit]); + p := Random(rg, [0..100]); + return Digraph(List([1..n], i -> Filtered([1..n], j -> Random(rg, [1..100]) <= p))); +end); diff --git a/gap/generators.g b/gap/generators.g index ad29cb1..38a63e6 100644 --- a/gap/generators.g +++ b/gap/generators.g @@ -40,25 +40,25 @@ end); -QC_ListOf := function(object) +InstallGlobalFunction(QC_ListOf, function(object) return function(rg, limit) - return List([1..Random([0..limit])], {x} -> QC_MakeRandomArgument(object, rg, limit)); + return List([1..Random(rg, [0..limit])], {x} -> QC_MakeRandomArgument(object, rg, limit)); end; -end; +end); -QC_FixedLengthListOf := function(object, length) +InstallGlobalFunction(QC_FixedLengthListOf, function(object, length) return function(rg, limit) return List([1..length], {x} -> QC_MakeRandomArgument(object, rg, limit)); end; -end; +end); -QC_PairOf := {o} -> QC_FixedLengthListOf(o, 2); +InstallGlobalFunction(QC_PairOf, {o} -> QC_FixedLengthListOf(o, 2)); -QC_SetOf := function(object) +InstallGlobalFunction(QC_SetOf, function(object) return function(rg, limit) local set, targetsize, testlimit; - targetsize := Random([0..limit]); + targetsize := Random(rg, [0..limit]); set := []; # In case we have trouble filling the set, put a limit on number of tests testlimit := targetsize * limit + 10; @@ -68,13 +68,13 @@ QC_SetOf := function(object) od; return set; end; -end; +end); -QC_ElementOf := function(object) +InstallGlobalFunction(QC_ElementOf, function(object) return function(rg, limit) return Random(rg, object); end; -end; +end); QC_RegisterFilterGen(IsCyclic, {rg, limit} -> CyclicGroup(Random(rg, [1..limit]))); diff --git a/gap/quickcheck.gd b/gap/quickcheck.gd index b6ca462..e362982 100644 --- a/gap/quickcheck.gd +++ b/gap/quickcheck.gd @@ -49,18 +49,22 @@ DeclareGlobalFunction("QC_Check"); DeclareGlobalFunction("QC_CheckEqual"); #! @Description -#! Return the function called, and arguments given, last time a QuickCheck -#! test failed. +#! Return the function called, and arguments given, if the most recent call to +#! `QC_Check` or `QC_CheckEqual` failed on a particular input. #! #! Returns a record containing args (the arguments) and a function `func` #! (if `QC_Check` failed) or a list of functions `funcs` (if `QC_CheckEqual` failed). -#! Returns false if no test has failed. +#! Returns false otherwise. +#! @Arguments DeclareGlobalFunction("QC_LastFailure"); #! @Description -#! Rerun the last test which failed. This is most useful if a test in a '.tst' file -#! failed, as this will allow the test to enter the break loop. +#! Rerun the last test which failed, as given by . +#! This is most useful if a test in a '.tst' file failed, as this will allow the +#! test to enter the break loop. Returns fail if +#! returns false. +#! @Arguments DeclareGlobalFunction("QC_RerunLastFailure"); #! @Description @@ -68,9 +72,13 @@ DeclareGlobalFunction("QC_RerunLastFailure"); #! to set all options. #! #! Current options are: -#! * tests: Number of tests to run -#! * limit: The size of the largest object to create -#! * seed: Initial random seed +#! * tests: Number of tests to run (default 500) +#! * limit: The size of the largest object to create (default 9) +#! * ramp: Number of tests, including skipped ones, to run at each size +#! before increasing it (default 30) +#! * seed: Initial random seed (default 1) +#! * catchErrors: If true, an error in the tested function counts +#! as a failure; if false, it enters the break loop (default true) #! #! @Arguments config DeclareGlobalFunction("QC_SetConfig"); @@ -78,8 +86,71 @@ DeclareGlobalFunction("QC_SetConfig"); #! @Description #! Get the current global configuration for QuickCheck, as a record +#! @Arguments DeclareGlobalFunction("QC_GetConfig"); +DeclareCategory("IsQCSkip", IsObject); + +#! @Description +#! A function tested by or +#! can return QC_Skip if its arguments do not satisfy its requirements +#! (for example, if it needs an intransitive group, or an integer which is not prime). +#! Skipped tests do not count as failures or towards the number of tests. +#! To avoid infinite loops, if 100 times the requested number of tests +#! are skipped, the check stops and returns false. +DeclareGlobalName("QC_Skip"); + +#! @Description +#! Register gen as a generator for arguments described by the filter +#! filter. gen is called as gen(rs, limit), where rs is a +#! random source and limit a positive integer bounding the size of the value. +#! It is up to gen to decide how to interpret limit. +#! Smaller values of limit are used first, so simple inputs are tested +#! before complex ones. +#! Values not in filter are discarded and regenerated, with an error +#! after 100 attempts. This lets a generator serve a more specific filter, for +#! example an abelian permutation group when only a gen for permutation +#! groups has been installed. +#! @Arguments filter, gen +DeclareGlobalFunction("QC_RegisterFilterGen"); + +#! @Section Argument descriptions +#! +#! An argument description is either a filter with a registered generator +#! (see ), or a function gen(rs, limit). +#! The functions below build descriptions from other descriptions. + +#! @Description +#! Describes a list of between 0 and limit values described by desc. +#! @Arguments desc +#! @BeginExampleSession +#! gap> QC_Check([QC_ListOf(QC_PairOf(IsPosInt))], +#! > l -> ForAll(l, p -> Length(p) = 2 and ForAll(p, IsPosInt))); +#! true +#! @EndExampleSession +DeclareGlobalFunction("QC_ListOf"); + +#! @Description +#! Describes a list of len values described by desc. +#! @Arguments desc, len +DeclareGlobalFunction("QC_FixedLengthListOf"); + +#! @Description +#! Describes a list of length two, whose entries are described by desc. +#! @Arguments desc +DeclareGlobalFunction("QC_PairOf"); + +#! @Description +#! Describes a set of between 0 and limit values described by desc. +#! @Arguments desc +DeclareGlobalFunction("QC_SetOf"); + +#! @Description +#! Describes a random element of the collection coll. This ignores +#! limit. +#! @Arguments coll +DeclareGlobalFunction("QC_ElementOf"); + ## For private data BindGlobal("_QC", rec()); diff --git a/gap/quickcheck.gi b/gap/quickcheck.gi index 2608178..92747b9 100644 --- a/gap/quickcheck.gi +++ b/gap/quickcheck.gi @@ -4,14 +4,16 @@ # Implementations # -QC_Skip := "SKIP TEST"; +# A unique object, compared with IsIdenticalObj +BindGlobal("QC_Skip", Objectify(NewType(NewFamily("QCSkipFamily"), IsQCSkip and IsPositionalObjectRep), [])); +InstallMethod(PrintObj, [IsQCSkip], function(x) Print("QC_Skip"); end); # Store the generators we support DeclareOperation("QC_Filters", [IsObject]); -QC_RegisterFilterGen := function(filt, func) +InstallGlobalFunction(QC_RegisterFilterGen, function(filt, func) InstallMethod(QC_Filters, [filt], function(x) return func; end); -end; +end); InstallGlobalFunction(QC_MakeRandomArgument, @@ -73,12 +75,13 @@ InstallGlobalFunction(QC_SetConfig, InstallGlobalFunction(QC_GetConfig, {} -> ShallowCopy(_QC.defaultConfig)); -_QC.LastFailure := rec(); +_QC.LastFailure := false; _QC.Check := function(argtypes, func, configarg...) local testCount, skipCount, args, rg, call, ret, config, testSize, breakOnError, instream; config := _QC.fillConfig(configarg); + Unbind(_QC.Counterexample); rg := RandomSource(IsMersenneTwister, config.seed); @@ -114,22 +117,23 @@ _QC.Check := function(argtypes, func, configarg...) fi; CloseStream(instream); Unbind(_QC.Ret); - # We leave _QC.Args and _QC.Function, so they can be returned by QC_LastFailure if IsEmpty(ret) then PrintFormatted("Test {} of {} did not return a value\n", testCount, config.tests); Print(" Input: ", args, "\n"); + _QC.Counterexample := args; return false; fi; ret := ret[1]; - if ret = QC_Skip then + if IsIdenticalObj(ret, QC_Skip) then skipCount := skipCount + 1; elif ret <> true then PrintFormatted("Test {} of {} failed:\n", testCount, config.tests); Print(" Input: ", args,"\n"); Print(" Output: ", ret,"\n"); + _QC.Counterexample := args; return false; else testCount := testCount + 1; @@ -145,12 +149,12 @@ end; InstallGlobalFunction(QC_Check, function(argtypes, func, configarg...) - local ret, savefailure; - savefailure := _QC.LastFailure; - _QC.LastFailure := rec(func := func); + local ret; + _QC.LastFailure := false; ret := CallFuncList(_QC.Check, Concatenation([argtypes, func], configarg)); - if ret then - _QC.LastFailure := savefailure; + if IsBound(_QC.Counterexample) then + _QC.LastFailure := rec(func := func, args := _QC.Counterexample); + _QC.LastFailureRerun := func; fi; return ret; end); @@ -158,21 +162,19 @@ end); InstallGlobalFunction(QC_CheckEqual, function(argtypes, funcL, funcR, configarg...) - local funccheck, ret, savefailure; - - savefailure := _QC.LastFailure; - _QC.LastFailure := rec(funcs := [funcL, funcR]); + local funccheck, ret; funccheck := function(args...) local retL, retR; retL := CallFuncListWrap(funcL, StructuralCopy(args)); retR := CallFuncListWrap(funcR, args); - if retL = [QC_Skip] or retL = [QC_Skip] then + if (not IsEmpty(retL) and IsIdenticalObj(retL[1], QC_Skip)) or + (not IsEmpty(retR) and IsIdenticalObj(retR[1], QC_Skip)) then return QC_Skip; fi; if IsEmpty(retL) or IsEmpty(retR) then - return "A least one function did no return a value"; + return "At least one function did not return a value"; fi; retL := retL[1]; @@ -183,27 +185,22 @@ InstallGlobalFunction(QC_CheckEqual, fi; return StringFormatted("Return values differ: {} and {}", retL, retR); end; + _QC.LastFailure := false; ret := CallFuncList(_QC.Check, Concatenation([argtypes, funccheck], configarg)); - if ret then - _QC.LastFailure := savefailure; + if IsBound(_QC.Counterexample) then + _QC.LastFailure := rec(funcs := [funcL, funcR], args := _QC.Counterexample); + _QC.LastFailureRerun := funccheck; fi; return ret; end); InstallGlobalFunction(QC_LastFailure, - function() - if IsBound(_QC.Args) then - return rec(func := _QC.Function, args := _QC.Args); - else - return fail; - fi; -end); + {} -> _QC.LastFailure); InstallGlobalFunction(QC_RerunLastFailure, function() - if IsBound(_QC.Args) then - return CallFuncList(_QC.Function, _QC.Args); - else + if _QC.LastFailure = false then return fail; fi; + return CallFuncList(_QC.LastFailureRerun, StructuralCopy(_QC.LastFailure.args)); end); diff --git a/makedoc.g b/makedoc.g index ad40f0c..40e133a 100644 --- a/makedoc.g +++ b/makedoc.g @@ -3,8 +3,8 @@ # # This file is a script which compiles the package manual. # -if fail = LoadPackage("AutoDoc", "2018.02.14") then - Error("AutoDoc version 2018.02.14 or newer is required."); +if fail = LoadPackage("AutoDoc", "2026.03.17") then + Error("AutoDoc version 2026.03.17 or newer is required."); fi; AutoDoc( rec( scaffold := true, autodoc := true, extract_examples := true ) ); diff --git a/tst/digraphs.tst b/tst/digraphs.tst index 0f60e2e..125660a 100644 --- a/tst/digraphs.tst +++ b/tst/digraphs.tst @@ -7,4 +7,10 @@ true gap> QC_Check([IsDigraph], function(d) local a; a := DigraphDiameter(d); return a = fail or a <= DigraphNrVertices(d); end); true gap> QC_Check([IsDigraph], {d} -> IsList(DigraphMaximalClique(d))); -true \ No newline at end of file +true +gap> rs := RandomSource(IsMersenneTwister, 3);; +gap> l := List([1..20], i -> QC_MakeRandomArgument(IsDigraph, rs, 9));; +gap> Reset(GlobalMersenneTwister, 77);; Random([1..10]);; +gap> rs := RandomSource(IsMersenneTwister, 3);; +gap> l = List([1..20], i -> QC_MakeRandomArgument(IsDigraph, rs, 9)); +true diff --git a/tst/quickcheck01.tst b/tst/quickcheck01.tst index fe9eb91..58b8c47 100644 --- a/tst/quickcheck01.tst +++ b/tst/quickcheck01.tst @@ -17,25 +17,25 @@ gap> testFunc := function(a,b) gap> QC_Check([IsInt, IsInt], testFunc); true -# doc/_Chapter_Tutorial.xml:30-36 +# doc/_Chapter_Tutorial.xml:29-35 gap> QC_Check([IsPerm, IsPerm], testFunc); Test 60 of 500 failed: Input: [ (1,3), (1,3,2) ] Output: false false -# doc/_Chapter_Tutorial.xml:44-47 +# doc/_Chapter_Tutorial.xml:42-45 gap> QC_LastFailure(); rec( args := [ (1,3), (1,3,2) ], func := function( a, b ) ... end ) -# doc/_Chapter_Tutorial.xml:53-59 +# doc/_Chapter_Tutorial.xml:50-56 gap> slowIntersection := function(g,h) > return Group(Filtered(g, p -> p in h)); > end;; gap> QC_CheckEqual([IsPermGroup, IsPermGroup], Intersection, slowIntersection); true -# doc/_Chapter_Tutorial.xml:82-92 +# doc/_Chapter_Tutorial.xml:77-87 gap> checkDiv := function(a,b) > if b = 0 then return QC_Skip; fi; > return b*(a/b); @@ -46,7 +46,7 @@ gap> justA := function(a,b) gap> QC_CheckEqual([IsInt, IsInt], checkDiv, justA); true -# doc/_Chapter_Tutorial.xml:183-197 +# doc/_Chapter_Tutorial.xml:176-190 gap> func := function(x) > local a, s; > if x < 2 then @@ -61,7 +61,7 @@ gap> func := function(x) gap> QC_Check([IsInt], func); true -# doc/_Chapter_Tutorial.xml:214-221 +# doc/_Chapter_Tutorial.xml:205-212 gap> makePosInt := function(rs, limit) > return Random(rs, [1..limit]); > end;; @@ -69,7 +69,7 @@ gap> makePerm := function(rs, limit) > return Random(rs, SymmetricGroup(limit)); > end;; -# doc/_Chapter_Tutorial.xml:227-230 +# doc/_Chapter_Tutorial.xml:217-220 gap> QC_Check([makePosInt, makePerm, makePerm], {r,p1,p2} -> (r^p1)^p2 = r^(p1*p2)); true diff --git a/tst/quickcheck02.tst b/tst/quickcheck02.tst new file mode 100644 index 0000000..8d92676 --- /dev/null +++ b/tst/quickcheck02.tst @@ -0,0 +1,19 @@ +# quickcheck, chapter 3 +# +# DO NOT EDIT THIS FILE - EDIT EXAMPLES IN THE SOURCE INSTEAD! +# +# This file has been generated by AutoDoc. It contains examples extracted from +# the package documentation. Each example is preceded by a comment which gives +# the name of a GAPDoc XML file and a line range from which the example were +# taken. Note that the XML file in turn may have been generated by AutoDoc +# from some other input. +# +gap> START_TEST("quickcheck02.tst"); + +# doc/_Chapter_Functionality.xml:139-143 +gap> QC_Check([QC_ListOf(QC_PairOf(IsPosInt))], +> l -> ForAll(l, p -> Length(p) = 2 and ForAll(p, IsPosInt))); +true + +# +gap> STOP_TEST("quickcheck02.tst", 1); diff --git a/tst/tst-gap/failures.tst b/tst/tst-gap/failures.tst new file mode 100644 index 0000000..33d188d --- /dev/null +++ b/tst/tst-gap/failures.tst @@ -0,0 +1,78 @@ +gap> START_TEST("failures.tst"); + +# Either function in QC_CheckEqual may skip +gap> skipZero := function(x) if x = 0 then return QC_Skip; fi; return x; end;; +gap> QC_CheckEqual([IsInt], skipZero, x -> x); +true +gap> QC_CheckEqual([IsInt], x -> x, skipZero); +true +gap> QC_Check([IsInt], x -> "SKIP TEST", rec(tests := 5)); +Test 0 of 5 failed: + Input: [ 0 ] + Output: SKIP TEST +false +gap> QC_Skip; +QC_Skip +gap> QC_CheckEqual([IsInt], x -> 1, function(x) end, rec(tests := 5)); +Test 0 of 5 failed: + Input: [ 0 ] + Output: At least one function did not return a value +false + +# Generators only use the given random source +gap> rs := RandomSource(IsMersenneTwister, 3);; +gap> a := QC_ListOf(IsInt)(rs, 9);; b := QC_SetOf(IsInt)(rs, 9);; +gap> Reset(GlobalMersenneTwister, 77);; Random([1..10]);; +gap> rs := RandomSource(IsMersenneTwister, 3);; +gap> a = QC_ListOf(IsInt)(rs, 9); +true +gap> b = QC_SetOf(IsInt)(rs, 9); +true + +# QC_LastFailure records the unmodified arguments of the last failure +gap> f := function(l) Add(l, 1000); return Length(l) < 5; end;; +gap> QC_Check([QC_ListOf(IsPosInt)], f); +Test 90 of 500 failed: + Input: [ [ 1, 4, 3, 3 ] ] + Output: false +false +gap> QC_LastFailure().args; +[ [ 1, 4, 3, 3 ] ] +gap> QC_LastFailure().func = f; +true +gap> QC_RerunLastFailure(); +false +gap> QC_LastFailure().args; +[ [ 1, 4, 3, 3 ] ] + +# Every check clears it, including passing ones and those with too many skips +gap> QC_Check([IsInt], x -> true); +true +gap> QC_LastFailure(); +false +gap> QC_RerunLastFailure(); +fail +gap> QC_Check([QC_ListOf(IsPosInt)], f);; +Test 90 of 500 failed: + Input: [ [ 1, 4, 3, 3 ] ] + Output: false +gap> QC_Check([IsInt], x -> QC_Skip, rec(tests := 2)); +Too many tests skipped. Only managed 0 out of 2 tests +false +gap> QC_LastFailure(); +false + +# QC_CheckEqual records both functions +gap> sq := x -> x^2;; +gap> QC_CheckEqual([IsPosInt], IdFunc, sq); +Test 31 of 500 failed: + Input: [ 2 ] + Output: Return values differ: 2 and 4 +false +gap> QC_LastFailure() = rec(args := [ 2 ], funcs := [ IdFunc, sq ]); +true +gap> QC_RerunLastFailure(); +"Return values differ: 2 and 4" + +# +gap> STOP_TEST("failures.tst", 1);