diff --git a/app/context/physical_quantity/graphs/match.py b/app/context/physical_quantity/graphs/match.py index 5b6c928..657ab62 100644 --- a/app/context/physical_quantity/graphs/match.py +++ b/app/context/physical_quantity/graphs/match.py @@ -17,6 +17,7 @@ is_proportional, parse_criterion_sides, ) +from app.context.symbolic.numerical import matches_written_precision from app.context.tolerance import values_within_tolerance from app.evaluation_parameters import EvaluationParameters from app.feedback.physical_quantity import feedback_string_generators as physical_quantity_feedback_string_generators @@ -25,10 +26,14 @@ def _tolerances(parameters: EvaluationParameters) -> tuple[float, float]: - """`(rtol, atol)`, with `rtol` taken from the significant figures of the answer if no tolerance is given.""" + """ + `(rtol, atol)`, with `rtol` taken from the significant figures of the answer if no tolerance is given + and the response isn't checked for its written precision (`sig_figs`/`dp`). + """ rtol = parameters.parsing_parameters["rtol"] atol = parameters.parsing_parameters["atol"] - if parameters.atol == 0 and parameters.rtol == 0: + uses_precision = parameters.sig_figs is not None or parameters.dp is not None + if parameters.atol == 0 and parameters.rtol == 0 and not uses_precision: ans = parameters.reserved_expressions["answer"].quantity.value if ans is not None: rtol = compute_relative_tolerance_from_significant_decimals(ans.content_string()) @@ -82,18 +87,42 @@ def _within_tolerance(parameters: EvaluationParameters, lhs_string: str, rhs_str return values_within_tolerance(res, ans, rtol, atol * answer_unit_factor) +def _matches_written_precision(parameters: EvaluationParameters) -> bool: + """ + Whether the value of the response, as written, is the value of the answer to `sig_figs`/`dp`. + The values are compared in SI base units, so `dp` assumes the response has the unit of the answer. + """ + response_value = parameters.reserved_expressions["response"].quantity.value + if response_value is None: + return False + return matches_written_precision( + parameters.reserved_expressions["response"].standard_value, + parameters.reserved_expressions["answer"].standard_value, + response_value.content_string(), + parameters, + ) + + def _quantity_match(parameters: EvaluationParameters, label: str, sides: CriterionSides, rtol: float, atol: float) -> Evaluate: quantities = sides.quantities.items() + is_response_answer_criterion = {sides.lhs_string, sides.rhs_string} == {"response", "answer"} + uses_precision = parameters.sig_figs is not None or parameters.dp is not None def quantity_match(_response: Any) -> ReachedCriteria: missing_or_unexpected = _missing_or_unexpected(parameters, label, sides.lhs_string, sides.rhs_string) if missing_or_unexpected is not None: return missing_or_unexpected - substitutions = [(key, expr.standard_value) for (key, expr) in quantities] - value_match = is_equal(sides.lhs, sides.rhs, substitutions) - if value_match is False: - value_match = _within_tolerance(parameters, sides.lhs_string, sides.rhs_string, rtol, atol) + # sig_figs/dp replace the value comparison rather than being a fallback, + # since a value equal to the answer but written to the wrong precision is incorrect + value_match: bool | None + if uses_precision and is_response_answer_criterion: + value_match = _matches_written_precision(parameters) + else: + substitutions = [(key, expr.standard_value) for (key, expr) in quantities] + value_match = is_equal(sides.lhs, sides.rhs, substitutions) + if value_match is False: + value_match = _within_tolerance(parameters, sides.lhs_string, sides.rhs_string, rtol, atol) substitutions = [(key, expr.standard_unit) for (key, expr) in quantities] unit_match = is_equal(sides.lhs, sides.rhs, substitutions) diff --git a/app/context/symbolic/comparison.py b/app/context/symbolic/comparison.py index e59fee3..9741142 100644 --- a/app/context/symbolic/comparison.py +++ b/app/context/symbolic/comparison.py @@ -12,7 +12,7 @@ from lambdafeedback.slr_parsing import ExprNode from app.context.operators import COMPARISON_OPERATORS -from app.context.symbolic.numerical import compare_numerically +from app.context.symbolic.numerical import compare_numerically, matches_written_precision from app.evaluation_parameters import EvaluationParameters from app.types import SubstitutionList, Substitutions @@ -105,10 +105,33 @@ def is_inequality_or_boolean(expr: object) -> bool: return isinstance(expr, (Relational, BooleanAtom)) and not isinstance(expr, Equality) +def uses_written_precision(criterion: ExprNode, parameters: EvaluationParameters) -> bool: + """Whether `sig_figs`/`dp` is set and `criterion` compares the response and the answer directly.""" + if parameters.sig_figs is None and parameters.dp is None: + return False + lhs_string = criterion.children[0].content_string().strip() + rhs_string = criterion.children[1].content_string().strip() + return {lhs_string, rhs_string} == {"response", "answer"} + + +def check_written_precision(criterion: ExprNode, lhs_expr: Basic, rhs_expr: Basic, parameters: EvaluationParameters) -> bool: + """Whether the response, as written, is the answer to `sig_figs`/`dp`, for a response/answer `criterion`.""" + response_string = parameters.reserved_expressions_strings["learner"]["response"] + if criterion.children[0].content_string().strip() == "response": + return matches_written_precision(lhs_expr, rhs_expr, response_string, parameters) + return matches_written_precision(rhs_expr, lhs_expr, response_string, parameters) + + def check_equality( criterion: ExprNode, parameters: EvaluationParameters, local_substitutions: Substitutions = () ) -> bool | None: lhs_expr, rhs_expr = create_expressions_for_comparison(criterion, parameters, local_substitutions) + + # sig_figs/dp replace the comparison rather than being a fallback, + # since a response equal to the answer but written to the wrong precision is incorrect + if uses_written_precision(criterion, parameters): + return check_written_precision(criterion, lhs_expr, rhs_expr, parameters) + if isinstance(lhs_expr, Equality) and not isinstance(rhs_expr, Equality): return False elif not isinstance(lhs_expr, Equality) and isinstance(rhs_expr, Equality): diff --git a/app/context/symbolic/numerical.py b/app/context/symbolic/numerical.py index eaec05d..85d6b64 100644 --- a/app/context/symbolic/numerical.py +++ b/app/context/symbolic/numerical.py @@ -1,10 +1,91 @@ from __future__ import annotations +import math + from sympy import Basic, pi, N +from lambdafeedback.expression_parsing import read_written_number, round_to_decimal_places, round_to_significant_figures + from app.context.tolerance import values_within_tolerance from app.evaluation_parameters import EvaluationParameters + +# REMARK: 'pi' should be a reserved symbol but it is sometimes not treated as one, possibly because of input symbols. +# Replacing it by name fixes the issue but a more robust solution should be found for cases where there +# are other reserved symbols. +def replace_pi(expr: Basic) -> Basic: + pi_symbol = pi + for s in expr.free_symbols: + if str(s) == 'pi': + pi_symbol = s + return expr.subs(pi_symbol, float(pi)) + + +def _rounded_values_equal(rounded_response: float, rounded_answer: float) -> bool: + # Allows for the last bit of the rounded float, e.g. when 0.1 + 0.2 is rounded + return abs(rounded_response - rounded_answer) <= math.ulp(abs(rounded_answer)) + + +def sig_figs_match(response_string: str, response_value: float, answer_value: float, sig_figs: int) -> bool: + """ + Whether `response_string`, as written, is the answer to exactly `sig_figs` significant figures: + it is a plain number, its value rounds to the same value as the answer, and it is written to `sig_figs` + significant figures (`response_value`, a float, has lost the trailing zeros that count). + A zero response is not checked for precision. + """ + written = read_written_number(response_string) + if written is None: + return False + + rounded_answer = round_to_significant_figures(answer_value, sig_figs) + rounded_response = round_to_significant_figures(response_value, sig_figs) + numeric_correct = _rounded_values_equal(rounded_response, rounded_answer) + + precision_correct = response_value == 0 or written.significant_figures == sig_figs + + return numeric_correct and precision_correct + + +def decimal_places_match(response_string: str, response_value: float, answer_value: float, decimal_places: int) -> bool: + """ + Whether `response_string`, as written, is the answer to exactly `decimal_places` decimal places: + it is a plain number, its value rounds to the same value as the answer, and it is written to + `decimal_places` decimal places. + """ + written = read_written_number(response_string) + if written is None: + return False + + rounded_answer = float(round_to_decimal_places(answer_value, decimal_places)) + rounded_response = float(round_to_decimal_places(response_value, decimal_places)) + numeric_correct = _rounded_values_equal(rounded_response, rounded_answer) + + # Unlike sig_figs_match, a zero response is checked too: "0" has 0 decimal places and "0.00" has 2 + precision_correct = written.decimal_places == decimal_places + + return numeric_correct and precision_correct + + +def matches_written_precision( + response_expr: Basic, answer_expr: Basic, response_string: str, parameters: EvaluationParameters +) -> bool: + """ + Whether the response, written as `response_string`, is the answer to the `sig_figs` significant figures + or `dp` decimal places in `parameters` (one of them is set). False if either side is not a number. + """ + try: + response_value = float(N(replace_pi(response_expr))) + answer_value = float(N(replace_pi(answer_expr))) + except TypeError: + return False + + if parameters.sig_figs is not None: + return sig_figs_match(response_string, response_value, answer_value, parameters.sig_figs) + elif parameters.dp is not None: + return decimal_places_match(response_string, response_value, answer_value, parameters.dp) + return False + + def compare_numerically(lhs_expr: Basic, rhs_expr: Basic, parameters: EvaluationParameters) -> bool: """ Compares `lhs_expr` and `rhs_expr` numerically using the `atol` and `rtol` parameters. @@ -17,15 +98,6 @@ def compare_numerically(lhs_expr: Basic, rhs_expr: Basic, parameters: Evaluation or float(parameters.atol) > 0): return False - # REMARK: 'pi' should be a reserved symbol but it is sometimes not treated as one, possibly because of input symbols. - # The two lines below this comments fixes the issue but a more robust solution should be found for cases where there - # are other reserved symbols. - def replace_pi(expr: Basic) -> Basic: - pi_symbol = pi - for s in expr.free_symbols: - if str(s) == 'pi': - pi_symbol = s - return expr.subs(pi_symbol, float(pi)) # NOTE: This code assumes that the left hand side is the response and the right hand side is the answer # Separates LHS and RHS, parses and evaluates them res = N(replace_pi(lhs_expr)) diff --git a/app/context/tolerance.py b/app/context/tolerance.py index 11bcc16..56eaa0c 100644 --- a/app/context/tolerance.py +++ b/app/context/tolerance.py @@ -22,3 +22,13 @@ def values_within_tolerance(res: Basic, ans: Basic, rtol: float, atol: float) -> return bool(abs(float((ans - res) / ans)) <= rtol) except (TypeError, ZeroDivisionError): return False + + +def relative_tolerance_from_sig_figs(sig_figs: int) -> float: + """The relative tolerance for agreement to `sig_figs` significant figures, 5*10**(-sig_figs).""" + return 5 * 10 ** (-sig_figs) + + +def absolute_tolerance_from_decimal_places(decimal_places: int) -> float: + """The absolute tolerance for agreement to `decimal_places` decimal places, 0.5*10**(-decimal_places).""" + return 0.5 * 10 ** (-decimal_places) diff --git a/app/docs/dev.md b/app/docs/dev.md index 2b6c2bc..df099b1 100644 --- a/app/docs/dev.md +++ b/app/docs/dev.md @@ -190,6 +190,18 @@ along the the following base tokens: **TODO** Describe shared default parameters +##### Significant figures and decimal places + +The `significant_figures`/`sig_figs`, `significant_figures_tolerance`/`sig_figs_tol`, `decimal_places`/`dp` and `decimal_places_tolerance`/`dp_tol` parameters (see the user docs) are read in `_build_parameters` (`app/evaluation.py`): `_apply_precision_aliases` copies the long names to the short ones and `_apply_precision_parameters` checks them. At most one of the four, or of `atol`/`rtol`, can be given: a conflict, or a count below the minimum (1 for `sig_figs`/`sig_figs_tol`, 0 for `dp`/`dp_tol`), raises a `ValueError`, and a count that isn't an `int` (a `bool` isn't one) raises a `TypeError`. + +The tolerance forms need no code in the contexts: `sig_figs_tol` sets `rtol` to `relative_tolerance_from_sig_figs(n)` (`5*10**(-n)`) and `dp_tol` sets `atol` to `absolute_tolerance_from_decimal_places(n)` (`0.5*10**(-n)`), both in `app/context/tolerance.py`, before the parsing parameters are made, so the usual `atol`/`rtol` handling applies. Unlike `compute_relative_tolerance_from_significant_decimals` there is no `DEFAULT_SIGNIFICANT_FIGURES` floor. + +`sig_figs` and `dp` are passed on as `EvaluationParameters.sig_figs`/`dp`. They check the precision the response is *written* to, so they need the response as a string: a parsed float loses trailing zeros (`92`, `92.0` and `92.00` are equal once parsed). `matches_written_precision` (`app/context/symbolic/numerical.py`) evaluates both sides numerically and calls `sig_figs_match` or `decimal_places_match`, which read the string with `read_written_number` from `expression_parsing`, compare the values rounded with `round_to_significant_figures`/`round_to_decimal_places` (within `math.ulp` of the rounded answer) and compare the written precision. `sig_figs_match` doesn't check the precision of a zero response, since the significant figures of zero aren't well defined; `decimal_places_match` does (`0` has 0 decimal places and `0.00` has 2). + +Both contexts only use them for a criterion that compares `response` and `answer` directly, and there they replace the comparison rather than being a fallback, since a response equal to the answer but written to the wrong precision is incorrect: +- `symbolic`: at the top of `check_equality` (`app/context/symbolic/comparison.py`), with the preprocessed response string from `reserved_expressions_strings`. +- `physical_quantity`: in place of the value comparison in `_quantity_match` (`app/context/physical_quantity/graphs/match.py`), with the value as written in the response and the SI standard values; the units are compared as usual. `_tolerances` skips the tolerance taken from the answer's significant figures when either is set. + ## Feedback and tag generation - Generate feedback procedures from criteria, each procedure return a boolean that indicates whether the corresponding criterion is satisfied or not, a string intended to be shown to the student, and a list of tags indicating what was found when checking the criteria diff --git a/app/docs/user.md b/app/docs/user.md index 3d2a5b6..b965557 100644 --- a/app/docs/user.md +++ b/app/docs/user.md @@ -8,7 +8,7 @@ Note that this function is designed to handle comparisons of mathematical expres ### Optional parameters -There are 15 optional parameters that can be set: `absolute_tolerance`, `complexNumbers`, `convention`, `criteria`, `elementary_functions`, `feedback_for_incorrect_response`, `multiple_answers_criteria`, `physical_quantity`, `plus_minus`/`minus_plus`, `rtol`, `specialFunctions`, `strict_syntax`, `strictness`, `symbol_assumptions`. +There are 19 optional parameters that can be set: `absolute_tolerance`, `complexNumbers`, `convention`, `criteria`, `decimal_places`, `decimal_places_tolerance`, `elementary_functions`, `feedback_for_incorrect_response`, `multiple_answers_criteria`, `physical_quantity`, `plus_minus`/`minus_plus`, `rtol`, `significant_figures`, `significant_figures_tolerance`, `specialFunctions`, `strict_syntax`, `strictness`, `symbol_assumptions`. #### `absolute_tolerance` (`atol`) Sets the absolute tolerance, $e_a$, i.e. if the answer, $x$, and response, $\tilde{x}$, are numerical values then the response is considered equal to the answer if $|x-\tilde{x}| \leq e_a$. If `relative_tolerance`, $e_r$, is also set, the two combine into $|x-\tilde{x}| \leq e_a + e_r|x|$ (the same rule as `numpy.isclose`). For physical quantities, $e_a$ is in the units of the answer. By default `absolute_tolerance` is set to `0`, which means the comparison will be done with as high accuracy as possible. If either the answer or the response aren't numerical expressions this parameter is ignored. @@ -76,9 +76,47 @@ When `physical_quantity` the evaluation function will generate feedback based on **TODO:** Generate new flowchart for updated physical quantity feedback generation procedure. +#### `decimal_places` (`dp`) + +Checks the response against the answer to a fixed number of decimal places, both for numerical correctness and for the precision the response was actually *written* to. It only applies to a plain numeric response (or, when `physical_quantity` is `true`, a numeric value with units) being compared directly against the answer — it is ignored for any other kind of criterion. This is the decimal-place counterpart of `significant_figures`. + +For example, with an answer of `3.14159` and `decimal_places` set to `2`: the response `3.14` is accepted (correct value, written to 2 decimal places). `3.1` is rejected for having too few decimal places, and `3.142` is rejected for having too many — even though both are numerically close to the answer. + +Decimal places are counted as written, after accounting for any scientific-notation exponent (`5.0e-3` is written to 4 decimal places, `5.02e4` to 0). `decimal_places` may be `0`, which requires the response to be written as a whole number. + +When `physical_quantity` is `true`, the numerical comparison is performed at the answer's scale (its SI value), so `decimal_places` is intended for a response given in the same unit as the answer; a response in a different unit is compared after conversion and its written decimal-place count may then not correspond to the answer's scale. `significant_figures`, being scale-invariant, does not have this restriction. + +`decimal_places` cannot be combined with `atol`/`absolute_tolerance`, `rtol`/`relative_tolerance`, `significant_figures`/`significant_figures_tolerance` or `decimal_places_tolerance` — setting more than one of them will raise an error. Like `significant_figures`, a `decimal_places` failure produces the same generic feedback as any other incorrect response; it does not distinguish "wrong value" from "wrong precision" from "not a number". + +#### `decimal_places_tolerance` (`dp_tol`) + +Uses a decimal-place count to derive a *numerical tolerance*, i.e. it accepts the response if it agrees with the answer to the given number of decimal places. With `decimal_places_tolerance` set to $n$ the response, $\tilde{x}$, is considered equal to the answer, $x$, when $|x-\tilde{x}| \leq 0.5\cdot10^{-n}$ (an *absolute* tolerance, unlike `significant_figures_tolerance` which is relative, because a decimal-place count is scale-dependent). It is implemented by setting `absolute_tolerance` to $0.5\cdot10^{-n}$. Like `atol`/`rtol` it only applies to a plain numeric response (or, when `physical_quantity` is `true`, a numeric value with units) being compared directly against the answer. + +Unlike `decimal_places`, the precision the response was *written* to is not checked — only its value matters. For example, with an answer of `3.14159` and `decimal_places_tolerance` set to `2` (so the tolerance is $0.005$): the responses `3.1416`, `3.14` and `3.14000` are all accepted, while `3.1` is rejected for being outside the tolerance. + +`decimal_places_tolerance` cannot be combined with `decimal_places`/`dp`, `significant_figures`/`significant_figures_tolerance`, `atol`/`absolute_tolerance` or `rtol`/`relative_tolerance` — setting more than one of them will raise an error. + #### `relative_tolerance` (`rtol`) Sets the relative tolerance, $e_r$, i.e. if the answer, $x$, and response, $\tilde{x}$, are numerical values then the response is considered equal to the answer if $\left|\frac{x-\tilde{x}}{x}\right| \leq e_r$. If `absolute_tolerance` is also set, see above for how the two combine. By default `relative_tolerance` is set to `0`, which means the comparison will be done with as high accuracy as possible. If either the answer or the response aren't numerical expressions this parameter is ignored. +#### `significant_figures` (`sig_figs`) + +Checks the response against the answer to a fixed number of significant figures, both for numerical correctness and for the precision the response was actually *written* to. It only applies to a plain numeric response (or, when `physical_quantity` is `true`, a numeric value with units) being compared directly against the answer — it is ignored for any other kind of criterion. + +For example, with an answer of `3.14159` and `significant_figures` set to `3`: the response `3.14` is accepted (correct value, written to 3 significant figures). `3.1` is rejected for having too few significant figures, and `3.14159` is rejected for having too many — even though both are numerically close to the answer. + +Significant figures are counted as written: leading zeros are never significant (`0.0032` has 2), trailing zeros after a decimal point are always significant (`92.00` has 4), and trailing zeros in a whole number are only significant if a decimal point is explicitly written (`540` has 2, but `540.` has 3). + +`significant_figures` cannot be combined with `atol`/`absolute_tolerance`, `rtol`/`relative_tolerance` or `decimal_places`/`decimal_places_tolerance` — setting more than one of them will raise an error. Unlike those tolerance parameters, a `significant_figures` failure produces the same generic feedback as any other incorrect response; it does not distinguish "wrong value" from "wrong precision" from "not a number". + +#### `significant_figures_tolerance` (`sig_figs_tol`) + +Uses a significant-figure count to derive a *numerical tolerance*, i.e. it accepts the response if it agrees with the answer to the given number of significant figures. With `significant_figures_tolerance` set to $n$ the response, $\tilde{x}$, is considered equal to the answer, $x$, when $\left|\frac{x-\tilde{x}}{x}\right| \leq 5\cdot10^{-n}$. Like `atol`/`rtol` it only applies to a plain numeric response (or, when `physical_quantity` is `true`, a numeric value with units) being compared directly against the answer. + +Unlike `significant_figures`, the precision the response was *written* to is not checked — only its value matters. For example, with an answer of `3.14159` and `significant_figures_tolerance` set to `3` (so the tolerance is $5\cdot10^{-3}$): the responses `3.1416`, `3.14` and `3.14159` are all accepted, while `3.1` is rejected for being outside the tolerance. + +`significant_figures_tolerance` cannot be combined with `significant_figures`/`sig_figs`, `decimal_places`/`decimal_places_tolerance`, `atol`/`absolute_tolerance` or `rtol`/`relative_tolerance` — setting more than one of them will raise an error. + #### `strictness` Controls the conventions used when parsing physical quantities. diff --git a/app/evaluation.py b/app/evaluation.py index df7cf94..7bbc099 100644 --- a/app/evaluation.py +++ b/app/evaluation.py @@ -24,6 +24,7 @@ from app.context.physical_quantity import context as quantity_context from app.context.symbolic import context as symbolic_context from app.context.symbolic.comparison import is_inequality_or_boolean +from app.context.tolerance import absolute_tolerance_from_decimal_places, relative_tolerance_from_sig_figs from app.feedback.symbolic import feedback_generators as symbolic_feedback_string_generators, internal_feedback from app.feedback.tags import FeedbackTag, FeedbackTagName from app.evaluation_parameters import EvaluationParameters @@ -280,6 +281,83 @@ def _apply_tolerance_aliases(params: dict[str, Any]) -> None: params["atol"] = params["absolute_tolerance"] +def _raise_unless_count(params: dict[str, Any], key: str, names: str, minimum: int) -> None: + """ + Raises a TaskConfigurationError if `params[key]` is not an integer (a bool is not one), + or if it is less than `minimum`, which is 0 or 1. + """ + if minimum == 1: + message = f"{names} must be a positive integer." + else: + message = f"{names} must be a non-negative integer." + + value = params[key] + if not isinstance(value, int) or isinstance(value, bool): + raise TaskConfigurationError(message) + if value < minimum: + raise TaskConfigurationError(message) + + +def _apply_precision_aliases(params: dict[str, Any]) -> None: + """ + Copies `significant_figures`, `significant_figures_tolerance`, `decimal_places` and `decimal_places_tolerance` + to `sig_figs`, `sig_figs_tol`, `dp` and `dp_tol` in `params`. + """ + if "significant_figures" in params: + params["sig_figs"] = params["significant_figures"] + + if "significant_figures_tolerance" in params: + params["sig_figs_tol"] = params["significant_figures_tolerance"] + + if "decimal_places" in params: + params["dp"] = params["decimal_places"] + + if "decimal_places_tolerance" in params: + params["dp_tol"] = params["decimal_places_tolerance"] + + +def _apply_precision_parameters(params: dict[str, Any]) -> None: + """ + Checks `sig_figs`, `sig_figs_tol`, `dp` and `dp_tol` in `params`: at most one of them, or of `atol`/`rtol`, + can be given. `sig_figs_tol` and `dp_tol` are applied by setting `rtol` and `atol`. + """ + uses_tolerance = "rtol" in params or "atol" in params + + if "sig_figs" in params: + if uses_tolerance or "sig_figs_tol" in params or "dp" in params or "dp_tol" in params: + raise TaskConfigurationError( + "`sig_figs`/`significant_figures` cannot be used together with `atol`/`rtol`, " + "`significant_figures_tolerance` or `decimal_places`/`decimal_places_tolerance`." + ) + _raise_unless_count(params, "sig_figs", "`sig_figs`/`significant_figures`", 1) + + if "sig_figs_tol" in params: + if uses_tolerance or "dp" in params or "dp_tol" in params: + raise TaskConfigurationError( + "`sig_figs_tol`/`significant_figures_tolerance` cannot be used together with `atol`/`rtol`, " + "`significant_figures` or `decimal_places`/`decimal_places_tolerance`." + ) + _raise_unless_count(params, "sig_figs_tol", "`sig_figs_tol`/`significant_figures_tolerance`", 1) + params["rtol"] = relative_tolerance_from_sig_figs(params["sig_figs_tol"]) + + if "dp" in params: + if uses_tolerance or "dp_tol" in params: + raise TaskConfigurationError( + "`dp`/`decimal_places` cannot be used together with `atol`/`rtol`, " + "`significant_figures`/`significant_figures_tolerance` or `decimal_places_tolerance`." + ) + _raise_unless_count(params, "dp", "`dp`/`decimal_places`", 0) + + if "dp_tol" in params: + if uses_tolerance: + raise TaskConfigurationError( + "`dp_tol`/`decimal_places_tolerance` cannot be used together with `atol`/`rtol`, " + "`significant_figures`/`significant_figures_tolerance` or `decimal_places`." + ) + _raise_unless_count(params, "dp_tol", "`dp_tol`/`decimal_places_tolerance`", 0) + params["atol"] = absolute_tolerance_from_decimal_places(params["dp_tol"]) + + def _build_parameters(context: Context, params: dict[str, Any], response: str, answer: str) -> dict[str, Any]: """ A copy of `params` completed with the context's defaults, the reserved expressions, @@ -288,6 +366,8 @@ def _build_parameters(context: Context, params: dict[str, Any], response: str, a parameters = deepcopy(params) # Applied to the copy, so the caller's `params` keep only what they set _apply_tolerance_aliases(parameters) + _apply_precision_aliases(parameters) + _apply_precision_parameters(parameters) reserved_expressions_strings = { "learner": { @@ -380,6 +460,8 @@ def _build_evaluation_parameters( numerical=parameters.get("numerical", False), atol=parameters.get("atol", 0), rtol=parameters.get("rtol", 0), + sig_figs=parameters.get("sig_figs"), + dp=parameters.get("dp"), custom_feedback=parameters.get("custom_feedback", {}), ) diff --git a/app/evaluation_parameters.py b/app/evaluation_parameters.py index 50e945f..90dd1a0 100644 --- a/app/evaluation_parameters.py +++ b/app/evaluation_parameters.py @@ -32,5 +32,8 @@ class EvaluationParameters: # The tolerances as given in the request (0 if not given) atol: float rtol: float + # The number of significant figures or decimal places the response must be written to (None if not given) + sig_figs: int | None + dp: int | None # Custom feedback strings by tag custom_feedback: Mapping[str, Any] diff --git a/app/tests/physical_quantity_evaluation_test.py b/app/tests/physical_quantity_evaluation_test.py index f8a162b..37a33c3 100644 --- a/app/tests/physical_quantity_evaluation_test.py +++ b/app/tests/physical_quantity_evaluation_test.py @@ -4,7 +4,7 @@ # Import necessary data and reference cases for tests from app import evaluation from app.context.physical_quantity.comparison import is_proportional -from app.evaluation import evaluation_function +from app.evaluation import TaskConfigurationError, evaluation_function from app.tests.physical_quantity_test_cases import slr_strict_si_syntax_test_cases, slr_natural_si_syntax_test_cases from lambdafeedback.physical_quantities import ( COMMON_UNITS, @@ -712,3 +712,208 @@ def test_greek_letter_units(self, ans, res): } result = evaluation_function(res, ans, params) assert result["is_correct"] is True + + +class TestSigFigs: + + base_params = { + "strict_syntax": False, + "physical_quantity": True, + "units_string": "SI", + "strictness": "natural", + } + + @pytest.mark.parametrize( + "description,response,answer,sig_figs,outcome", + [ + ("Correct value and precision, with units", "92.00 m", "92 m", 4, True), + ("Numerically equal but wrong precision", "92.00 m", "92 m", 2, False), + ("Wrong value", "91.00 m", "92 m", 4, False), + ("Unit mismatch fails regardless of sig figs", "92 s", "92 m", 2, False), + ("No units, plain numeric", "3.14", "3.14159", 3, True), + ("Non-numeric response", "two m", "92 m", 2, False), + ] + ) + def test_sig_figs(self, description, response, answer, sig_figs, outcome): + params = dict(self.base_params, sig_figs=sig_figs) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_significant_figures_alias(self): + params = dict(self.base_params, significant_figures=4) + result = evaluation_function("92.00 m", "92 m", params) + assert result["is_correct"] is True + + def test_sig_figs_mutually_exclusive_with_atol(self): + params = dict(self.base_params, sig_figs=4, atol=0.1) + with pytest.raises(TaskConfigurationError): + evaluation_function("92.00 m", "92 m", params) + + @pytest.mark.parametrize("sig_figs", [0, -1, 3.5, True]) + def test_sig_figs_invalid_value_raises(self, sig_figs): + params = dict(self.base_params, sig_figs=sig_figs) + with pytest.raises(TaskConfigurationError): + evaluation_function("92.00 m", "92 m", params) + + @pytest.mark.parametrize( + "description,response,answer,sig_figs,outcome", + [ + ("Different unit, correct value and precision", "1.000 mile", "1609.344 m", 4, True), + ("Different unit the other way round", "1609 m", "1 mile", 4, True), + ("Correct value, but too few sig figs as written in its own unit", "1 mile", "1609.344 m", 4, False), + ("Precision must match exactly, not just be at least sig_figs", "1.0000 mile", "1609.344 m", 4, False), + ("Third unit entirely, correct value and precision", "1.6093 km", "1 mile", 5, True), + ("Dimension mismatch across unit systems", "1 mile", "1 kg", 4, False), + ] + ) + def test_sig_figs_across_units(self, description, response, answer, sig_figs, outcome): + params = dict(self.base_params, units_string="SI common imperial", sig_figs=sig_figs) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + +class TestSigFigsTolerance: + + base_params = { + "strict_syntax": False, + "physical_quantity": True, + "units_string": "SI", + "strictness": "natural", + } + + @pytest.mark.parametrize( + "description,response,answer,sig_figs_tol,outcome", + [ + ("Exact value, written precision ignored", "92.00 m", "92 m", 2, True), + ("Within tolerance", "91 m", "92 m", 2, True), + ("Outside tolerance", "99 m", "92 m", 2, False), + ("Tighter tolerance rejects", "92.5 m", "92 m", 4, False), + ("Unit mismatch fails regardless", "92 s", "92 m", 2, False), + ("No units, plain numeric within tolerance", "3.1416 m", "3.14159 m", 3, True), + ("Non-numeric response", "two m", "92 m", 2, False), + ] + ) + def test_sig_figs_tolerance(self, description, response, answer, sig_figs_tol, outcome): + params = dict(self.base_params, significant_figures_tolerance=sig_figs_tol) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_sig_figs_tol_alias(self): + params = dict(self.base_params, sig_figs_tol=2) + result = evaluation_function("91 m", "92 m", params) + assert result["is_correct"] is True + + @pytest.mark.parametrize("conflicting", [{"atol": 0.1}, {"rtol": 0.1}, {"significant_figures": 2}]) + def test_sig_figs_tolerance_mutually_exclusive(self, conflicting): + params = dict(self.base_params, significant_figures_tolerance=2, **conflicting) + with pytest.raises(TaskConfigurationError): + evaluation_function("92 m", "92 m", params) + + @pytest.mark.parametrize("sig_figs_tol", [0, -1, 3.5, True]) + def test_sig_figs_tolerance_invalid_value_raises(self, sig_figs_tol): + params = dict(self.base_params, significant_figures_tolerance=sig_figs_tol) + with pytest.raises(TaskConfigurationError): + evaluation_function("92 m", "92 m", params) + + @pytest.mark.parametrize( + "description,response,answer,sig_figs_tol,outcome", + [ + ("Different unit, within tolerance", "1609 m", "1 mile", 3, True), + ("Different unit the other way round", "1.000 mile", "1609.344 m", 4, True), + ("Different unit, outside tolerance", "1600 m", "1 mile", 4, False), + ("Third unit entirely, within tolerance", "1.6093 km", "1 mile", 4, True), + ("Dimension mismatch across unit systems", "1 mile", "1 kg", 4, False), + ] + ) + def test_sig_figs_tolerance_across_units(self, description, response, answer, sig_figs_tol, outcome): + params = dict(self.base_params, units_string="SI common imperial", significant_figures_tolerance=sig_figs_tol) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + +class TestDecimalPlaces: + + base_params = { + "strict_syntax": False, + "physical_quantity": True, + "units_string": "SI", + "strictness": "natural", + } + + @pytest.mark.parametrize( + "description,response,answer,decimal_places,outcome", + [ + ("Correct value and precision, with units", "92.00 m", "92 m", 2, True), + ("Numerically equal but too few decimal places", "92.0 m", "92 m", 2, False), + ("Numerically equal but too many decimal places", "92.000 m", "92 m", 2, False), + ("Wrong value", "91.00 m", "92 m", 2, False), + ("Unit mismatch fails regardless of decimal places", "92.00 s", "92 m", 2, False), + ("decimal_places=0 requires a whole number", "92 m", "92 m", 0, True), + ("No units, plain numeric", "3.14", "3.14159", 2, True), + ("Non-numeric response", "two m", "92 m", 2, False), + ] + ) + def test_decimal_places(self, description, response, answer, decimal_places, outcome): + params = dict(self.base_params, dp=decimal_places) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_decimal_places_alias(self): + params = dict(self.base_params, decimal_places=2) + result = evaluation_function("92.00 m", "92 m", params) + assert result["is_correct"] is True + + def test_decimal_places_mutually_exclusive_with_atol(self): + params = dict(self.base_params, dp=2, atol=0.1) + with pytest.raises(TaskConfigurationError): + evaluation_function("92.00 m", "92 m", params) + + @pytest.mark.parametrize("decimal_places", [-1, 3.5, True]) + def test_decimal_places_invalid_value_raises(self, decimal_places): + params = dict(self.base_params, dp=decimal_places) + with pytest.raises(TaskConfigurationError): + evaluation_function("92.00 m", "92 m", params) + + +class TestDecimalPlacesTolerance: + + base_params = { + "strict_syntax": False, + "physical_quantity": True, + "units_string": "SI", + "strictness": "natural", + } + + @pytest.mark.parametrize( + "description,response,answer,decimal_places_tol,outcome", + [ + ("Exact value, written precision ignored", "92.00 m", "92 m", 2, True), + ("Within tolerance", "92.004 m", "92 m", 2, True), + ("Outside tolerance", "92.5 m", "92 m", 2, False), + ("Tighter tolerance rejects", "92.02 m", "92 m", 3, False), + ("Unit mismatch fails regardless", "92 s", "92 m", 2, False), + ("No units, plain numeric within tolerance", "3.1416 m", "3.14159 m", 3, True), + ("Non-numeric response", "two m", "92 m", 2, False), + ] + ) + def test_decimal_places_tolerance(self, description, response, answer, decimal_places_tol, outcome): + params = dict(self.base_params, decimal_places_tolerance=decimal_places_tol) + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_decimal_places_tol_alias(self): + params = dict(self.base_params, dp_tol=2) + result = evaluation_function("92.004 m", "92 m", params) + assert result["is_correct"] is True + + @pytest.mark.parametrize("conflicting", [{"atol": 0.1}, {"rtol": 0.1}, {"significant_figures": 2}, {"decimal_places": 2}]) + def test_decimal_places_tolerance_mutually_exclusive(self, conflicting): + params = dict(self.base_params, decimal_places_tolerance=2, **conflicting) + with pytest.raises(TaskConfigurationError): + evaluation_function("92 m", "92 m", params) + + @pytest.mark.parametrize("decimal_places_tol", [-1, 3.5, True]) + def test_decimal_places_tolerance_invalid_value_raises(self, decimal_places_tol): + params = dict(self.base_params, decimal_places_tolerance=decimal_places_tol) + with pytest.raises(TaskConfigurationError): + evaluation_function("92 m", "92 m", params) diff --git a/app/tests/precision_test.py b/app/tests/precision_test.py new file mode 100644 index 0000000..f6e4a5a --- /dev/null +++ b/app/tests/precision_test.py @@ -0,0 +1,114 @@ +import pytest + +from app.context.symbolic.numerical import decimal_places_match, sig_figs_match +from app.context.tolerance import absolute_tolerance_from_decimal_places, relative_tolerance_from_sig_figs + + +class TestRelativeToleranceFromSigFigs: + + @pytest.mark.parametrize( + "sig_figs, expected", + [ + (1, 0.5), + (2, 5e-2), + (3, 5e-3), + (6, 5e-6), + ] + ) + def test_relative_tolerance_from_sig_figs(self, sig_figs, expected): + assert relative_tolerance_from_sig_figs(sig_figs) == pytest.approx(expected) + + def test_no_lower_floor(self): + # Unlike compute_relative_tolerance_from_significant_decimals there is no + # DEFAULT_SIGNIFICANT_FIGURES floor + assert relative_tolerance_from_sig_figs(1) == pytest.approx(0.5) + + +class TestSigFigsMatch: + + @pytest.mark.parametrize( + "response_string, response_value, answer_value, sig_figs, expected", + [ + # Correct value and precision + ("3.14", 3.14, 3.14159, 3, True), + # Numerically wrong + ("3.15", 3.15, 3.14159, 3, False), + # Numerically correct but too many digits written + ("3.14159", 3.14159, 3.14159, 3, False), + # Numerically correct but too few digits written + ("3.1", 3.1, 3.10, 3, False), + # Negative numbers + ("-3.14", -3.14, -3.14159, 3, True), + # Zero answer: precision check is bypassed + ("0", 0.0, 0.0, 3, True), + # Trailing decimal zeros are significant + ("92.00", 92.00, 92, 4, True), + # Leading zeros are not significant + ("0.0032", 0.0032, 0.0032, 2, True), + # Whole number trailing zeros are not significant + ("540", 540, 540, 2, True), + ("540", 540, 540, 3, False), + # Explicit trailing decimal point makes trailing zeros significant + ("540.", 540, 540, 3, True), + # Scientific notation + ("5.02e4", 50200, 50200, 3, True), + # Non-numeric response + ("two", 0, 3.14159, 3, False), + ("3.14e", 3.14, 3.14159, 3, False), + ] + ) + def test_sig_figs_match(self, response_string, response_value, answer_value, sig_figs, expected): + assert sig_figs_match(response_string, response_value, answer_value, sig_figs) is expected + + +class TestAbsoluteToleranceFromDecimalPlaces: + + @pytest.mark.parametrize( + "decimal_places, expected", + [ + (0, 0.5), + (1, 5e-2), + (2, 5e-3), + (3, 5e-4), + ] + ) + def test_absolute_tolerance_from_decimal_places(self, decimal_places, expected): + assert absolute_tolerance_from_decimal_places(decimal_places) == pytest.approx(expected) + + +class TestDecimalPlacesMatch: + + @pytest.mark.parametrize( + "response_string, response_value, answer_value, decimal_places, expected", + [ + # Correct value and precision + ("3.14", 3.14, 3.14159, 2, True), + # Numerically wrong + ("3.15", 3.15, 3.14159, 2, False), + # Numerically correct but too many decimal places written + ("3.142", 3.142, 3.14159, 2, False), + # Numerically correct but too few decimal places written + ("3.1", 3.1, 3.1, 2, False), + ("3.10", 3.1, 3.1, 2, True), + # Negative numbers + ("-3.14", -3.14, -3.14159, 2, True), + # Zero value: precision is still checked, unlike sig_figs_match + ("0", 0.0, 0.0, 2, False), + ("0.00", 0.0, 0.0, 2, True), + # Trailing decimal zeros count + ("92.00", 92.00, 92, 2, True), + ("92.0", 92.0, 92, 2, False), + # decimal_places = 0 requires a whole-number response + ("3", 3.0, 3.14159, 0, True), + ("3.0", 3.0, 3.14159, 0, False), + # Scientific notation + ("5e-3", 0.005, 0.005, 3, True), + ("5.0e-3", 0.005, 0.005, 3, False), + ("5.02e4", 50200, 50200, 0, True), + # Non-numeric response + ("two", 0, 3.14159, 2, False), + ("3.14e", 3.14, 3.14159, 2, False), + ] + ) + def test_decimal_places_match(self, response_string, response_value, answer_value, decimal_places, expected): + assert decimal_places_match(response_string, response_value, answer_value, decimal_places) is expected diff --git a/app/tests/symbolic_evaluation_test.py b/app/tests/symbolic_evaluation_test.py index b9dce87..e747684 100644 --- a/app/tests/symbolic_evaluation_test.py +++ b/app/tests/symbolic_evaluation_test.py @@ -2490,3 +2490,174 @@ def test_greek_letters_in_expressions(self, unicode_expr, letter_expr): } result = evaluation_function(response, answer, params) assert result["is_correct"] is True + + +class TestSigFigs: + + @pytest.mark.parametrize( + "description,response,answer,sig_figs,outcome", + [ + ("Correct value and precision", "3.14", "3.14159", 3, True), + ("Wrong value", "3.15", "3.14159", 3, False), + ("Numerically equal but too many digits written", "3.14159", "3.14159", 3, False), + ("Numerically equal but too few digits written", "3.1", "3.10", 3, False), + ("Negative numbers", "-3.14", "-3.14159", 3, True), + ("Zero answer bypasses precision check", "0", "0", 3, True), + ("Trailing decimal zeros are significant", "92.00", "92", 4, True), + ("Leading zeros are not significant", "0.0032", "0.0032", 2, True), + ("Whole number trailing zeros are not significant", "540", "540", 2, True), + ("Whole number, wrong sig fig count", "540", "540", 3, False), + ("Scientific notation", "5.02e4", "50200", 3, True), + ("Non-numeric response", "two", "3.14159", 3, False), + ("Response with a symbol is rejected", "3.14*x", "3.14159", 3, False), + ] + ) + def test_sig_figs(self, description, response, answer, sig_figs, outcome): + params = {"strict_syntax": False, "sig_figs": sig_figs} + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_significant_figures_alias(self): + params = {"strict_syntax": False, "significant_figures": 3} + result = evaluation_function("3.14", "3.14159", params) + assert result["is_correct"] is True + + def test_sig_figs_mutually_exclusive_with_atol(self): + params = {"strict_syntax": False, "sig_figs": 3, "atol": 0.1} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + def test_sig_figs_mutually_exclusive_with_rtol(self): + params = {"strict_syntax": False, "significant_figures": 3, "relative_tolerance": 0.1} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + @pytest.mark.parametrize("sig_figs", [0, -1, 3.5, True]) + def test_sig_figs_invalid_value_raises(self, sig_figs): + params = {"strict_syntax": False, "sig_figs": sig_figs} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + +class TestSigFigsTolerance: + + @pytest.mark.parametrize( + "description,response,answer,sig_figs_tol,outcome", + [ + ("Within tolerance", "3.1416", "3.14159", 3, True), + ("Within tolerance, fewer digits", "3.14", "3.14159", 3, True), + ("Exact response", "3.14159", "3.14159", 3, True), + ("Written precision is not checked", "3.14000", "3.14159", 3, True), + ("Too few digits, outside tolerance", "3.1", "3.14159", 3, False), + ("Wrong value", "3.2", "3.14159", 3, False), + ("Tighter tolerance rejects rounded value", "3.14", "3.14159", 5, False), + ("Negative numbers", "-3.1416", "-3.14159", 3, True), + ("Non-numeric response", "two", "3.14159", 3, False), + ] + ) + def test_sig_figs_tolerance(self, description, response, answer, sig_figs_tol, outcome): + params = {"strict_syntax": False, "significant_figures_tolerance": sig_figs_tol} + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_sig_figs_tol_alias(self): + params = {"strict_syntax": False, "sig_figs_tol": 3} + result = evaluation_function("3.14", "3.14159", params) + assert result["is_correct"] is True + + @pytest.mark.parametrize("conflicting", [{"atol": 0.1}, {"rtol": 0.1}, {"significant_figures": 3}]) + def test_sig_figs_tolerance_mutually_exclusive(self, conflicting): + params = {"strict_syntax": False, "significant_figures_tolerance": 3, **conflicting} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + @pytest.mark.parametrize("sig_figs_tol", [0, -1, 3.5, True]) + def test_sig_figs_tolerance_invalid_value_raises(self, sig_figs_tol): + params = {"strict_syntax": False, "significant_figures_tolerance": sig_figs_tol} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + +class TestDecimalPlaces: + + @pytest.mark.parametrize( + "description,response,answer,decimal_places,outcome", + [ + ("Correct value and precision", "3.14", "3.14159", 2, True), + ("Wrong value", "3.15", "3.14159", 2, False), + ("Numerically equal but too many digits written", "3.14159", "3.14159", 2, False), + ("Numerically equal but too few digits written", "3.1", "3.10", 2, False), + ("Negative numbers", "-3.14", "-3.14159", 2, True), + ("Zero value, precision is still checked", "0", "0", 2, False), + ("Zero value, written to the required precision", "0.00", "0", 2, True), + ("Trailing decimal zeros count", "92.00", "92", 2, True), + ("Whole number rejected when decimals required", "3", "3.14159", 2, False), + ("decimal_places=0 requires a whole number", "3", "3.14159", 0, True), + ("decimal_places=0 rejects a written decimal", "3.0", "3.14159", 0, False), + ("Scientific notation", "5.02e4", "50200", 0, True), + ("Non-numeric response", "two", "3.14159", 2, False), + ("Response with a symbol is rejected", "3.14*x", "3.14159", 2, False), + ] + ) + def test_decimal_places(self, description, response, answer, decimal_places, outcome): + params = {"strict_syntax": False, "dp": decimal_places} + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_decimal_places_alias(self): + params = {"strict_syntax": False, "decimal_places": 2} + result = evaluation_function("3.14", "3.14159", params) + assert result["is_correct"] is True + + @pytest.mark.parametrize( + "conflicting", [{"atol": 0.1}, {"rtol": 0.1}, {"significant_figures": 3}, {"decimal_places_tolerance": 2}] + ) + def test_decimal_places_mutually_exclusive(self, conflicting): + params = {"strict_syntax": False, "decimal_places": 2, **conflicting} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + @pytest.mark.parametrize("decimal_places", [-1, 3.5, True]) + def test_decimal_places_invalid_value_raises(self, decimal_places): + params = {"strict_syntax": False, "dp": decimal_places} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + +class TestDecimalPlacesTolerance: + + @pytest.mark.parametrize( + "description,response,answer,decimal_places_tol,outcome", + [ + ("Within tolerance", "3.1416", "3.14159", 2, True), + ("Within tolerance, fewer digits", "3.14", "3.14159", 2, True), + ("Exact response", "3.14159", "3.14159", 2, True), + ("Written precision is not checked", "3.14000", "3.14159", 2, True), + ("Too few digits, outside tolerance", "3.1", "3.14159", 2, False), + ("Wrong value", "3.2", "3.14159", 2, False), + ("Tighter tolerance rejects rounded value", "3.14", "3.14159", 4, False), + ("Negative numbers", "-3.1416", "-3.14159", 2, True), + ("Non-numeric response", "two", "3.14159", 2, False), + ] + ) + def test_decimal_places_tolerance(self, description, response, answer, decimal_places_tol, outcome): + params = {"strict_syntax": False, "decimal_places_tolerance": decimal_places_tol} + result = evaluation_function(response, answer, params) + assert result["is_correct"] is outcome + + def test_decimal_places_tol_alias(self): + params = {"strict_syntax": False, "dp_tol": 2} + result = evaluation_function("3.14", "3.14159", params) + assert result["is_correct"] is True + + @pytest.mark.parametrize("conflicting", [{"atol": 0.1}, {"rtol": 0.1}, {"significant_figures": 3}, {"decimal_places": 2}]) + def test_decimal_places_tolerance_mutually_exclusive(self, conflicting): + params = {"strict_syntax": False, "decimal_places_tolerance": 2, **conflicting} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params) + + @pytest.mark.parametrize("decimal_places_tol", [-1, 3.5, True]) + def test_decimal_places_tolerance_invalid_value_raises(self, decimal_places_tol): + params = {"strict_syntax": False, "decimal_places_tolerance": decimal_places_tol} + with pytest.raises(TaskConfigurationError): + evaluation_function("3.14", "3.14159", params)