Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions deepmd/dpmodel/atomic_model/base_atomic_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
check_preset_out_bias,
normalize_preset_out_bias,
preset_out_bias_rows,
recover_nonfinite_preset_bias,
remap_preset_out_bias,
)

Expand Down Expand Up @@ -822,6 +823,7 @@ def change_out_bias(
stat_file_path: DPPath | None = None,
bias_adjust_mode: str = "change-by-statistic",
observed_type: list[str] | None = None,
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias according to the input data and the pretrained model.

Expand All @@ -845,6 +847,14 @@ def change_out_bias(
('set-by-statistic') or their stored bias ('change-by-statistic'); no
statistics are computed, read or written for such an output, whose
output std keeps its stored value.
preset_out_bias : dict, optional
An explicit downstream preset. Assigned outputs use this bias and
skip fitting; other outputs fit residuals against the stored bias.
The model preset is used when omitted, preserving training and
fine-tuning behavior. Pass an empty dict to fit from the data
without reapplying the model's stored preset.
Before residual fitting, non-finite stored cells covered by the
checkpoint preset are recovered; finite adjusted cells are retained.
stat_file_path : Optional[DPPath]
The path to the stat file.
observed_type : list[str], optional
Expand All @@ -859,17 +869,32 @@ def change_out_bias(
if bias_adjust_mode not in ("change-by-statistic", "set-by-statistic"):
raise RuntimeError("Unknown bias_adjust_mode mode: " + bias_adjust_mode)
change = bias_adjust_mode == "change-by-statistic"
preset = (
normalize_preset_out_bias(preset_out_bias, self.type_map)
if preset_out_bias is not None
else self.preset_out_bias
)
distinguish_types = self.get_compute_stats_distinguish_types()
check_preset_out_bias(self.preset_out_bias, self.bias_keys, distinguish_types)
check_preset_out_bias(preset, self.bias_keys, distinguish_types)
sampled = sample_merged
# A per-call override must not propagate a corrupt checkpoint baseline.
if change and preset_out_bias is not None and self.preset_out_bias:
recovered = recover_nonfinite_preset_bias(
self.preset_out_bias,
self.type_map,
to_numpy_array(self.out_bias),
self.bias_keys,
[self.atomic_output_def()[kk].size for kk in self.bias_keys],
)
self._store_out_stat(recovered)
# === Step 1. Outputs fixed by the preset ===
fitted_keys = self.bias_keys
if self.preset_out_bias:
if preset:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Blocking: with an explicit preset_out_bias={} — which is exactly what dp change-bias now passes — preset normalizes to {} and this if preset: gate is False, so Step 1 is skipped entirely. Step 1 is not only where preset-assigned types receive their preset values; preset_out_bias_rows(...) also recovers non-finite stored bias cells of preset-assigned types back to the finite preset (behavior pinned by test_nonfinite_assigned_bias_from_checkpoint in source/tests/pt/model/test_atomic_model_global_stat.py, which sets stored out_bias cells to nan/±inf and expects [-10.0, -13.6, 0.0] back). A checkpoint whose stored bias has NaN cells for a preset-assigned type is the normal case for a zero-sample type covered by the preset: on master, dp change-bias recovers the finite preset; after this PR the CLI path skips Step 1 and change-by-statistic folds its shift onto the NaN stored cell (add=True), so NaN propagates into the adjusted model. I see the Copilot threads on this were resolved, but the mechanism is unchanged at this head. If dropping that recovery on the CLI path is intended, please make that explicit (the pinned test's CLI analogue would need to change too); otherwise recover non-finite stored cells before skipping Step 1 when an explicit empty preset is passed.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed at exact head fce767f2554895f65782858492725ed244ab5d02 with a bounded NumPy reproduction using the exact-source bias/statistics methods and an atomic-forward test double, not a native CLI/backend run. With stored O/H biases [NaN, NaN] and finite presets, base 668b60c and the current omitted-override path restore the presets; preset_out_bias={} instead evaluates the non-finite baseline and leaves non-finite output. Positive/negative infinity reproduce too, and a preset-covered absent type retains its non-finite stored cell after adding a zero shift. All 26 comparison/control cases passed. The existing non-finite checkpoint test omits the override, so it does not cover this CLI path.

One scope qualification: this is a recovery regression for an already non-finite checkpoint. Ordinary preset initialization writes assigned values even for absent types, so zero samples alone do not establish that such a checkpoint is normal. A recovery step before residual fitting should replace only non-finite assigned cells, preserve finite adjusted baselines, and still allow {} to fit the output from data.

Agent: dot

if observed_type is None:
sampled = sample_merged() if callable(sample_merged) else sample_merged
observed_type = collect_observed_types(sampled, self.type_map)
rows = preset_out_bias_rows(
self.preset_out_bias,
preset,
self.type_map,
observed_type,
to_numpy_array(self.out_bias),
Expand Down
12 changes: 11 additions & 1 deletion deepmd/dpmodel/model/make_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,7 @@ def change_out_bias(
self,
merged: Any,
bias_adjust_mode: str = "change-by-statistic",
preset_out_bias: dict | None = None,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking: the new preset_out_bias parameter is documented in BaseAtomicModel.change_out_bias but in none of the six forwarding wrappers that now expose it — deepmd/dpmodel/model/make_model.py:925, deepmd/pt/model/model/make_model.py:334, deepmd/pd/model/model/make_model.py:230, deepmd/dpmodel/model/spin_model.py:606, deepmd/pt/model/model/spin_model.py:521, deepmd/pt/model/model/sezm_spin_model.py:433. Please add numpydoc Parameters entries consistent with the rest of the package; for the spin wrappers in particular, the remap_preset_out_bias step means the dict the caller passes is not the dict the atomic model sees, and that is not inferable without a docstring entry.

) -> None:
"""Change the output bias according to the input data and the pretrained model.

Expand All @@ -936,8 +937,17 @@ def change_out_bias(
bias_adjust_mode : str
The mode for changing output bias:
'change-by-statistic' or 'set-by-statistic'.
preset_out_bias : dict, optional
Per-call preset override. Omit to use the configured preset;
pass an empty dict to fit without fixing preset outputs.
Non-finite stored cells covered by the checkpoint preset are
recovered before residual fitting; finite values are retained.
"""
self.atomic_model.change_out_bias(merged, bias_adjust_mode=bias_adjust_mode)
self.atomic_model.change_out_bias(
merged,
bias_adjust_mode=bias_adjust_mode,
preset_out_bias=preset_out_bias,
)

def _input_type_cast(
self,
Expand Down
16 changes: 16 additions & 0 deletions deepmd/dpmodel/model/spin_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@
from deepmd.dpmodel.output_def import (
ModelOutputDef,
)
from deepmd.utils.preset_out_bias import (
normalize_preset_out_bias,
remap_preset_out_bias,
)
from deepmd.utils.spin import (
Spin,
)
Expand Down Expand Up @@ -603,6 +607,7 @@ def change_out_bias(
self,
merged: Callable[[], list[dict]] | list[dict],
bias_adjust_mode: str = "change-by-statistic",
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias of atomic model according to the input data and the pretrained model.

Expand All @@ -620,13 +625,24 @@ def change_out_bias(
'change-by-statistic' : perform predictions on labels of target dataset,
and do least square on the errors to obtain the target shift as bias.
'set-by-statistic' : directly use the statistic output bias in the target dataset.
preset_out_bias : dict, optional
Preset on the public real-atom type map. It is normalized and
extended with unassigned virtual types before forwarding. Omit
to retain the model preset; an empty dict fits without fixing
preset outputs. Non-finite stored cells covered by a checkpoint
preset are recovered before residual fitting.
"""
preset_out_bias = remap_preset_out_bias(
normalize_preset_out_bias(preset_out_bias, self.get_type_map()),
list(range(self.get_ntypes())) + [-1] * self.get_ntypes(),
)
spin_sampled_func = self._get_spin_sampled_func(
merged if callable(merged) else lambda: merged
)
self.backbone_model.change_out_bias(
spin_sampled_func,
bias_adjust_mode=bias_adjust_mode,
preset_out_bias=preset_out_bias,
Comment thread
Copilot marked this conversation as resolved.
)

def change_type_map(
Expand Down
5 changes: 5 additions & 0 deletions deepmd/dpmodel/train/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,17 @@ def change_model_out_bias(
*,
bias_adjust_mode: str = "change-by-statistic",
recompute_input_stats: bool = False,
preset_out_bias: dict | None = None,
) -> Any:
"""Change one model's output bias and log the before/after values."""
old_bias = deepcopy(model.get_out_bias())
preset_kwargs = (
{"preset_out_bias": preset_out_bias} if preset_out_bias is not None else {}
)
model.change_out_bias(
sample_func,
bias_adjust_mode=bias_adjust_mode,
**preset_kwargs,
)
new_bias = deepcopy(model.get_out_bias())

Expand Down
5 changes: 4 additions & 1 deletion deepmd/pd/entrypoints/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,10 @@ def change_bias(
nbatches,
)
updated_model = training.model_change_out_bias(
model_to_change, sampled_data, _bias_adjust_mode=bias_adjust_mode
model_to_change,
sampled_data,
_bias_adjust_mode=bias_adjust_mode,
preset_out_bias={},
Comment thread
anyangml marked this conversation as resolved.
)

if not multi_task:
Expand Down
33 changes: 30 additions & 3 deletions deepmd/pd/model/atomic_model/base_atomic_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
check_preset_out_bias,
normalize_preset_out_bias,
preset_out_bias_rows,
recover_nonfinite_preset_bias,
remap_preset_out_bias,
)

Expand Down Expand Up @@ -501,6 +502,7 @@ def change_out_bias(
stat_file_path: DPPath | None = None,
bias_adjust_mode: str = "change-by-statistic",
observed_type: list[str] | None = None,
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias according to the input data and the pretrained model.

Expand All @@ -524,6 +526,14 @@ def change_out_bias(
('set-by-statistic') or their stored bias ('change-by-statistic'); no
statistics are computed, read or written for such an output, whose
output std keeps its stored value.
preset_out_bias : dict, optional
An explicit downstream preset. Assigned outputs use this bias and
skip fitting; other outputs fit residuals against the stored bias.
The model preset is used when omitted, preserving training and
fine-tuning behavior. Pass an empty dict to fit from the data
without reapplying the model's stored preset.
Before residual fitting, non-finite stored cells covered by the
checkpoint preset are recovered; finite adjusted cells are retained.
stat_file_path : Optional[DPPath]
The path to the stat file.
observed_type : list[str], optional
Expand All @@ -533,17 +543,34 @@ def change_out_bias(
if bias_adjust_mode not in ("change-by-statistic", "set-by-statistic"):
raise RuntimeError("Unknown bias_adjust_mode mode: " + bias_adjust_mode)
change = bias_adjust_mode == "change-by-statistic"
preset = (
normalize_preset_out_bias(preset_out_bias, self.type_map)
if preset_out_bias is not None
else self.preset_out_bias
)
distinguish_types = self.get_compute_stats_distinguish_types()
check_preset_out_bias(self.preset_out_bias, self.bias_keys, distinguish_types)
check_preset_out_bias(preset, self.bias_keys, distinguish_types)
sampled = sample_merged
# A per-call override must not propagate a corrupt checkpoint baseline.
if change and preset_out_bias is not None and self.preset_out_bias:
recovered = recover_nonfinite_preset_bias(
self.preset_out_bias,
self.type_map,
to_numpy_array(self.out_bias),
self.bias_keys,
[self.atomic_output_def()[kk].size for kk in self.bias_keys],
)
self._store_out_stat(
{kk: to_paddle_tensor(vv) for kk, vv in recovered.items()}
)
# === Step 1. Outputs fixed by the preset ===
fitted_keys = self.bias_keys
if self.preset_out_bias:
if preset:
if observed_type is None:
sampled = sample_merged() if callable(sample_merged) else sample_merged
observed_type = collect_observed_types(sampled, self.type_map)
rows = preset_out_bias_rows(
self.preset_out_bias,
preset,
self.type_map,
observed_type,
to_numpy_array(self.out_bias),
Expand Down
7 changes: 7 additions & 0 deletions deepmd/pd/model/model/make_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ def change_out_bias(
self,
merged: list[dict] | Callable[[], list[dict]],
bias_adjust_mode: str = "change-by-statistic",
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias of atomic model according to the input data and the pretrained model.

Expand All @@ -248,10 +249,16 @@ def change_out_bias(
'change-by-statistic' : perform predictions on labels of target dataset,
and do least square on the errors to obtain the target shift as bias.
'set-by-statistic' : directly use the statistic output bias in the target dataset.
preset_out_bias : dict, optional
Per-call preset override. Omit to use the configured preset;
pass an empty dict to fit without fixing preset outputs.
Non-finite stored cells covered by the checkpoint preset are
recovered before residual fitting; finite values are retained.
"""
self.atomic_model.change_out_bias(
merged,
bias_adjust_mode=bias_adjust_mode,
preset_out_bias=preset_out_bias,
)

def forward_common_lower(
Expand Down
5 changes: 5 additions & 0 deletions deepmd/pd/train/training.py
Original file line number Diff line number Diff line change
Expand Up @@ -1455,11 +1455,16 @@ def model_change_out_bias(
_model: Any,
_sample_func: Any,
_bias_adjust_mode: str = "change-by-statistic",
preset_out_bias: dict | None = None,
) -> None:
old_bias = deepcopy(_model.get_out_bias())
preset_kwargs = (
{"preset_out_bias": preset_out_bias} if preset_out_bias is not None else {}
)
_model.change_out_bias(
_sample_func,
bias_adjust_mode=_bias_adjust_mode,
**preset_kwargs,
)
new_bias = deepcopy(_model.get_out_bias())

Expand Down
5 changes: 4 additions & 1 deletion deepmd/pt/entrypoints/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,10 @@ def change_bias(
min_pair_dist=0.0 if pair_filter is None else float(pair_filter.default),
)
updated_model = training.model_change_out_bias(
model_to_change, sampled_data, _bias_adjust_mode=bias_adjust_mode
model_to_change,
sampled_data,
_bias_adjust_mode=bias_adjust_mode,
preset_out_bias={},
)

if not multi_task:
Expand Down
33 changes: 30 additions & 3 deletions deepmd/pt/model/atomic_model/base_atomic_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
check_preset_out_bias,
normalize_preset_out_bias,
preset_out_bias_rows,
recover_nonfinite_preset_bias,
remap_preset_out_bias,
)

Expand Down Expand Up @@ -629,6 +630,7 @@ def change_out_bias(
bias_adjust_mode: str = "change-by-statistic",
model_forward: Callable[..., dict[str, torch.Tensor]] | None = None,
observed_type: list[str] | None = None,
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias according to the input data and the pretrained model.

Expand All @@ -654,6 +656,14 @@ def change_out_bias(
('set-by-statistic') or their stored bias ('change-by-statistic'); no
statistics are computed, read or written for such an output, whose
output std keeps its stored value.
preset_out_bias : dict, optional
An explicit downstream preset. Assigned outputs use this bias and
skip fitting; other outputs fit residuals against the stored bias.
The model preset is used when omitted, preserving training and
fine-tuning behavior. Pass an empty dict to fit from the data
without reapplying the model's stored preset.
Before residual fitting, non-finite stored cells covered by the
checkpoint preset are recovered; finite adjusted cells are retained.
model_forward : Callable[..., dict[str, torch.Tensor]], optional
Predictor of the complete atomic outputs used by 'change-by-statistic'.
Defaults to the forward of this atomic model; a model that adds
Expand All @@ -665,17 +675,34 @@ def change_out_bias(
if bias_adjust_mode not in ("change-by-statistic", "set-by-statistic"):
raise RuntimeError("Unknown bias_adjust_mode mode: " + bias_adjust_mode)
change = bias_adjust_mode == "change-by-statistic"
preset = (
normalize_preset_out_bias(preset_out_bias, self.type_map)
if preset_out_bias is not None
else self.preset_out_bias
)
distinguish_types = self.get_compute_stats_distinguish_types()
check_preset_out_bias(self.preset_out_bias, self.bias_keys, distinguish_types)
check_preset_out_bias(preset, self.bias_keys, distinguish_types)
sampled = sample_merged
# A per-call override must not propagate a corrupt checkpoint baseline.
if change and preset_out_bias is not None and self.preset_out_bias:
recovered = recover_nonfinite_preset_bias(
self.preset_out_bias,
self.type_map,
to_numpy_array(self.out_bias),
self.bias_keys,
[self.atomic_output_def()[kk].size for kk in self.bias_keys],
)
self._store_out_stat(
{kk: to_torch_tensor(vv) for kk, vv in recovered.items()}
)
# === Step 1. Outputs fixed by the preset ===
fitted_keys = self.bias_keys
if self.preset_out_bias:
if preset:
if observed_type is None:
sampled = sample_merged() if callable(sample_merged) else sample_merged
observed_type = collect_observed_types(sampled, self.type_map)
rows = preset_out_bias_rows(
self.preset_out_bias,
preset,
self.type_map,
observed_type,
to_numpy_array(self.out_bias),
Expand Down
7 changes: 7 additions & 0 deletions deepmd/pt/model/model/make_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ def change_out_bias(
self,
merged: Any,
bias_adjust_mode: str = "change-by-statistic",
preset_out_bias: dict | None = None,
) -> None:
"""Change the output bias of atomic model according to the input data and the pretrained model.

Expand All @@ -355,10 +356,16 @@ def change_out_bias(
'change-by-statistic' : perform predictions on labels of target dataset,
and do least square on the errors to obtain the target shift as bias.
'set-by-statistic' : directly use the statistic output bias in the target dataset.
preset_out_bias : dict, optional
Per-call preset override. Omit to use the configured preset;
pass an empty dict to fit without fixing preset outputs.
Non-finite stored cells covered by the checkpoint preset are
recovered before residual fitting; finite values are retained.
"""
self.atomic_model.change_out_bias(
merged,
bias_adjust_mode=bias_adjust_mode,
preset_out_bias=preset_out_bias,
model_forward=self.predict_atomic_outputs_for_stat,
)

Expand Down
Loading
Loading