From 3a864cde3dd1aec314f1596db46f170d6d3c9845 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 30 Jul 2026 21:32:08 +0200 Subject: [PATCH] new BORG_UNITS env var: si / iec / raw size formatting, #5513 BORG_UNITS determines how borg formats sizes in human-readable output: - si (default): decimal units, e.g. 1.23 MB (1kB = 1000B) - iec: binary units, e.g. 1.18 MiB (1KiB = 1024B) - raw: exact byte counts, e.g. 1234567 B raw is meant for scripts (e.g. monitoring) that want to parse sizes without having to deal with scaled values and units. An invalid BORG_UNITS value is warned about (once) and ignored. BORG_IEC was removed, use BORG_UNITS=iec. format_file_size now determines the units itself (via BORG_UNITS), so the iec argument plumbing through Archive / Statistics / ArchiveFormatter / Cache / ArchiveChecker / ArchiveGarbageCollector and use_iec_units() are all gone. Beside being less code, this also makes the setting effective at the call sites that never passed iec=use_iec_units(), e.g. borg repo-space. --- docs/changes.rst | 7 +- docs/usage/general/environment.rst.inc | 19 +++- src/borg/archive.py | 36 +++----- src/borg/archiver/_common.py | 4 - src/borg/archiver/analyze_cmd.py | 7 +- src/borg/archiver/check_cmd.py | 3 +- src/borg/archiver/compact_cmd.py | 27 ++---- src/borg/archiver/create_cmd.py | 10 +- src/borg/archiver/extract_cmd.py | 6 +- src/borg/archiver/info_cmd.py | 4 +- src/borg/archiver/prune_cmd.py | 4 +- src/borg/archiver/repo_list_cmd.py | 6 +- src/borg/archiver/repo_space_cmd.py | 6 +- src/borg/archiver/tar_cmds.py | 2 - src/borg/cache.py | 13 +-- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/parseformat.py | 44 +++++---- .../testsuite/archiver/compact_cmd_test.py | 26 +++--- .../testsuite/helpers/parseformat_test.py | 91 +++++++++++++++---- 19 files changed, 178 insertions(+), 139 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 6047cbadb0..bd48fd29fe 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -110,7 +110,7 @@ Compatibility notes: - removed --rsh option (use the BORG_RSH environment variable) - removed --upload-ratelimit and --upload-buffer options (they only ever affected uploads to a borg 1.x repository via ssh://) -- removed --iec option (use the BORG_IEC environment variable) +- removed --iec option (use the BORG_UNITS environment variable) - removed --debug-profile option (use the BORG_DEBUG_PROFILE environment variable) - removed borg config command (only worked locally anyway) - compact command now requires access to the borg key if the repo is encrypted @@ -174,6 +174,11 @@ New features: deleting the whole set would free). Chunk ids and sizes are read from the per-archive references cache maintained by ``borg compact``, so unchanged archives usually do not need to be opened. +- new ``BORG_UNITS`` environment variable, #5513. + It determines how sizes are formatted in human-readable output: ``si`` (default, + 1kB = 1000B), ``iec`` (1KiB = 1024B) or ``raw`` (exact byte counts, e.g. for easy + parsing by monitoring scripts). + ``BORG_IEC`` was removed, use ``BORG_UNITS=iec`` instead. Version 2.0.0b22 (2026-07-22) ----------------------------- diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index f11d3ea46f..0689010132 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -71,10 +71,21 @@ General: BORG_REMOTE_PATH When set, use the given path as borg executable on the remote (defaults to "borg" if unset). This is the replacement for the removed ``--remote-path PATH`` command line option. - BORG_IEC - When set to ``yes``, format sizes using IEC units (1KiB = 1024B) instead of - decimal units (1kB = 1000B). - This is the replacement for the removed ``--iec`` command line option. + BORG_UNITS + Determines how borg formats sizes in its human-readable output: + + - ``si`` (default): decimal units, e.g. ``1.23 MB`` (1kB = 1000B) + - ``iec``: binary units, e.g. ``1.18 MiB`` (1KiB = 1024B) + - ``raw``: exact byte counts, e.g. ``1234567 B`` + + Use ``raw`` if you want to parse sizes with scripts (e.g. for monitoring), + so you do not have to deal with scaled values and different units. + Alternatively, use a command's ``--json`` output or, for the commands + supporting ``--format``, the size related format keys - sizes are given + as byte counts there anyway. + + ``BORG_UNITS=iec`` is the replacement for the removed ``BORG_IEC`` environment + variable (and for the ``--iec`` command line option removed before that). BORG_DEBUG_PROFILE When set to a filename, write an execution profile in Borg format into that file (see :ref:`debugging`). If the filename ends with ``.pyprof``, a Python-compatible diff --git a/src/borg/archive.py b/src/borg/archive.py index 4af2d47f18..c7a187d9dc 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -86,14 +86,14 @@ del _method -def format_store_stats(stats, iec=False): +def format_store_stats(stats): """Render a borgstore stats dict as one "Store : " line per entry.""" def format_value(key, value): if key.endswith("_throughput"): - return f"{format_file_size(value, iec=iec)}/s" + return f"{format_file_size(value)}/s" if key.endswith("_volume"): - return format_file_size(value, iec=iec) + return format_file_size(value) if key.endswith("_time"): return f"{value:.3f} seconds" if key.endswith("_ratio"): @@ -106,9 +106,8 @@ def format_value(key, value): class Statistics: - def __init__(self, output_json=False, iec=False): + def __init__(self, output_json=False): self.output_json = output_json - self.iec = iec self.osize = self.usize = self.nfiles = 0 self.last_progress = 0 # timestamp when last progress was shown self.files_stats = defaultdict(int) @@ -124,7 +123,7 @@ def update(self, size, unique): def __add__(self, other): if not isinstance(other, Statistics): raise TypeError("can only add Statistics objects") - stats = Statistics(self.output_json, self.iec) + stats = Statistics(self.output_json) stats.osize = self.osize + other.osize stats.usize = self.usize + other.usize stats.nfiles = self.nfiles + other.nfiles @@ -160,7 +159,7 @@ def __str__(self): files_changed_while_reading=self.files_stats["C"], ) if self.store_stats: - result += format_store_stats(self.store_stats, iec=self.iec) + "\n" + result += format_store_stats(self.store_stats) + "\n" return result def __repr__(self): @@ -170,7 +169,7 @@ def __repr__(self): def as_dict(self): return { - "original_size": FileSize(self.osize, iec=self.iec), + "original_size": FileSize(self.osize), "nfiles": self.nfiles, "hashing_time": self.hashing_time, "chunking_time": self.chunking_time, @@ -190,11 +189,11 @@ def from_raw_dict(cls, **kw): @property def osize_fmt(self): - return format_file_size(self.osize, iec=self.iec) + return format_file_size(self.osize) @property def usize_fmt(self): - return format_file_size(self.usize, iec=self.iec) + return format_file_size(self.usize) def show_progress(self, item=None, final=False, stream=None, dt=None): now = time.monotonic() @@ -518,7 +517,6 @@ def __init__( start=None, end=None, log_json=False, - iec=False, deleted=False, ): name_is_id = isinstance(name, bytes) @@ -534,8 +532,7 @@ def __init__( self.repo_objs = manifest.repo_objs self.repository = manifest.repository self.cache = cache - self.stats = Statistics(output_json=log_json, iec=iec) - self.iec = iec + self.stats = Statistics(output_json=log_json) self.show_progress = progress self.name = name # overwritten later with name from archive metadata self.name_in_manifest = name # can differ from .name later (if borg check fixed duplicate archive names) @@ -738,7 +735,7 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m return metadata def calc_stats(self, cache, want_unique=True): - stats = Statistics(iec=self.iec) + stats = Statistics() stats.usize = 0 # this is expensive to compute stats.nfiles = self.metadata.nfiles stats.osize = self.metadata.size @@ -1318,7 +1315,6 @@ def __init__( show_progress, sparse, log_json, - iec, file_status_printer=None, files_changed="mtime" if is_win32 else "ctime", ): @@ -1332,7 +1328,7 @@ def __init__( self.files_changed = files_changed self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None - self.stats = Statistics(output_json=log_json, iec=iec) # threading: done by cache (including progress) + self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) self.cwd = os.getcwd() self.chunker = get_chunker(*chunker_params, key=key, sparse=sparse) @@ -1616,7 +1612,6 @@ def __init__( chunker_params, show_progress, log_json, - iec, file_status_printer=None, ): self.cache = cache @@ -1626,7 +1621,7 @@ def __init__( self.show_progress = show_progress self.print_file_status = file_status_printer or (lambda *args: None) - self.stats = Statistics(output_json=log_json, iec=iec) # threading: done by cache (including progress) + self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) self.chunker = get_chunker(*chunker_params, key=key, sparse=False) self.hlm = HardLinkManager(id_type=str, info_type=list) # normalized/safe path -> chunks @@ -1829,7 +1824,6 @@ def check( newer=None, oldest=None, newest=None, - iec=False, ): """Perform a set of checks on 'repository' @@ -1841,7 +1835,6 @@ def check( :param oldest/newest: only check archives older/newer than timedelta from oldest/newest archive timestamp :param verify_data: integrity verification of data referenced by archives :param format: format string used to describe an archive in the log output - :param iec: format sizes using IEC units (1KiB = 1024B) """ if not isinstance(repository, Repository): logger.error("Checking legacy repositories is not supported.") @@ -1850,7 +1843,6 @@ def check( self.check_all = not any((first, last, match, older, newer, oldest, newest)) self.repair = repair self.format = format - self.iec = iec self.repository = repository # A normal (non-repair) archives check trusts the in-repo index: the repository check verified # each index object's sha256, and the index is the authoritative record of which chunks exist, @@ -2210,7 +2202,7 @@ def valid_item(obj): else: archive_infos = self.manifest.archives.list(sort_by=sort_by) num_archives = len(archive_infos) - formatter = ArchiveFormatter(self.format, self.repository, self.manifest, self.key, iec=self.iec) + formatter = ArchiveFormatter(self.format, self.repository, self.manifest, self.key) pi = ProgressIndicatorPercent( total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives" diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index c5c68ff2ff..77787c26a5 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -9,7 +9,6 @@ from ..helpers import Error from ..helpers import SortBySpec, location_validator, Location, relative_time_marker_validator from ..helpers import Highlander, octal_int -from ..helpers import use_iec_units from ..helpers.argparsing import SUPPRESS, PositiveInt from ..helpers.nanorst import rst_to_terminal from ..manifest import Manifest, AI_HUMAN_SORT_KEYS @@ -167,7 +166,6 @@ def wrapper(self, args, **kwargs): progress=getattr(args, "progress", False), cache_mode=getattr(args, "files_cache_mode", FILES_CACHE_MODE_DISABLED), start_backup=getattr(self, "start_backup", None), - iec=use_iec_units(), ) as cache_: return method(self, args, repository=repository, cache=cache_, **kwargs) else: @@ -238,7 +236,6 @@ def wrapper(self, args, **kwargs): manifest_, progress=False, cache_mode=getattr(args, "files_cache_mode", FILES_CACHE_MODE_DISABLED), - iec=use_iec_units(), ) as cache_: kwargs["other_cache"] = cache_ return method(self, args, **kwargs) @@ -265,7 +262,6 @@ def wrapper(self, args, repository, manifest, **kwargs): noxattrs=getattr(args, "noxattrs", False), cache=kwargs.get("cache"), log_json=args.log_json, - iec=use_iec_units(), ) return method(self, args, repository=repository, manifest=manifest, archive=archive, **kwargs) diff --git a/src/borg/archiver/analyze_cmd.py b/src/borg/archiver/analyze_cmd.py index 8aa811648f..55330d447b 100644 --- a/src/borg/archiver/analyze_cmd.py +++ b/src/borg/archiver/analyze_cmd.py @@ -5,7 +5,7 @@ from ..archive import Archive from ..cache import get_archive_references, list_archive_reference_caches from ..constants import * # NOQA -from ..helpers import bin_to_hex, Error, format_file_size, use_iec_units +from ..helpers import bin_to_hex, Error, format_file_size from ..helpers import ProgressIndicatorPercent from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -38,7 +38,6 @@ def __init__(self, args, repository, manifest): self.repository = repository assert isinstance(repository, Repository) self.manifest = manifest - self.iec = use_iec_units() self.difference_by_path = defaultdict(int) # directory path -> count of chunks changed def analyze(self): @@ -63,7 +62,7 @@ def analyze(self): logger.info("Finished archives analysis.") def fmt(self, value): - return format_file_size(value, iec=self.iec) + return format_file_size(value) def factor(self, stored, plaintext): """The compression factor stored/plaintext, or n/a if there is no plaintext size to relate to @@ -89,7 +88,7 @@ def mark_references(self, archive_infos, update_flags): ) for i, info in enumerate(archive_infos): references = get_archive_references( - self.repository, self.manifest, info.id, cached=bin_to_hex(info.id) in cached_hex_ids, iec=self.iec + self.repository, self.manifest, info.id, cached=bin_to_hex(info.id) in cached_hex_ids ) for id, ref in references.ids.items(): entry = index.get(id) diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index f5433a898d..9a97d98cf4 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -4,7 +4,7 @@ from ..archive import ArchiveChecker from ..constants import * # NOQA from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError -from ..helpers import yes, ArchiveFormatter, use_iec_units +from ..helpers import yes, ArchiveFormatter from ..helpers.argparsing import ArgumentParser from ..logger import create_logger @@ -80,7 +80,6 @@ def do_check(self, args, repository): oldest=args.oldest, newest=args.newest, format=format, - iec=use_iec_units(), ): set_ec(EXIT_WARNING) return diff --git a/src/borg/archiver/compact_cmd.py b/src/borg/archiver/compact_cmd.py index a00ec10402..6fc9ed3008 100644 --- a/src/borg/archiver/compact_cmd.py +++ b/src/borg/archiver/compact_cmd.py @@ -14,7 +14,6 @@ from ..hashindex import ChunkIndex from ..helpers import set_ec, EXIT_ERROR, Error, sig_int, format_file_size, bin_to_hex, hex_to_bin, IntegrityError from ..helpers import ProgressIndicatorPercent -from ..helpers import use_iec_units from ..manifest import Manifest from ..repository import Repository @@ -24,7 +23,7 @@ class ArchiveGarbageCollector: - def __init__(self, repository, manifest, *, stats, iec, threshold, dry_run=False): + def __init__(self, repository, manifest, *, stats, threshold, dry_run=False): self.repository = repository assert isinstance(repository, Repository) self.manifest = manifest @@ -35,7 +34,6 @@ def __init__(self, repository, manifest, *, stats, iec, threshold, dry_run=False self.archives_count = None # number of archives self.stats = stats # compute repo space usage before/after - lists all repo objects, can be slow. self.threshold = threshold # rewrite a mixed pack only when its wasted-bytes fraction reaches this percent - self.iec = iec # formats statistics using IEC units (1KiB = 1024B) self.dry_run = dry_run self.store_changed = False # True once compact_packs() has changed the store (and its index) @@ -160,7 +158,6 @@ def analyze_archives(self) -> tuple[set, int, int, int]: self.manifest, info.id, cached=bin_to_hex(info.id) in cached_hex_ids, - iec=self.iec, store=not self.dry_run, ) total_files += references.file_count # every fs object counts, not just regular files @@ -200,22 +197,18 @@ def report_and_delete(self): count = len(self.chunks) logger.info(f"Overall statistics, considering all {self.archives_count} archives in this repository:") logger.info( - f"Source data size was {format_file_size(self.total_size, precision=0, iec=self.iec)} " + f"Source data size was {format_file_size(self.total_size, precision=0)} " f"in {self.total_files} files." ) - logger.info(f"Deduplicated size is {format_file_size(deduplicated_size, precision=0, iec=self.iec)}.") + logger.info(f"Deduplicated size is {format_file_size(deduplicated_size, precision=0)}.") dedup_factor = deduplicated_size / self.total_size if self.total_size else 1.0 logger.info(f"Deduplication factor is {dedup_factor:.2f}.") - logger.info( - f"Repository size is {format_file_size(repo_size_after, precision=0, iec=self.iec)} " - f"in {count} objects." - ) + logger.info(f"Repository size is {format_file_size(repo_size_after, precision=0)} " f"in {count} objects.") comp_factor = repo_size_after / deduplicated_size if deduplicated_size else 1.0 logger.info(f"Compression factor is {comp_factor:.2f}.") if not self.dry_run: # nothing was deleted on a dry run, so before == after logger.info( - f"Compaction saved " - f"{format_file_size(repo_size_before - repo_size_after, precision=0, iec=self.iec)}." + f"Compaction saved " f"{format_file_size(repo_size_before - repo_size_after, precision=0)}." ) def mark_soft_deleted_used(self): @@ -228,7 +221,7 @@ def mark_soft_deleted_used(self): for archive_info in self.manifest.archives.list(sort_by=["ts"], deleted=True): name, hex_id = archive_info.name, bin_to_hex(archive_info.id) try: - archive = Archive(self.manifest, archive_info.id, iec=self.iec, deleted=True) + archive = Archive(self.manifest, archive_info.id, deleted=True) missing = sum(not self._mark_object_used(id) for id in self._archive_object_ids(archive)) if missing: logger.warning( @@ -306,7 +299,7 @@ def compact_packs(self): unindexed = sum(total - pack_indexed[pid] for pid, total in pack_total.items() if total > pack_indexed[pid]) if unindexed: logger.info( - f"{format_file_size(unindexed, iec=self.iec)} in pack files is not covered by the index; " + f"{format_file_size(unindexed)} in pack files is not covered by the index; " 'redundant copies are reclaimed on pack rewrite, the rest by "borg check --repair".' ) @@ -357,7 +350,7 @@ def compact_packs(self): if self.dry_run: freed = sum(pack_reclaim.values()) logger.info( - f"Would free {format_file_size(freed, iec=self.iec)} by dropping {len(drop_packs)} " + f"Would free {format_file_size(freed)} by dropping {len(drop_packs)} " f"packs, rewriting {len(rewrite_packs)} packs and merging {len(merge_packs)} tiny packs." ) return repo_size_before, repo_size_before # dry run: report only, change nothing @@ -385,7 +378,7 @@ def compact_packs(self): # objects cut from rewritten packs. reclaimed counts the bytes freed. deleted = sum(len(ids) for ids in forget.values()) + sum(len(ids) for ids in drop.values()) reclaimed = sum(pack_reclaim.values()) - logger.info(f"Deleting {deleted} unused objects, freeing {format_file_size(reclaimed, iec=self.iec)}...") + logger.info(f"Deleting {deleted} unused objects, freeing {format_file_size(reclaimed)}...") pi = ProgressIndicatorPercent( total=len(drop_packs) + len(rewrite_packs) + (1 if merge_packs else 0), msg="Compacting packs %3.1f%%", @@ -437,7 +430,7 @@ def do_compact(self, args, repository, manifest): # is nothing to do and make compaction a silent no-op. repository.assert_writable() ArchiveGarbageCollector( - repository, manifest, stats=args.stats, iec=use_iec_units(), threshold=args.threshold, dry_run=args.dry_run + repository, manifest, stats=args.stats, threshold=args.threshold, dry_run=args.dry_run ).garbage_collect() def build_parser_compact(self, subparsers, common_parser, mid_common_parser): diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 0c03eddb01..a34d9fa30e 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -18,7 +18,6 @@ from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, FilesCacheMode, octal_int from ..helpers import eval_escapes -from ..helpers import use_iec_units from ..helpers import timestamp, archive_ts_now from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify from ..helpers import dir_is_tagged @@ -225,12 +224,7 @@ def create_inner(archive, cache, fso): logger.info('Creating archive "%s" in repository %s' % (args.name, args.location.processed)) if not dry_run: with Cache( - repository, - manifest, - progress=args.progress, - cache_mode=args.files_cache_mode, - iec=use_iec_units(), - archive_name=args.name, + repository, manifest, progress=args.progress, cache_mode=args.files_cache_mode, archive_name=args.name ) as cache: archive = Archive( manifest, @@ -244,7 +238,6 @@ def create_inner(archive, cache, fso): chunker_params=args.chunker_params, start=t0, log_json=args.log_json, - iec=use_iec_units(), ) metadata_collector = MetadataCollector( noatime=not args.atime, @@ -273,7 +266,6 @@ def create_inner(archive, cache, fso): show_progress=args.progress, sparse=args.sparse, log_json=args.log_json, - iec=use_iec_units(), file_status_printer=self.print_file_status, files_changed=args.files_changed, ) diff --git a/src/borg/archiver/extract_cmd.py b/src/borg/archiver/extract_cmd.py index f45b6338cf..2e6d07a02f 100644 --- a/src/borg/archiver/extract_cmd.py +++ b/src/borg/archiver/extract_cmd.py @@ -10,7 +10,6 @@ from ..helpers import remove_surrogates from ..helpers import HardLinkManager from ..helpers import log_multi -from ..helpers import use_iec_units from ..helpers import ProgressIndicatorPercent from ..helpers import BackupWarning, IncludePatternNeverMatchedWarning from ..helpers.argparsing import ArgumentParser @@ -122,10 +121,7 @@ def do_extract(self, args, repository, manifest, archive): pi.finish() if args.stats: - log_multi( - format_store_stats(repository.store.stats, iec=use_iec_units()), - logger=logging.getLogger("borg.output.stats"), - ) + log_multi(format_store_stats(repository.store.stats), logger=logging.getLogger("borg.output.stats")) def build_parser_extract(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog diff --git a/src/borg/archiver/info_cmd.py b/src/borg/archiver/info_cmd.py index 1405805c71..bb43706fc2 100644 --- a/src/borg/archiver/info_cmd.py +++ b/src/borg/archiver/info_cmd.py @@ -4,7 +4,7 @@ from ._common import with_repository from ..archive import Archive from ..constants import * # NOQA -from ..helpers import format_timedelta, json_print, basic_json_data, archivename_validator, use_iec_units +from ..helpers import format_timedelta, json_print, basic_json_data, archivename_validator from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -26,7 +26,7 @@ def do_info(self, args, repository, manifest, cache): output_data = [] for i, archive_info in enumerate(archive_infos, 1): - archive = Archive(manifest, archive_info.id, cache=cache, iec=use_iec_units()) + archive = Archive(manifest, archive_info.id, cache=cache) info = archive.info() if args.json: output_data.append(info) diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 5f172303b2..337071aff9 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -9,7 +9,7 @@ from ..constants import * # NOQA from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error from ..helpers import archivename_validator, int_or_interval, sig_int, timestamp -from ..helpers import json_print, basic_json_data, use_iec_units +from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser from ..manifest import ArchiveInfo, Manifest @@ -224,7 +224,7 @@ def do_prune(self, args, repository, manifest): format = "{archive}" else: format = os.environ.get("BORG_PRUNE_FORMAT", "{archive:<36} {time} [{id}]") - formatter = ArchiveFormatter(format, repository, manifest, manifest.key, iec=use_iec_units()) + formatter = ArchiveFormatter(format, repository, manifest, manifest.key) if args.json: output_data = [] diff --git a/src/borg/archiver/repo_list_cmd.py b/src/borg/archiver/repo_list_cmd.py index 2c353bd702..533b202195 100644 --- a/src/borg/archiver/repo_list_cmd.py +++ b/src/borg/archiver/repo_list_cmd.py @@ -4,7 +4,7 @@ from ._common import with_repository, Highlander from ..constants import * # NOQA -from ..helpers import BaseFormatter, ArchiveFormatter, json_print, basic_json_data, use_iec_units +from ..helpers import BaseFormatter, ArchiveFormatter, json_print, basic_json_data from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest @@ -26,9 +26,7 @@ def do_repo_list(self, args, repository, manifest): "BORG_REPO_LIST_FORMAT", "{id:.8} {time} {archive:<15} {tags:<10} {username:<10} {hostname:<10} {comment:.40}{NL}", ) - formatter = ArchiveFormatter( - format, repository, manifest, manifest.key, iec=use_iec_units(), deleted=args.deleted - ) + formatter = ArchiveFormatter(format, repository, manifest, manifest.key, deleted=args.deleted) output_data = [] diff --git a/src/borg/archiver/repo_space_cmd.py b/src/borg/archiver/repo_space_cmd.py index 37ed12d884..421d0d4943 100644 --- a/src/borg/archiver/repo_space_cmd.py +++ b/src/borg/archiver/repo_space_cmd.py @@ -26,7 +26,7 @@ def do_repo_space(self, args, repository): data = os.urandom(storage_space_reserve_object_size) # counter-act fs compression/dedup repository.store_store(f"config/space-reserve.{i}", data) size += len(data) - print(f"There is {format_file_size(size, iec=False)} reserved space in this repository now.") + print(f"There is {format_file_size(size)} reserved space in this repository now.") elif args.free_space: infos = repository.store_list("config") size = 0 @@ -35,7 +35,7 @@ def do_repo_space(self, args, repository): if info.name.startswith("space-reserve."): size += info.size repository.store_delete(f"config/{info.name}") - print(f"Freed {format_file_size(size, iec=False)} in the repository.") + print(f"Freed {format_file_size(size)} in the repository.") print("Now run borg prune or borg delete plus borg compact to free more space.") print("After that, do not forget to reserve space again for next time!") else: # print amount currently reserved @@ -45,7 +45,7 @@ def do_repo_space(self, args, repository): info = ItemInfo(*info) # RPC does not give namedtuple if info.name.startswith("space-reserve."): size += info.size - print(f"There is {format_file_size(size, iec=False)} reserved space in this repository.") + print(f"There is {format_file_size(size)} reserved space in this repository.") print("In case you want to change the amount, use --free first to free all reserved space,") print("then use --reserve with the desired amount.") diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 58e6095b40..7bd423b390 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -9,7 +9,6 @@ from ..helpers import HardLinkManager, IncludePatternNeverMatchedWarning from ..helpers import ProgressIndicatorPercent from ..helpers import dash_open -from ..helpers import use_iec_units from ..helpers import msgpack from ..helpers import create_filter_process from ..helpers import ChunkIteratorFileWrapper @@ -293,7 +292,6 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): chunker_params=args.chunker_params, show_progress=args.progress, log_json=args.log_json, - iec=use_iec_units(), file_status_printer=self.print_file_status, ) diff --git a/src/borg/cache.py b/src/borg/cache.py index 1d818e7b33..0f539adbb3 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -198,7 +198,6 @@ def __new__( warn_if_unencrypted=True, progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, - iec=False, archive_name=None, start_backup=None, ): @@ -207,7 +206,6 @@ def __new__( path=path, warn_if_unencrypted=warn_if_unencrypted, progress=progress, - iec=iec, cache_mode=cache_mode, archive_name=archive_name, start_backup=start_backup, @@ -971,13 +969,13 @@ def cleanup_archive_reference_caches(repository, stale_hex_ids: set) -> None: logger.debug(f"Removed {len(stale_hex_ids)} stale archive references caches.") -def scan_archive_references(manifest, archive_id: bytes, *, iec: bool = False): +def scan_archive_references(manifest, archive_id: bytes): """Open the archive and scan its items, collecting the objects it references (id -> plaintext size) plus its source file count and content size (both counted per occurrence, like a full scan). Opening the archive fetches and decrypts its metadata and item-metadata objects.""" from .archive import Archive # avoid circular import (archive.py imports from cache.py) - archive = Archive(manifest, archive_id, iec=iec) + archive = Archive(manifest, archive_id) ids = HashTableNT(key_size=32, value_type=ArchiveReferenceEntry, value_format=ArchiveReferenceEntryFormat) # archive metadata objects: only their ids matter for GC, their content size is unknown here # and not part of the source data size, so record them with size 0. @@ -996,9 +994,7 @@ def scan_archive_references(manifest, archive_id: bytes, *, iec: bool = False): return ArchiveReferences(file_count=file_count, content_size=content_size, ids=ids) -def get_archive_references( - repository, manifest, archive_id: bytes, *, cached: bool, iec: bool = False, store: bool = True -): +def get_archive_references(repository, manifest, archive_id: bytes, *, cached: bool, store: bool = True): """Return what the archive references, read from its per-archive cache in the repo if present, else computed by scanning the archive and (when *store*) cached for next time. @@ -1008,7 +1004,7 @@ def get_archive_references( """ references = load_archive_references(repository, archive_id) if cached else None if references is None: - references = scan_archive_references(manifest, archive_id, iec=iec) + references = scan_archive_references(manifest, archive_id) if store: store_archive_references(repository, archive_id, references) return references @@ -1119,7 +1115,6 @@ def __init__( warn_if_unencrypted=True, progress=False, cache_mode=FILES_CACHE_MODE_DISABLED, - iec=False, archive_name=None, start_backup=None, ): diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index fdbdfbd480..3a26ee8419 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -39,7 +39,7 @@ partial_format, DatetimeWrapper, ) -from .parseformat import format_file_size, parse_file_size, FileSize, use_iec_units +from .parseformat import format_file_size, parse_file_size, FileSize, get_size_units from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator from .parseformat import format_line, replace_placeholders, PlaceholderError, relative_time_marker_validator from .parseformat import format_archive, parse_stringified_list, clean_lines diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index bfd6f62e80..7280b41bda 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -502,25 +502,36 @@ def SortBySpec(text): return text.replace("timestamp", "ts").replace("archive", "name") -def use_iec_units(): - """Shall sizes be formatted using IEC units (1KiB = 1024B)? See BORG_IEC.""" - return os.environ.get("BORG_IEC", "no").strip().lower() in ("yes", "true", "1") - - -def format_file_size(v, precision=2, sign=False, iec=False): - """Format file size into a human friendly format""" - fn = sizeof_fmt_iec if iec else sizeof_fmt_decimal +SIZE_UNITS = ("si", "iec", "raw") + +_warned_units: set[str] = set() # invalid BORG_UNITS values already complained about + + +def get_size_units(): + """Which units shall sizes be formatted with: "si" (default), "iec" or "raw"? See BORG_UNITS.""" + units = os.environ.get("BORG_UNITS", "").strip().lower() + if units in SIZE_UNITS: + return units + if units and units not in _warned_units: + _warned_units.add(units) + logger.warning(f"Invalid BORG_UNITS value {units!r}, must be one of {', '.join(SIZE_UNITS)}. Ignoring it.") + return "si" + + +def format_file_size(v, precision=2, sign=False): + """Format file size into a human friendly format, using the units requested via BORG_UNITS.""" + units = get_size_units() + if units == "raw": + # exact byte counts, so scripts can easily parse the output + v = round(v) # v might be a float, e.g. a throughput value + return f"{v:{'+' if sign and v > 0 else ''}d} B" + fn = sizeof_fmt_iec if units == "iec" else sizeof_fmt_decimal return fn(v, suffix="B", sep=" ", precision=precision, sign=sign) class FileSize(int): - def __new__(cls, value, iec=False): - obj = int.__new__(cls, value) - obj.iec = iec - return obj - def __format__(self, format_spec): - return format_file_size(int(self), iec=self.iec).__format__(format_spec) + return format_file_size(int(self)).__format__(format_spec) def parse_file_size(s): @@ -979,7 +990,7 @@ class ArchiveFormatter(BaseFormatter): ("size", "nfiles"), ) - def __init__(self, format, repository, manifest, key, *, iec=False, deleted=False): + def __init__(self, format, repository, manifest, key, *, deleted=False): static_data = {} | self.FIXED_KEYS # here could be stuff on repo level, above archive level super().__init__(format, static_data) self.repository = repository @@ -989,7 +1000,6 @@ def __init__(self, format, repository, manifest, key, *, iec=False, deleted=Fals self.id = None self._archive = None self.deleted = deleted # True if we want to deal with deleted archives. - self.iec = iec self.format_keys = {f[1] for f in Formatter().parse(format)} self.call_keys = { "hostname": partial(self.get_meta, "hostname", ""), @@ -1031,7 +1041,7 @@ def archive(self): if self._archive is None or self._archive.id != self.id: from ..archive import Archive - self._archive = Archive(self.manifest, self.id, iec=self.iec, deleted=self.deleted) + self._archive = Archive(self.manifest, self.id, deleted=self.deleted) return self._archive def get_meta(self, key, default=None): diff --git a/src/borg/testsuite/archiver/compact_cmd_test.py b/src/borg/testsuite/archiver/compact_cmd_test.py index 88e2bf5620..f7f6efb93a 100644 --- a/src/borg/testsuite/archiver/compact_cmd_test.py +++ b/src/borg/testsuite/archiver/compact_cmd_test.py @@ -127,7 +127,7 @@ def test_compact_interrupted_does_not_poison_chunk_index(archivers, request, mon repository = open_repository(archiver) with repository: manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) - gc = ArchiveGarbageCollector(repository, manifest, stats=stats, iec=False, threshold=40.0) + gc = ArchiveGarbageCollector(repository, manifest, stats=stats, threshold=40.0) def interrupt(): raise KeyboardInterrupt("simulated interruption before save_chunk_index") @@ -171,7 +171,7 @@ def test_compact_soft_interrupt_persists_valid_index(archivers, request, monkeyp assert len(pack_names_before) >= 2 # need several packs to observe an early stop manifest = Manifest.load(repository, (Manifest.Operation.DELETE,)) - gc = ArchiveGarbageCollector(repository, manifest, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest, stats=False, threshold=10) original_store_delete = repository.store_delete calls = [] @@ -228,7 +228,7 @@ def test_compact_packs_respects_threshold(tmp_path): flags = ChunkIndex.F_USED if H(i) in used else ChunkIndex.F_NONE repository.chunks[H(i)] = entry._replace(flags=flags) - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=40) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=40) gc.chunks = repository.chunks gc.compact_packs() @@ -278,7 +278,7 @@ def test_compact_superseded_duplicate(tmp_path): flags = ChunkIndex.F_USED if H(i) in used else ChunkIndex.F_NONE repository.chunks[H(i)] = entry._replace(flags=flags) - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -312,7 +312,7 @@ def test_compact_keeps_orphan_pack(tmp_path): repository.store_store(orphan_key, b"orphan pack bytes") assert "ab" * 32 in [info.name for info in repository.store_list("packs")] - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -342,7 +342,7 @@ def test_compact_keeps_unindexed_waste(tmp_path): # ... but H(1)'s big object becomes an unindexed superseded span (well over threshold if counted). del repository.chunks[H(1)] - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -377,7 +377,7 @@ def test_compact_reclaims_indexed_waste_only(tmp_path): repository.chunks[H(2)] = repository.chunks[H(2)]._replace(flags=ChunkIndex.F_USED) del repository.chunks[H(3)] # its bytes remain in unindexed_pack as unindexed data - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -439,7 +439,7 @@ def test_compact_keeps_stale_index_entries(tmp_path): repository.chunks[H(0)] = repository.chunks[H(0)]._replace(flags=ChunkIndex.F_USED) repository.store_delete("packs/" + bin_to_hex(gone_pack)) # delete the pack file the index still references - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -460,7 +460,7 @@ def test_compact_skips_oversized_index_entry(tmp_path): entry = repository.chunks[H(0)] repository.chunks[H(0)] = entry._replace(flags=ChunkIndex.F_USED, obj_size=entry.obj_size + 10000) - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() @@ -492,7 +492,7 @@ def test_compact_packs_merges_tiny_packs(tmp_path, monkeypatch): total_bytes = sum(repository.store.info("packs/" + name).size for name in packs_before) assert total_bytes >= repository.pack_max_size # combined size crosses the merge threshold - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() assert gc.store_changed is True # the merge changed the store @@ -510,7 +510,7 @@ def test_compact_packs_merges_tiny_packs(tmp_path, monkeypatch): # a merged full-size pack is no longer tiny (the tiny limit is pack_max_size // 2 here), so a # second compact finds nothing to merge and leaves the store unchanged. - gc2 = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc2 = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc2.chunks = repository.chunks gc2.compact_packs() assert gc2.store_changed is False @@ -537,7 +537,7 @@ def test_compact_packs_below_merge_size_gate_leaves_tiny_packs(tmp_path, monkeyp packs_before = {info.name for info in repository.store_list("packs")} assert len(packs_before) == 3 - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() assert gc.store_changed is False # combined tiny bytes stay far below one full pack: leave them alone @@ -566,7 +566,7 @@ def test_compact_packs_below_all_packs_gate_changes_nothing(tmp_path): packs_before = {info.name for info in repository.store_list("packs")} assert len(packs_before) == 2 - gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, iec=False, threshold=10) + gc = ArchiveGarbageCollector(repository, manifest=None, stats=False, threshold=10) gc.chunks = repository.chunks gc.compact_packs() assert gc.store_changed is False # below the all-packs gate: nothing was touched diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index b00f6fff89..d30f4c0294 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -29,7 +29,7 @@ swidth_slice, eval_escapes, ChunkerParams, - use_iec_units, + get_size_units, normalize_local_path, ) from ...helpers.time import format_timedelta, parse_timestamp @@ -576,30 +576,85 @@ def test_file_size(size, fmt): (-(2**20), "-1.00 MiB"), ], ) -def test_file_size_iec(size, fmt): +def test_file_size_iec(monkeypatch, size, fmt): """test the size formatting routines""" - assert format_file_size(size, iec=True) == fmt + monkeypatch.setenv("BORG_UNITS", "iec") + assert format_file_size(size) == fmt + + +@pytest.mark.parametrize( + "units, expected", + [ + (None, "si"), # BORG_UNITS not set + ("", "si"), + ("si", "si"), + ("iec", "iec"), + ("raw", "raw"), + ("IEC", "iec"), # value is case insensitive + (" raw ", "raw"), # and gets stripped + ("bytes", "si"), # invalid value: warn and ignore + ("yes", "si"), # the removed BORG_IEC's value is not accepted either + ], +) +def test_get_size_units(monkeypatch, units, expected): + """size units are requested via the BORG_UNITS environment variable""" + monkeypatch.delenv("BORG_UNITS", raising=False) + if units is not None: + monkeypatch.setenv("BORG_UNITS", units) + assert get_size_units() == expected + + +def test_borg_iec_is_gone(monkeypatch): + """the removed BORG_IEC environment variable has no effect any more""" + monkeypatch.delenv("BORG_UNITS", raising=False) + monkeypatch.setenv("BORG_IEC", "yes") + assert get_size_units() == "si" + assert format_file_size(2**20) == "1.05 MB" + + +def test_get_size_units_invalid_warns(monkeypatch, caplog): + """an invalid BORG_UNITS value is complained about, but only once""" + from ...helpers import parseformat + + monkeypatch.delenv("BORG_UNITS", raising=False) + monkeypatch.setattr(parseformat, "_warned_units", set()) + monkeypatch.setenv("BORG_UNITS", "kibibytes") + with caplog.at_level("WARNING"): + assert get_size_units() == "si" + assert get_size_units() == "si" + assert len([record for record in caplog.records if "kibibytes" in record.message]) == 1 @pytest.mark.parametrize( - "value, expected", + "size, kwargs, fmt", [ - (None, False), - ("", False), - ("no", False), - ("0", False), - ("yes", True), - ("YES", True), - ("true", True), - ("1", True), + (0, {}, "0 B"), + (1, {}, "1 B"), + (1234, {}, "1234 B"), # not scaled down to 1.23 kB + (10**15, {}, "1000000000000000 B"), + (-1234, {}, "-1234 B"), + (1234, dict(precision=0), "1234 B"), # precision does not matter + (1234, dict(sign=True), "+1234 B"), + (-1234, dict(sign=True), "-1234 B"), + (0, dict(sign=True), "0 B"), + (1234.56, {}, "1235 B"), # a float (e.g. a throughput value) is rounded ], ) -def test_use_iec_units(monkeypatch, value, expected): - """IEC units are requested via the BORG_IEC environment variable""" - monkeypatch.delenv("BORG_IEC", raising=False) - if value is not None: - monkeypatch.setenv("BORG_IEC", value) - assert use_iec_units() is expected +def test_file_size_raw(monkeypatch, size, kwargs, fmt): + """BORG_UNITS=raw gives exact byte counts, so scripts can easily parse them""" + monkeypatch.setenv("BORG_UNITS", "raw") + assert format_file_size(size, **kwargs) == fmt + + +@pytest.mark.parametrize( + "units, fmt", [(None, "1.05 MB"), ("si", "1.05 MB"), ("iec", "1.00 MiB"), ("raw", "1048576 B")] # si is the default +) +def test_file_size_units_from_env(monkeypatch, units, fmt): + """format_file_size uses the units requested via BORG_UNITS""" + monkeypatch.delenv("BORG_UNITS", raising=False) + if units is not None: + monkeypatch.setenv("BORG_UNITS", units) + assert format_file_size(2**20) == fmt @pytest.mark.parametrize(