Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion Documentation/git-repack.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ SYNOPSIS
'git repack' [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [-m]
[--window=<n>] [--depth=<n>] [--threads=<n>] [--keep-pack=<pack-name>]
[--write-midx[=<mode>]] [--name-hash-version=<n>] [--path-walk]
[--filter=<filter-spec>] [--drop-filtered [--dry-run]]
[--filter=<filter-spec> [--filter-to-loose=<dir>]] [--drop-filtered [--dry-run]]

DESCRIPTION
-----------
Expand Down Expand Up @@ -183,6 +183,18 @@ depth is 4095.
`objects` and `objects/info/alternates` sections of
linkgit:gitrepository-layout[5].

--filter-to-loose=<dir>::
Like `--filter-to`, but treat `<dir>` as an object directory
and write the filtered out objects there as loose objects
instead of as a pack. Cannot be combined with `--filter-to`
or `--max-pack-size`. Objects already present in `<dir>` are
not rewritten. All objects are written before any existing
pack is removed.

Loose objects in this repository that the filter excludes are
copied to `<dir>` as well, and with `-d` the source copies are
removed once everything has been written.

--drop-filtered::
Delete the local objects that match the `--filter` specification
instead of keeping them in a separate packfile, reclaiming the
Expand Down
32 changes: 28 additions & 4 deletions builtin/repack.c
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ static const char *const git_repack_usage[] = {
N_("git repack [-a] [-A] [-d] [-f] [-F] [-l] [-n] [-q] [-b] [-m]\n"
"[--window=<n>] [--depth=<n>] [--threads=<n>] [--keep-pack=<pack-name>]\n"
"[--write-midx[=<mode>]] [--name-hash-version=<n>] [--path-walk]\n"
"[--filter=<filter-spec>] [--drop-filtered [--dry-run]]"),
"[--filter=<filter-spec> [--filter-to-loose=<dir>]] [--drop-filtered [--dry-run]]"),
NULL
};

Expand Down Expand Up @@ -182,6 +182,8 @@ int cmd_repack(int argc,
const char *cruft_expiration = NULL;
const char *expire_to = NULL;
const char *filter_to = NULL;
const char *filter_to_loose = NULL;
struct oidset moved_loose = OIDSET_INIT;
const char *opt_window = NULL;
const char *opt_window_memory = NULL;
const char *opt_depth = NULL;
Expand Down Expand Up @@ -257,6 +259,8 @@ int cmd_repack(int argc,
N_("pack prefix to store a pack containing pruned objects")),
OPT_STRING(0, "filter-to", &filter_to, N_("dir"),
N_("pack prefix to store a pack containing filtered out objects")),
OPT_STRING(0, "filter-to-loose", &filter_to_loose, N_("dir"),
N_("object directory to write filtered out objects into as loose objects")),
OPT_BOOL(0, "drop-filtered", &drop_filtered,
N_("delete filtered out objects (requires --filter)")),
OPT_BOOL(0, "dry-run", &dry_run,
Expand All @@ -282,8 +286,16 @@ int cmd_repack(int argc,
po_args.depth = xstrdup_or_null(opt_depth);
po_args.threads = xstrdup_or_null(opt_threads);

die_for_incompatible_opt2(drop_filtered, "--drop-filtered",
!!filter_to, "--filter-to");
die_for_incompatible_opt3(drop_filtered, "--drop-filtered",
!!filter_to, "--filter-to",
!!filter_to_loose, "--filter-to-loose");
die_for_incompatible_opt2(!!filter_to, "--filter-to",
!!filter_to_loose, "--filter-to-loose");
die_for_incompatible_opt2(!!filter_to_loose, "--filter-to-loose",
!!po_args.max_pack_size, "--max-pack-size");

if (filter_to_loose)
filter_to = filter_to_loose;

if (dry_run && !drop_filtered)
die(_("--dry-run only takes effect with --drop-filtered"));
Expand Down Expand Up @@ -553,7 +565,8 @@ int cmd_repack(int argc,
strvec_pushf(&cmd.args, "--filter=%s",
expand_list_objects_filter_spec(&po_args.filter_options));
else if (filter_to)
die(_("option '%s' can only be used along with '%s'"), "--filter-to", "--filter");
die(_("option '%s' can only be used along with '%s'"),
filter_to_loose ? "--filter-to-loose" : "--filter-to", "--filter");

if (geometry.split_factor)
cmd.in = -1;
Expand Down Expand Up @@ -698,6 +711,7 @@ int cmd_repack(int argc,
struct write_pack_opts opts = {
.po_args = &po_args,
.destination = filter_to,
.loose = !!filter_to_loose,
.packdir = packdir,
.packtmp = packtmp,
};
Expand All @@ -708,6 +722,14 @@ int cmd_repack(int argc,
ret = write_filtered_pack(&opts, &existing, &names);
if (ret)
goto cleanup;

if (filter_to_loose) {
ret = move_loose_filtered_objects(repo,
&po_args.filter_options, filter_to,
&moved_loose);
if (ret)
goto cleanup;
}
}

string_list_sort(&names);
Expand Down Expand Up @@ -764,6 +786,7 @@ int cmd_repack(int argc,
if (show_progress)
opts |= PRUNE_PACKED_VERBOSE;
prune_packed_objects(repo, opts);
remove_moved_loose_objects(repo, &moved_loose);

if (!keep_unreachable &&
(!(pack_everything & LOOSEN_UNREACHABLE) ||
Expand All @@ -788,6 +811,7 @@ int cmd_repack(int argc,
string_list_clear(&keep_pack_list, 0);
string_list_clear(&names, 1);
oidset_clear(&drop_oids);
oidset_clear(&moved_loose);
existing_packs_release(&existing);
pack_geometry_release(&geometry);
pack_objects_args_release(&po_args);
Expand Down
152 changes: 150 additions & 2 deletions repack-filtered.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,64 @@
#include "list-objects-filter.h"
#include "odb.h"
#include "promisor-remote.h"
#include "copy.h"
#include "object-file.h"
#include "odb/source-files.h"
#include "oidset.h"
#include "path.h"
#include "sigchain.h"

static void start_unpack_objects(struct child_process *unpack,
const char *objdir)
{
strvec_push(&unpack->args, "unpack-objects");
strvec_push(&unpack->args, "-q");
strvec_pushf(&unpack->env, "GIT_OBJECT_DIRECTORY=%s", objdir);
strvec_push(&unpack->env, "GIT_ALTERNATE_OBJECT_DIRECTORIES");
unpack->git_cmd = 1;
unpack->in = -1;
if (start_command(unpack))
die(_("could not start unpack-objects for '%s'"), objdir);
}

int write_filtered_pack(const struct write_pack_opts *opts,
struct existing_packs *existing,
struct string_list *names)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct child_process unpack = CHILD_PROCESS_INIT;
struct string_list_item *item;
FILE *in;
int ret;
const char *caret;
const char *pack_prefix = write_pack_opts_pack_prefix(opts);

prepare_pack_objects(&cmd, opts->po_args, opts->destination);
if (opts->loose) {
sigchain_push(SIGPIPE, SIG_IGN);
start_unpack_objects(&unpack, opts->destination);
prepare_pack_objects(&cmd, opts->po_args, "--stdout");
} else {
prepare_pack_objects(&cmd, opts->po_args, opts->destination);
}

strvec_push(&cmd.args, "--stdin-packs");

for_each_string_list_item(item, &existing->kept_packs)
strvec_pushf(&cmd.args, "--keep-pack=%s", item->string);

cmd.in = -1;
if (opts->loose)
cmd.out = unpack.in;

ret = start_command(&cmd);
if (ret)
if (ret) {
if (opts->loose) {
close(unpack.in);
finish_command(&unpack);
sigchain_pop(SIGPIPE);
}
return ret;
}

/*
* Here 'names' contains only the pack(s) that were just
Expand All @@ -52,6 +86,17 @@ int write_filtered_pack(const struct write_pack_opts *opts,
fprintf(in, "%s%s.pack\n", caret, item->string);
fclose(in);

if (opts->loose) {
ret = finish_command(&cmd);
if (finish_command(&unpack))
ret = -1;
sigchain_pop(SIGPIPE);
if (ret)
ret = error(_("failed to write filtered out objects to '%s'"),
opts->destination);
return ret;
}

return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd,
names);
}
Expand Down Expand Up @@ -131,3 +176,106 @@ int enumerate_promisor_blobs(struct repository *repo,
oidset_clear(&all_promisor_blobs);
return ret;
}

static int collect_loose_oid(const struct object_id *oid,
const char *path UNUSED, void *cb_data)
{
oidset_insert(cb_data, oid);
return 0;
}

static void dest_loose_path(struct strbuf *buf, const char *objdir,
const struct object_id *oid)
{
const char *hex = oid_to_hex(oid);

strbuf_reset(buf);
strbuf_addf(buf, "%s/%.2s/%s", objdir, hex, hex + 2);
}

int move_loose_filtered_objects(struct repository *repo,
const struct list_objects_filter_options *filter,
const char *dest_objdir,
struct oidset *moved)
{
struct odb_source_files *files =
odb_source_files_downcast(repo->objects->source);
struct oidset loose = OIDSET_INIT;
struct oidset omitted = OIDSET_INIT;
struct oidset_iter iter;
const struct object_id *oid;
struct strbuf src = STRBUF_INIT;
struct strbuf dst = STRBUF_INIT;
struct strbuf tmp = STRBUF_INIT;
int ret = 0;

if (filter->choice != LOFC_BLOB_LIMIT)
return 0;

ret = for_each_loose_file_in_source(&files->dirs->loose->base,
collect_loose_oid, NULL, NULL,
&loose);
if (ret)
goto out;
ret = list_objects_filter__filter_oidset(repo, filter, &loose,
&omitted);
if (ret)
goto out;

oidset_iter_init(&omitted, &iter);
while ((oid = oidset_iter_next(&iter))) {
odb_loose_path(files->dirs->loose, &src, oid);
dest_loose_path(&dst, dest_objdir, oid);

/*
* Only the destination decides whether the copy is needed;
* the object being present in the source says nothing.
*/
if (!access(dst.buf, F_OK)) {
oidset_insert(moved, oid);
continue;
}

if (safe_create_leading_directories(repo, dst.buf) < 0) {
ret = error_errno(_("unable to create leading directories of '%s'"),
dst.buf);
goto out;
}
strbuf_reset(&tmp);
strbuf_addf(&tmp, "%s.tmp-%"PRIuMAX, dst.buf,
(uintmax_t)getpid());
if (copy_file(repo, tmp.buf, src.buf, 0444) ||
finalize_object_file(repo, tmp.buf, dst.buf)) {
unlink(tmp.buf);
ret = error(_("unable to copy loose object '%s' to '%s'"),
src.buf, dst.buf);
goto out;
}
oidset_insert(moved, oid);
}

out:
oidset_clear(&loose);
oidset_clear(&omitted);
strbuf_release(&src);
strbuf_release(&dst);
strbuf_release(&tmp);
return ret;
}

void remove_moved_loose_objects(struct repository *repo,
const struct oidset *moved)
{
struct odb_source_files *files =
odb_source_files_downcast(repo->objects->source);
struct strbuf path = STRBUF_INIT;
struct oidset_iter iter;
const struct object_id *oid;

oidset_iter_init(moved, &iter);
while ((oid = oidset_iter_next(&iter))) {
odb_loose_path(files->dirs->loose, &path, oid);
unlink(path.buf);
}
strbuf_release(&path);
}
14 changes: 14 additions & 0 deletions repack.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name,
struct write_pack_opts {
struct pack_objects_args *po_args;
const char *destination;
bool loose;
const char *packdir;
const char *packtmp;
};
Expand Down Expand Up @@ -167,6 +168,19 @@ int write_filtered_pack(const struct write_pack_opts *opts,
struct existing_packs *existing,
struct string_list *names);

/*
* Copy loose objects that the filter excludes into the loose-object store
* at `dest_objdir`, recording them in `moved`.
*/
int move_loose_filtered_objects(struct repository *repo,
const struct list_objects_filter_options *filter,
const char *dest_objdir,
struct oidset *moved);

/* Delete the source copies of objects copied by move_loose_filtered_objects. */
void remove_moved_loose_objects(struct repository *repo,
const struct oidset *moved);

int enumerate_promisor_blobs(struct repository *repo,
const struct list_objects_filter_options *filter,
struct oidset *to_drop);
Expand Down
Loading
Loading