Skip to content

Commit

Permalink
remote: allow specifying refs to prefetch
Browse files Browse the repository at this point in the history 
When using 'git fetch --prefetch', all fetchable refs are prefetched
by default. In large repositories with many refs, this can lead to
unnecessary network traffic and increased disk space use.

Introduce a new configuration option 'remote.<name>.prefetchref' that
allows users to specify specific ref patterns to be prefetched during
a 'git fetch --prefetch' operation.

The 'prefetchref' option accepts a space-separated list of ref
patterns (e.g., 'refs/heads/main !refs/heads/feature/*'). When the
'--prefetch' option is used with 'git fetch', only the refs matching
these patterns will be prefetched.

Signed-off-by: Shubham Kanodia <[email protected]>
  • Loading branch information
@pastelsky
pastelsky committed Oct 4, 2024
1 parent 2e7b89e commit bc74270
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 8 deletions.
7 changes: 7 additions & 0 deletions Documentation/config/remote.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ remote.<name>.fetch::
The default set of "refspec" for linkgit:git-fetch[1]. See
linkgit:git-fetch[1].

remote.<name>.prefetchref::
Specify the refs to be prefetched when fetching from this
remote. The value is a space-separated list of ref patterns
(e.g., "refs/heads/main !refs/heads/develop*"). This can be
used to optimize fetch operations by specifying exactly which
refs should be prefetched.

remote.<name>.push::
The default set of "refspec" for linkgit:git-push[1]. See
linkgit:git-push[1].
Expand Down
53 changes: 53 additions & 0 deletions builtin/fetch.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,32 @@ static void filter_prefetch_refspec(struct refspec *rs)
}
}

static int pattern_matches_ref(const char *pattern, const char *refname)
{
if (strchr(pattern, '*'))
return match_refspec_name_with_pattern(pattern, refname, NULL, NULL) != 0;
return strcmp(pattern, refname) == 0;
}

static int matches_prefetch_refs(const char *refname, const struct string_list *prefetch_refs)
{
int has_positive = 0, matched_positive = 0, matched_negative = 0;

for (int i = 0; i < prefetch_refs->nr; i++) {
const char *pattern = prefetch_refs->items[i].string;
int is_negative = (*pattern == '!');
if (is_negative) pattern++;
else has_positive = 1;

if (pattern_matches_ref(pattern, refname)) {
if (is_negative) matched_negative = 1;
else matched_positive = 1;
}
}

return has_positive ? (matched_positive && !matched_negative) : !matched_negative;
}

static struct ref *get_ref_map(struct remote *remote,
const struct ref *remote_refs,
struct refspec *rs,
Expand All @@ -501,7 +527,11 @@ static struct ref *get_ref_map(struct remote *remote,
struct hashmap existing_refs;
int existing_refs_populated = 0;

struct ref *prefetch_filtered_ref_map = NULL, **ref_map_tail = &prefetch_filtered_ref_map;
struct ref *next;

filter_prefetch_refspec(rs);

if (remote)
filter_prefetch_refspec(&remote->fetch);

Expand Down Expand Up @@ -610,6 +640,29 @@ static struct ref *get_ref_map(struct remote *remote,
else
ref_map = apply_negative_refspecs(ref_map, &remote->fetch);

/**
* Filter out advertised refs that we don't want to fetch during
* prefetch if a prefetchref config is set
*/

if (prefetch && remote->prefetch_refs.nr) {
prefetch_filtered_ref_map = NULL;
ref_map_tail = &prefetch_filtered_ref_map;

for (rm = ref_map; rm; rm = next) {
next = rm->next;
rm->next = NULL;

if (matches_prefetch_refs(rm->name, &remote->prefetch_refs)) {
*ref_map_tail = rm;
ref_map_tail = &rm->next;
} else {
free_one_ref(rm);
}
}
ref_map = prefetch_filtered_ref_map;
}

ref_map = ref_remove_duplicates(ref_map);

for (rm = ref_map; rm; rm = rm->next) {
Expand Down
24 changes: 16 additions & 8 deletions remote.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ static struct remote *make_remote(struct remote_state *remote_state,
ret->prune = -1; /* unspecified */
ret->prune_tags = -1; /* unspecified */
ret->name = xstrndup(name, len);
string_list_init_dup(&ret->prefetch_refs);
refspec_init(&ret->push, REFSPEC_PUSH);
refspec_init(&ret->fetch, REFSPEC_FETCH);

Expand All @@ -166,6 +167,7 @@ static void remote_clear(struct remote *remote)
free((char *)remote->uploadpack);
FREE_AND_NULL(remote->http_proxy);
FREE_AND_NULL(remote->http_proxy_authmethod);
string_list_clear(&remote->prefetch_refs, 0);
}

static void add_merge(struct branch *branch, const char *name)
Expand Down Expand Up @@ -456,6 +458,12 @@ static int handle_config(const char *key, const char *value,
remote->prune = git_config_bool(key, value);
else if (!strcmp(subkey, "prunetags"))
remote->prune_tags = git_config_bool(key, value);
else if (!strcmp(subkey, "prefetchref")) {
if (!value)
return config_error_nonbool(key);
string_list_split(&remote->prefetch_refs, value, ' ', -1);
return 0;
}
else if (!strcmp(subkey, "url")) {
if (!value)
return config_error_nonbool(key);
Expand Down Expand Up @@ -868,7 +876,7 @@ struct strvec *push_url_of_remote(struct remote *remote)
return remote->pushurl.nr ? &remote->pushurl : &remote->url;
}

static int match_name_with_pattern(const char *key, const char *name,
int match_refspec_name_with_pattern(const char *key, const char *name,
const char *value, char **result)
{
const char *kstar = strchr(key, '*');
Expand Down Expand Up @@ -900,7 +908,7 @@ static int refspec_match(const struct refspec_item *refspec,
const char *name)
{
if (refspec->pattern)
return match_name_with_pattern(refspec->src, name, NULL, NULL);
return match_refspec_name_with_pattern(refspec->src, name, NULL, NULL);

return !strcmp(refspec->src, name);
}
Expand Down Expand Up @@ -969,7 +977,7 @@ static int query_matches_negative_refspec(struct refspec *rs, struct refspec_ite
const char *key = refspec->dst ? refspec->dst : refspec->src;
const char *value = refspec->src;

if (match_name_with_pattern(key, needle, value, &expn_name))
if (match_refspec_name_with_pattern(key, needle, value, &expn_name))
string_list_append_nodup(&reversed, expn_name);
} else if (refspec->matching) {
/* For the special matching refspec, any query should match */
Expand Down Expand Up @@ -1014,7 +1022,7 @@ static void query_refspecs_multiple(struct refspec *rs,
if (!refspec->dst || refspec->negative)
continue;
if (refspec->pattern) {
if (match_name_with_pattern(key, needle, value, result))
if (match_refspec_name_with_pattern(key, needle, value, result))
string_list_append_nodup(results, *result);
} else if (!strcmp(needle, key)) {
string_list_append(results, value);
Expand Down Expand Up @@ -1043,7 +1051,7 @@ int query_refspecs(struct refspec *rs, struct refspec_item *query)
if (!refspec->dst || refspec->negative)
continue;
if (refspec->pattern) {
if (match_name_with_pattern(key, needle, value, result)) {
if (match_refspec_name_with_pattern(key, needle, value, result)) {
query->force = refspec->force;
return 0;
}
Expand Down Expand Up @@ -1456,9 +1464,9 @@ static char *get_ref_match(const struct refspec *rs, const struct ref *ref,
const char *dst_side = item->dst ? item->dst : item->src;
int match;
if (direction == FROM_SRC)
match = match_name_with_pattern(item->src, ref->name, dst_side, &name);
match = match_refspec_name_with_pattern(item->src, ref->name, dst_side, &name);
else
match = match_name_with_pattern(dst_side, ref->name, item->src, &name);
match = match_refspec_name_with_pattern(dst_side, ref->name, item->src, &name);
if (match) {
matching_refs = i;
break;
Expand Down Expand Up @@ -2076,7 +2084,7 @@ static struct ref *get_expanded_map(const struct ref *remote_refs,

if (strchr(ref->name, '^'))
continue; /* a dereference item */
if (match_name_with_pattern(refspec->src, ref->name,
if (match_refspec_name_with_pattern(refspec->src, ref->name,
refspec->dst, &expn_name) &&
!ignore_symref_update(expn_name, &scratch)) {
struct ref *cpy = copy_ref(ref);
Expand Down
6 changes: 6 additions & 0 deletions remote.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "hashmap.h"
#include "refspec.h"
#include "strvec.h"
#include "string-list.h"

struct option;
struct transport_ls_refs_options;
Expand Down Expand Up @@ -77,6 +78,8 @@ struct remote {

struct refspec fetch;

struct string_list prefetch_refs;

/*
* The setting for whether to fetch tags (as a separate rule from the
* configured refspecs);
Expand Down Expand Up @@ -207,6 +210,9 @@ int count_refspec_match(const char *, struct ref *refs, struct ref **matched_ref

int check_ref_type(const struct ref *ref, int flags);

int match_refspec_name_with_pattern(const char *key, const char *name,
const char *value, char **result);

/*
* Free a single ref and its peer, or an entire list of refs and their peers,
* respectively.
Expand Down
85 changes: 85 additions & 0 deletions t/t7900-maintenance.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,91 @@ test_expect_success 'prefetch multiple remotes' '
test_subcommand git fetch remote2 $fetchargs <skip-remote1.txt
'

test_expect_success 'prefetch with positive prefetch ref patterns' '
test_create_repo filter-prefetch-positive &&
(
cd filter-prefetch-positive &&
test_commit initial &&
git clone . clone2 &&
git remote add remote2 "file://$(pwd)/clone2" &&
cd clone2 &&
git checkout -b feature && test_commit feature-commit-2 &&
git checkout -b wip/test && test_commit wip-test-commit-2 &&
git checkout -b topic/x && test_commit topic-x-commit-2 &&
git push -f origin feature wip/test topic/x &&
cd .. &&
git config remote.remote2.prefetchref "refs/heads/feature" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-positive.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote2 $fetchargs <prefetch-positive.txt &&
git rev-parse refs/prefetch/remotes/remote2/feature &&
test_must_fail git rev-parse refs/prefetch/remotes/remote2/wip/test &&
test_must_fail git rev-parse refs/prefetch/remotes/remote2/topic/x
)
'

test_expect_success 'prefetch with negative prefetch ref patterns' '
test_create_repo filter-prefetch-negative &&
(
cd filter-prefetch-negative &&
test_commit initial &&
git clone . clone3 &&
git remote add remote3 "file://$(pwd)/clone3" &&
cat .git/config &&
cd clone3 &&
git checkout -b feature && test_commit feature-commit-3 &&
git checkout -b wip/test && test_commit wip-test-commit-3 &&
git checkout -b topic/x && test_commit topic-x-commit-3 &&
git push -f origin feature wip/test topic/x &&
cd .. &&
git config remote.remote3.prefetchref "!refs/heads/wip/*" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-negative.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote3 $fetchargs <prefetch-negative.txt &&
git rev-parse refs/prefetch/remotes/remote3/feature &&
git rev-parse refs/prefetch/remotes/remote3/topic/x &&
test_must_fail git rev-parse refs/prefetch/remotes/remote3/wip/test
)
'

test_expect_success 'prefetch with positive & negative prefetch ref patterns' '
test_create_repo filter-prefetch-mixed &&
(
cd filter-prefetch-mixed &&
test_commit initial &&
git clone . clone4 &&
git remote add remote4 "file://$(pwd)/clone4" &&
cd clone4 &&
git checkout -b feature && test_commit feature-commit-4 &&
git checkout -b topic/x && test_commit topic-x-commit-4 &&
git checkout -b topic/y && test_commit topic-y-commit-4 &&
git push -f origin feature topic/x topic/y &&
cd .. &&
git config remote.remote4.prefetchref \
"refs/heads/topic/* !refs/heads/topic/y" &&
fetchargs="--prefetch --prune --no-tags --no-write-fetch-head \
--recurse-submodules=no --quiet" &&
GIT_TRACE2_EVENT="$(pwd)/prefetch-mixed.txt" \
git maintenance run --task=prefetch 2>/dev/null &&
test_subcommand git fetch remote4 $fetchargs <prefetch-mixed.txt &&
test_must_fail git rev-parse refs/prefetch/remotes/remote4/feature &&
test_must_fail git rev-parse refs/prefetch/remotes/remote4/topic/y &&
git rev-parse refs/prefetch/remotes/remote4/topic/x
)
'

test_expect_success 'loose-objects task' '
# Repack everything so we know the state of the object dir
git repack -adk &&
Expand Down

0 comments on commit bc74270

Please sign in to comment.