diff --git a/Documentation/config/index.txt b/Documentation/config/index.txt index 23c7985eb40974..404e7bd37f80f3 100644 --- a/Documentation/config/index.txt +++ b/Documentation/config/index.txt @@ -1,3 +1,9 @@ +index.deleteSparseDirectories:: + When enabled, the cone mode sparse-checkout feature will delete + directories that are outside of the sparse-checkout cone, unless + such a directory contains an untracked, non-ignored file. Defaults + to true. + index.recordEndOfIndexEntries:: Specifies whether the index file should include an "End Of Index Entry" section. This reduces index load time on multiprocessor diff --git a/builtin/add.c b/builtin/add.c index 03ee5a3a7a2e35..620430cabf6f3d 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -5,6 +5,7 @@ */ #define USE_THE_INDEX_VARIABLE #include "builtin.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "lockfile.h" @@ -48,6 +49,7 @@ static int chmod_pathspec(struct pathspec *pathspec, char flip, int show_only) int err; if (!include_sparse && + !core_virtualfilesystem && (ce_skip_worktree(ce) || !path_in_sparse_checkout(ce->name, &the_index))) continue; @@ -128,8 +130,9 @@ static int refresh(int verbose, const struct pathspec *pathspec) if (!seen[i]) { const char *path = pathspec->items[i].original; - if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) || - !path_in_sparse_checkout(path, &the_index)) { + if (!core_virtualfilesystem && + (matches_skip_worktree(pathspec, i, &skip_worktree_seen) || + !path_in_sparse_checkout(path, &the_index))) { string_list_append(&only_match_skip_worktree, pathspec->items[i].original); } else { @@ -139,7 +142,11 @@ static int refresh(int verbose, const struct pathspec *pathspec) } } - if (only_match_skip_worktree.nr) { + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!core_virtualfilesystem && only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); ret = 1; } @@ -517,7 +524,11 @@ int cmd_add(int argc, const char **argv, const char *prefix) if (seen[i]) continue; - if (!include_sparse && + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!include_sparse && !core_virtualfilesystem && matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) { string_list_append(&only_match_skip_worktree, pathspec.items[i].original); @@ -541,7 +552,6 @@ int cmd_add(int argc, const char **argv, const char *prefix) } } - if (only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); exit_status = 1; diff --git a/builtin/gc.c b/builtin/gc.c index cf5a1e279b9cb3..d2d10b34e515e6 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1309,6 +1309,8 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts) char *lock_path = xstrfmt("%s/maintenance", r->objects->odb->path); if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) { + struct stat st; + struct strbuf lock_dot_lock = STRBUF_INIT; /* * Another maintenance command is running. * @@ -1319,6 +1321,25 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts) if (!opts->auto_flag && !opts->quiet) warning(_("lock file '%s' exists, skipping maintenance"), lock_path); + + /* + * Check timestamp on .lock file to see if we should + * delete it to recover from a fail state. + */ + strbuf_addstr(&lock_dot_lock, lock_path); + strbuf_addstr(&lock_dot_lock, ".lock"); + if (lstat(lock_dot_lock.buf, &st)) + warning_errno(_("unable to stat '%s'"), lock_dot_lock.buf); + else { + if (st.st_mtime < time(NULL) - (6 * 60 * 60)) { + if (unlink(lock_dot_lock.buf)) + warning_errno(_("unable to delete stale lock file")); + else + warning(_("deleted stale lock file")); + } + } + + strbuf_release(&lock_dot_lock); free(lock_path); return 0; } diff --git a/builtin/reset.c b/builtin/reset.c index 7a3e08bd1388b2..349a9d0d036c79 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -40,6 +40,8 @@ #include "add-interactive.h" #include "strbuf.h" #include "quote.h" +#include "dir.h" +#include "entry.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -156,9 +158,47 @@ static void update_index_from_diff(struct diff_queue_struct *q, for (i = 0; i < q->nr; i++) { int pos; + int respect_skip_worktree = 1; struct diff_filespec *one = q->queue[i]->one; + struct diff_filespec *two = q->queue[i]->two; int is_in_reset_tree = one->mode && !is_null_oid(&one->oid); + int is_missing = !(one->mode && !is_null_oid(&one->oid)); + int was_missing = !two->mode && is_null_oid(&two->oid); struct cache_entry *ce; + struct cache_entry *ceBefore; + struct checkout state = CHECKOUT_INIT; + + /* + * When using the virtual filesystem feature, the cache entries that are + * added here will not have the skip-worktree bit set. + * + * Without this code there is data that is lost because the files that + * would normally be in the working directory are not there and show as + * deleted for the next status or in the case of added files just disappear. + * We need to create the previous version of the files in the working + * directory so that they will have the right content and the next + * status call will show modified or untracked files correctly. + */ + if (core_virtualfilesystem && !file_exists(two->path)) + { + pos = index_name_pos(&the_index, two->path, strlen(two->path)); + if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) && + (is_missing || !was_missing)) + { + state.force = 1; + state.refresh_cache = 1; + state.istate = &the_index; + ceBefore = make_cache_entry(&the_index, two->mode, + &two->oid, two->path, + 0, 0); + if (!ceBefore) + die(_("make_cache_entry failed for path '%s'"), + two->path); + + checkout_entry(ceBefore, &state, NULL, NULL); + respect_skip_worktree = 0; + } + } if (!is_in_reset_tree && !intent_to_add) { remove_file_from_index(&the_index, one->path); @@ -177,8 +217,14 @@ static void update_index_from_diff(struct diff_queue_struct *q, * to properly construct the reset sparse directory. */ pos = index_name_pos(&the_index, one->path, strlen(one->path)); - if ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) || - (pos < 0 && !path_in_sparse_checkout(one->path, &the_index))) + + /* + * Do not add the SKIP_WORKTREE bit back if we populated the + * file on purpose in a virtual filesystem scenario. + */ + if (respect_skip_worktree && + ((pos >= 0 && ce_skip_worktree(the_index.cache[pos])) || + (pos < 0 && !path_in_sparse_checkout(one->path, &the_index)))) ce->ce_flags |= CE_SKIP_WORKTREE; if (!ce) diff --git a/builtin/rm.c b/builtin/rm.c index dff819ae5098ff..82501a3df8ada1 100644 --- a/builtin/rm.c +++ b/builtin/rm.c @@ -5,6 +5,7 @@ */ #define USE_THE_INDEX_VARIABLE #include "builtin.h" +#include "environment.h" #include "advice.h" #include "config.h" #include "lockfile.h" @@ -312,7 +313,7 @@ int cmd_rm(int argc, const char **argv, const char *prefix) for (i = 0; i < the_index.cache_nr; i++) { const struct cache_entry *ce = the_index.cache[i]; - if (!include_sparse && + if (!include_sparse && !core_virtualfilesystem && (ce_skip_worktree(ce) || !path_in_sparse_checkout(ce->name, &the_index))) continue; @@ -349,7 +350,11 @@ int cmd_rm(int argc, const char **argv, const char *prefix) *original ? original : "."); } - if (only_match_skip_worktree.nr) { + /* + * When using a virtual filesystem, we might re-add a path + * that is currently virtual and we want that to succeed. + */ + if (!core_virtualfilesystem && only_match_skip_worktree.nr) { advise_on_updating_sparse_paths(&only_match_skip_worktree); ret = 1; } diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 5c8ffb1f7598b0..75295951eb7974 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -111,7 +111,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix) static void clean_tracked_sparse_directories(struct repository *r) { - int i, was_full = 0; + int i, value, was_full = 0; struct strbuf path = STRBUF_INIT; size_t pathlen; struct string_list_item *item; @@ -127,6 +127,13 @@ static void clean_tracked_sparse_directories(struct repository *r) !r->index->sparse_checkout_patterns->use_cone_patterns) return; + /* + * Users can disable this behavior. + */ + if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) && + !value) + return; + /* * Use the sparse index as a data structure to assist finding * directories that are safe to delete. This conversion to a diff --git a/diff.c b/diff.c index ee3eb629e3dc5e..9797ad009ceb68 100644 --- a/diff.c +++ b/diff.c @@ -3994,6 +3994,13 @@ static int reuse_worktree_file(struct index_state *istate, if (!FAST_WORKING_DIRECTORY && !want_file && has_object_pack(oid)) return 0; + /* + * If this path does not match our sparse-checkout definition, + * then the file will not be in the working directory. + */ + if (!path_in_sparse_checkout(name, istate)) + return 0; + /* * Similarly, if we'd have to convert the file contents anyway, that * makes the optimization not worthwhile. diff --git a/dir.c b/dir.c index 45c7b9c3c00879..2f354a509ac00b 100644 --- a/dir.c +++ b/dir.c @@ -1533,6 +1533,13 @@ static int path_in_sparse_checkout_1(const char *path, enum pattern_match_result match = UNDECIDED; const char *end, *slash; + /* + * When using a virtual filesystem, there aren't really patterns + * to follow, but be extra careful to skip this check. + */ + if (core_virtualfilesystem) + return 1; + /* * We default to accepting a path if the path is empty, there are no * patterns, or the patterns are of the wrong type. diff --git a/read-cache.c b/read-cache.c index a08daf87dcd1ae..ea7e08f350b35e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -3943,7 +3943,7 @@ static void update_callback(struct diff_queue_struct *q, struct diff_filepair *p = q->queue[i]; const char *path = p->one->path; - if (!data->include_sparse && + if (!data->include_sparse && !core_virtualfilesystem && !path_in_sparse_checkout(path, data->index)) continue; diff --git a/repo-settings.c b/repo-settings.c index fb76ee4c69ca7e..b11ce98a5499bf 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -87,7 +87,7 @@ void prepare_repo_settings(struct repository *r) /* Boolean config or default, does not cascade (simple) */ repo_cfg_bool(r, "pack.usesparse", &r->settings.pack_use_sparse, 1); repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); - repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); + repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 1); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); repo_cfg_bool(r, "pack.usebitmapboundarytraversal", diff --git a/sequencer.c b/sequencer.c index adc9cfb4df39f2..1039fedec79599 100644 --- a/sequencer.c +++ b/sequencer.c @@ -699,7 +699,7 @@ static int do_recursive_merge(struct repository *r, o.branch2 = next ? next_label : "(empty tree)"; if (is_rebase_i(opts)) o.buffer_output = 2; - o.show_rename_progress = 1; + o.show_rename_progress = isatty(2); head_tree = parse_tree_indirect(head); next_tree = next ? repo_get_commit_tree(r, next) : empty_tree(r); diff --git a/t/perf/p2000-sparse-operations.sh b/t/perf/p2000-sparse-operations.sh index 96ed3e1d698815..376d3c6abc9c77 100755 --- a/t/perf/p2000-sparse-operations.sh +++ b/t/perf/p2000-sparse-operations.sh @@ -56,7 +56,7 @@ test_expect_success 'setup repo and indexes' ' git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v3 && ( cd full-v3 && - git sparse-checkout init --cone && + git sparse-checkout init --cone --no-sparse-index && git sparse-checkout set $SPARSE_CONE && git config index.version 3 && git update-index --index-version=3 && @@ -65,7 +65,7 @@ test_expect_success 'setup repo and indexes' ' git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v4 && ( cd full-v4 && - git sparse-checkout init --cone && + git sparse-checkout init --cone --no-sparse-index && git sparse-checkout set $SPARSE_CONE && git config index.version 4 && git update-index --index-version=4 && diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 9ceb17f9118919..8a9ae66d578fda 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -782,6 +782,10 @@ test_expect_success 'cone mode clears ignored subdirectories' ' git -C repo status --porcelain=v2 >out && test_must_be_empty out && + git -C repo -c index.deleteSparseDirectories=false sparse-checkout reapply && + test_path_is_dir repo/folder1 && + test_path_is_dir repo/deep/deeper2 && + git -C repo sparse-checkout reapply && test_path_is_missing repo/folder1 && test_path_is_missing repo/deep/deeper2 && diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 8a95adf4b50de6..542bb96c82982a 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -155,6 +155,7 @@ init_repos () { git -C sparse-index reset --hard && # initialize sparse-checkout definitions + git -C sparse-checkout config index.sparse false && git -C sparse-checkout sparse-checkout init --cone && git -C sparse-checkout sparse-checkout set deep && git -C sparse-index sparse-checkout init --cone --sparse-index && @@ -517,6 +518,43 @@ test_expect_success 'diff --cached' ' test_all_match git diff --cached ' +test_expect_success 'diff partially-staged' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + # Add file within cone + test_all_match git sparse-checkout set deep && + run_on_all ../edit-contents deep/testfile && + test_all_match git add deep/testfile && + run_on_all ../edit-contents deep/testfile && + + test_all_match git diff && + test_all_match git diff --staged && + + # Add file outside cone + test_all_match git reset --hard && + run_on_all mkdir newdirectory && + run_on_all ../edit-contents newdirectory/testfile && + test_all_match git sparse-checkout set newdirectory && + test_all_match git add newdirectory/testfile && + run_on_all ../edit-contents newdirectory/testfile && + test_all_match git sparse-checkout set && + + test_all_match git diff && + test_all_match git diff --staged && + + # Merge conflict outside cone + test_all_match git reset --hard && + test_all_match git checkout merge-left && + test_all_match test_must_fail git merge merge-right && + + test_all_match git diff && + test_all_match git diff --staged +' + # NEEDSWORK: sparse-checkout behaves differently from full-checkout when # running this test with 'df-conflict-2' after 'df-conflict-1'. test_expect_success 'diff with renames and conflicts' ' @@ -991,7 +1029,9 @@ test_expect_success 'read-tree --merge with directory-file conflicts' ' test_expect_success 'merge, cherry-pick, and rebase' ' init_repos && - for OPERATION in "merge -m merge" cherry-pick "rebase --apply" "rebase --merge" + # microsoft/git specific: we need to use "quiet" mode + # to avoid different stderr for some rebases. + for OPERATION in "merge -m merge" cherry-pick "rebase -q --apply" "rebase -q --merge" do test_all_match git checkout -B temp update-deep && test_all_match git $OPERATION update-folder1 && @@ -1452,6 +1492,11 @@ test_expect_success 'sparse-index is not expanded' ' ensure_not_expanded reset --merge update-deep && ensure_not_expanded reset --hard && + echo a test change >>sparse-index/README.md && + ensure_not_expanded diff && + git -C sparse-index add README.md && + ensure_not_expanded diff --staged && + ensure_not_expanded reset base -- deep/a && ensure_not_expanded reset base -- nonexistent-file && ensure_not_expanded reset deepest -- deep && @@ -1772,6 +1817,46 @@ test_expect_success 'sparse index is not expanded: sparse-checkout' ' ensure_not_expanded sparse-checkout set ' +# NEEDSWORK: although the full repository's index is _not_ expanded as part of +# stash, a temporary index, which is _not_ sparse, is created when stashing and +# applying a stash of untracked files. As a result, the test reports that it +# finds an instance of `ensure_full_index`, but it does not carry with it the +# performance implications of expanding the full repository index. +test_expect_success 'sparse index is not expanded: stash -u' ' + init_repos && + + mkdir -p sparse-index/folder1 && + echo >>sparse-index/README.md && + echo >>sparse-index/a && + echo >>sparse-index/folder1/new && + + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index stash -u && + test_region index ensure_full_index trace2.txt && + + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index stash pop && + test_region index ensure_full_index trace2.txt +' + +# NEEDSWORK: similar to `git add`, untracked files outside of the sparse +# checkout definition are successfully stashed and unstashed. +test_expect_success 'stash -u outside sparse checkout definition' ' + init_repos && + + write_script edit-contents <<-\EOF && + echo text >>$1 + EOF + + run_on_sparse mkdir -p folder1 && + run_on_all ../edit-contents folder1/new && + test_all_match git stash -u && + test_all_match git status --porcelain=v2 && + + test_all_match git stash pop -q && + test_all_match git status --porcelain=v2 +' + # NEEDSWORK: a sparse-checkout behaves differently from a full checkout # in this scenario, but it shouldn't. test_expect_success 'reset mixed and checkout orphan' ' diff --git a/t/t7817-grep-sparse-checkout.sh b/t/t7817-grep-sparse-checkout.sh index eb595645657fad..db3004c4fe71c0 100755 --- a/t/t7817-grep-sparse-checkout.sh +++ b/t/t7817-grep-sparse-checkout.sh @@ -49,7 +49,7 @@ test_expect_success 'setup' ' echo "text" >B/b && git add A B && git commit -m sub && - git sparse-checkout init --cone && + git sparse-checkout init --cone --no-sparse-index && git sparse-checkout set B ) && diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 487e326b3fac12..5ab6263b792d3f 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -54,6 +54,23 @@ test_expect_success 'run [--auto|--quiet]' ' test_subcommand git gc --no-quiet err && + grep "lock file .* exists, skipping maintenance" err && + + test-tool chmtime =-22000 .git/objects/maintenance.lock && + git maintenance run --schedule=hourly --no-quiet 2>err && + grep "deleted stale lock file" err && + test_path_is_missing .git/objects/maintenance.lock && + + git maintenance run --schedule=hourly 2>err && + test_must_be_empty err +' + test_expect_success 'maintenance.auto config option' ' GIT_TRACE2_EVENT="$(pwd)/default" git commit --quiet --allow-empty -m 1 && test_subcommand git maintenance run --auto --quiet