Deduplicate the recursive file enumeration when installing packages. (#1770)

Co-authored-by: Robert Schumacher <[email protected]>

Author: BillyONeal

include/vcpkg/base/files.h

@@ -155,6 +155,11 @@ namespace vcpkg
         std::vector<Path> get_files_recursive(const Path& dir, LineInfo li) const;
         ExpectedL<std::vector<Path>> try_get_files_recursive(const Path& dir) const;
 
+        virtual std::vector<Path> get_files_recursive_lexically_proximate(const Path& dir,
+                                                                          std::error_code& ec) const = 0;
+        std::vector<Path> get_files_recursive_lexically_proximate(const Path& dir, LineInfo li) const;
+        ExpectedL<std::vector<Path>> try_get_files_recursive_lexically_proximate(const Path& dir) const;
+
         virtual std::vector<Path> get_files_non_recursive(const Path& dir, std::error_code& ec) const = 0;
         std::vector<Path> get_files_non_recursive(const Path& dir, LineInfo li) const;
         ExpectedL<std::vector<Path>> try_get_files_non_recursive(const Path& dir) const;

include/vcpkg/base/path.h

@@ -56,4 +56,7 @@ namespace vcpkg
     private:
         std::string m_str;
     };
-}
+
+    // attempt to parse str as a path and return the filename if it exists; otherwise, an empty view
+    StringView parse_filename(StringView str) noexcept;
+}

include/vcpkg/commands.export.h

@@ -4,6 +4,9 @@
 #include <vcpkg/fwd/vcpkgcmdarguments.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
+#include <string>
+#include <vector>
+
 namespace vcpkg
 {
     extern const CommandMetadata CommandExportMetadata;
@@ -14,4 +17,33 @@ namespace vcpkg
                                  Triplet host_triplet);
 
     void export_integration_files(const Path& raw_exported_dir_path, const VcpkgPaths& paths);
+
+    // treat the installed/triplet directory as if it were the packages directory by removing the triplet
+    // e.g.
+    // x64-windows/
+    // x64-windows/include/
+    // x64-windows/include/CONFLICT-A-HEADER-ONLY-CAPS.h
+    // x64-windows/include/CONFLICT-a-header-ONLY-mixed.h
+    // x64-windows/include/CONFLICT-a-header-ONLY-mixed2.h
+    // x64-windows/include/conflict-a-header-only-lowercase.h
+    // x64-windows/share/
+    // x64-windows/share/a-conflict/
+    // x64-windows/share/a-conflict/copyright
+    // x64-windows/share/a-conflict/vcpkg.spdx.json
+    // x64-windows/share/a-conflict/vcpkg_abi_info.txt
+    //
+    // becomes
+    //
+    // include
+    // include/CONFLICT-A-HEADER-ONLY-CAPS.h
+    // include/CONFLICT-a-header-ONLY-mixed.h
+    // include/CONFLICT-a-header-ONLY-mixed2.h
+    // include/conflict-a-header-only-lowercase.h
+    // share
+    // share/a-conflict
+    // share/a-conflict/copyright
+    // share/a-conflict/vcpkg.spdx.json
+    // share/a-conflict/vcpkg_abi_info.txt
+    std::vector<std::string> convert_list_to_proximate_files(std::vector<std::string>&& lines,
+                                                             StringView triplet_canonical_name);
 }

include/vcpkg/commands.install.h

@@ -64,23 +64,25 @@ namespace vcpkg
         void print_complete_message() const;
     };
 
-    struct InstallDir
-    {
-        static InstallDir from_destination_root(const InstalledPaths& ip, Triplet t, const BinaryParagraph& pgh);
-
-    private:
-        Path m_destination;
-        Path m_listfile;
-
-    public:
-        const Path& destination() const;
-        const Path& listfile() const;
-    };
-
+    // First, writes triplet_canonical_name / (including the trailing slash) to listfile. Then:
+    // For each directory in source_dir / proximate_files
+    //  * create directory destination_installed / triplet_canonical_name / proximate_file
+    //  * write a line in listfile triplet_canonical_name / proximate_file /  (note the trailing slash)
+    // For each regular file in source_dir / proximate_files
+    //  * copy source_dir / proximate_file -> destination_installed / triplet_canonical_name / proximate_file
+    //  * write a line in listfile triplet_canonical_name / proximate_file
+    // For each symlink or junction in source_dir / proximate_files:
+    //  * if hydrate == SymlinkHydrate::yes, resolve symlinks and follow the rules above, otherwise,
+    //    * copy the symlink or junction source_dir / proximate_file
+    //       -> destination_installed / triplet_canonical_name / proximate_file
+    //    * write a line in listfile triplet_canonical_name / proximate_file
+    //      (note *no* trailing slash, even for directory symlinks)
     void install_files_and_write_listfile(const Filesystem& fs,
                                           const Path& source_dir,
-                                          const std::vector<Path>& files,
-                                          const InstallDir& destination_dir,
+                                          const std::vector<std::string>& proximate_files,
+                                          const Path& destination_installed,
+                                          StringView triplet_canonical_name,
+                                          const Path& listfile,
                                           const SymlinkHydrate hydrate);
 
     struct CMakeUsageInfo

src/vcpkg-test/export.cpp

@@ -0,0 +1,72 @@
+#include <vcpkg-test/util.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/util.h>
+
+#include <vcpkg/commands.export.h>
+
+#include <string>
+#include <vector>
+
+using namespace vcpkg;
+
+TEST_CASE ("convert_list_to_proximate_files basic example", "[export]")
+{
+    std::vector<std::string> lines = {
+        "x64-windows/",
+        "x64-windows/include/", // directory (trailing slash removed)
+        "x64-windows/include/CONFLICT-A-HEADER-ONLY-CAPS.h",
+        "x64-windows/include/CONFLICT-a-header-ONLY-mixed.h",
+        "x64-windows/include/CONFLICT-a-header-ONLY-mixed2.h",
+        "x64-windows/include/conflict-a-header-only-lowercase.h",
+        "x64-windows/share/",            // directory (trailing slash removed)
+        "x64-windows/share/a-conflict/", // directory (trailing slash removed)
+        "x64-windows/share/a-conflict/copyright",
+        "x64-windows/share/a-conflict/vcpkg.spdx.json",
+        "x64-windows/share/a-conflict/vcpkg_abi_info.txt",
+    };
+
+    const auto result = convert_list_to_proximate_files(std::move(lines), "x64-windows");
+    const std::vector<std::string> expected = {
+        "include",
+        "include/CONFLICT-A-HEADER-ONLY-CAPS.h",
+        "include/CONFLICT-a-header-ONLY-mixed.h",
+        "include/CONFLICT-a-header-ONLY-mixed2.h",
+        "include/conflict-a-header-only-lowercase.h",
+        "share",
+        "share/a-conflict",
+        "share/a-conflict/copyright",
+        "share/a-conflict/vcpkg.spdx.json",
+        "share/a-conflict/vcpkg_abi_info.txt",
+    };
+    REQUIRE(result == expected);
+}
+
+TEST_CASE ("convert_list_to_proximate_files preserves order and trims trailing slashes", "[export]")
+{
+    std::vector<std::string> lines = {
+        "x64-windows/share/",           // -> share
+        "x64-windows/share/pkg/",       // -> share/pkg
+        "x64-windows/share/pkg/file",   // -> share/pkg/file (unchanged)
+        "x64-windows/include/",         // -> include
+        "x64-windows/include/file.h",   // -> include/file.h
+        "x64-windows/share/other/file2" // -> share/other/file2
+    };
+
+    auto result = convert_list_to_proximate_files(std::move(lines), "x64-windows");
+    const std::vector<std::string> expected = {
+        "share",
+        "share/pkg",
+        "share/pkg/file",
+        "include",
+        "include/file.h",
+        "share/other/file2",
+    };
+    REQUIRE(result == expected);
+}
+
+TEST_CASE ("convert_list_to_proximate_files empty input", "[export]")
+{
+    auto result = convert_list_to_proximate_files(std::vector<std::string>{}, "x64-windows");
+    REQUIRE(result.empty());
+}

src/vcpkg/base/files.cpp

@@ -6,6 +6,7 @@
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/message_sinks.h>
 #include <vcpkg/base/messages.h>
+#include <vcpkg/base/path.h>
 #include <vcpkg/base/span.h>
 #include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.h>
@@ -223,15 +224,6 @@ namespace
         return last;
     }
 
-    StringView parse_filename(const StringView str) noexcept
-    {
-        // attempt to parse str as a path and return the filename if it exists; otherwise, an empty view
-        const auto first = str.data();
-        const auto last = first + str.size();
-        const auto filename = find_filename(first, last);
-        return StringView(filename, static_cast<size_t>(last - filename));
-    }
-
     constexpr const char* find_extension(const char* const filename, const char* const ads) noexcept
     {
         // find dividing point between stem and extension in a generic format filename consisting of [filename, ads)
@@ -1367,6 +1359,14 @@ namespace vcpkg
 
     const char* to_printf_arg(const Path& p) noexcept { return p.m_str.c_str(); }
 
+    StringView parse_filename(const StringView str) noexcept
+    {
+        const auto first = str.data();
+        const auto last = first + str.size();
+        const auto filename = find_filename(first, last);
+        return StringView(filename, static_cast<size_t>(last - filename));
+    }
+
     bool is_symlink(FileType s) { return s == FileType::symlink || s == FileType::junction; }
     bool is_regular_file(FileType s) { return s == FileType::regular; }
     bool is_directory(FileType s) { return s == FileType::directory; }
@@ -1674,6 +1674,23 @@ namespace vcpkg
         return maybe_files;
     }
 
+    std::vector<Path> ReadOnlyFilesystem::get_files_recursive_lexically_proximate(const Path& dir, LineInfo li) const
+    {
+        return this->try_get_files_recursive_lexically_proximate(dir).value_or_exit(li);
+    }
+
+    ExpectedL<std::vector<Path>> ReadOnlyFilesystem::try_get_files_recursive_lexically_proximate(const Path& dir) const
+    {
+        std::error_code ec;
+        auto maybe_files = this->get_files_recursive_lexically_proximate(dir, ec);
+        if (ec)
+        {
+            return format_filesystem_call_error(ec, __func__, {dir});
+        }
+
+        return maybe_files;
+    }
+
     std::vector<Path> ReadOnlyFilesystem::get_files_non_recursive(const Path& dir, LineInfo li) const
     {
         return this->try_get_files_non_recursive(dir).value_or_exit(li);
@@ -2666,6 +2683,21 @@ namespace vcpkg
             return get_directories_impl<stdfs::recursive_directory_iterator>(dir, ec);
         }
 
+        virtual std::vector<Path> get_files_recursive_lexically_proximate(const Path& dir,
+                                                                          std::error_code& ec) const override
+        {
+            auto ret = this->get_files_recursive(dir, ec);
+            if (!ec)
+            {
+                const auto base = to_stdfs_path(dir);
+                for (auto& p : ret)
+                {
+                    p = from_stdfs_path(to_stdfs_path(p).lexically_proximate(base));
+                }
+            }
+            return ret;
+        }
+
         virtual std::vector<Path> get_directories_recursive_lexically_proximate(const Path& dir,
                                                                                 std::error_code& ec) const override
         {
@@ -2972,6 +3004,15 @@ namespace vcpkg
             return result;
         }
 
+        virtual std::vector<Path> get_files_recursive_lexically_proximate(const Path& dir,
+                                                                          std::error_code& ec) const override
+        {
+            std::vector<Path> result;
+            Path out_base;
+            get_files_recursive_impl(result, dir, out_base, ec, true, true, true);
+            return result;
+        }
+
         virtual std::vector<Path> get_files_non_recursive(const Path& dir, std::error_code& ec) const override
         {
             std::vector<Path> result;

src/vcpkg/commands.export.cpp

@@ -419,25 +419,17 @@ namespace
                 msg::println(msgExportingPackage, msg::package_name = action.spec);
 
                 const BinaryParagraph& binary_paragraph = action.core_paragraph().value_or_exit(VCPKG_LINE_INFO);
-
-                const InstallDir dirs =
-                    InstallDir::from_destination_root(export_paths, action.spec.triplet(), binary_paragraph);
+                const auto& triplet_canonical_name = action.spec.triplet().canonical_name();
 
                 auto lines =
                     fs.read_lines(paths.installed().listfile_path(binary_paragraph)).value_or_exit(VCPKG_LINE_INFO);
-                std::vector<Path> files;
-                for (auto&& suffix : lines)
-                {
-                    if (suffix.empty()) continue;
-                    if (suffix.back() == '/') suffix.pop_back();
-                    if (suffix == action.spec.triplet().to_string()) continue;
-                    files.push_back(paths.installed().root() / suffix);
-                }
-
+                auto proximate_files = convert_list_to_proximate_files(std::move(lines), triplet_canonical_name);
                 install_files_and_write_listfile(fs,
                                                  paths.installed().triplet_dir(action.spec.triplet()),
-                                                 files,
-                                                 dirs,
+                                                 proximate_files,
+                                                 export_paths.root(),
+                                                 triplet_canonical_name,
+                                                 export_paths.listfile_path(binary_paragraph),
                                                  opts.dereference_symlinks ? SymlinkHydrate::CopyData
                                                                            : SymlinkHydrate::CopySymlinks);
             }
@@ -602,4 +594,25 @@ namespace vcpkg
 
         Checks::exit_success(VCPKG_LINE_INFO);
     }
+
+    std::vector<std::string> convert_list_to_proximate_files(std::vector<std::string>&& lines,
+                                                             StringView triplet_canonical_name)
+    {
+        auto prefix_length = triplet_canonical_name.size() + 1; // +1 for the trailing '/'
+        std::vector<std::string> proximate_files;
+        for (auto&& suffix : lines)
+        {
+            if (suffix.size() <= prefix_length)
+            {
+                continue;
+            }
+
+            suffix.erase(0, prefix_length);
+            if (suffix.back() == '/') suffix.pop_back();
+            if (suffix.empty()) continue;
+            proximate_files.emplace_back(std::move(suffix));
+        }
+
+        return proximate_files;
+    }
 } // namespace vcpkg