From 7f5743227ce12ee86ace1ff70c1de86d17bd6d4d Mon Sep 17 00:00:00 2001 From: Asval Date: Wed, 22 Jan 2025 13:20:21 +0100 Subject: [PATCH] sketchy read order, also see #40 --- .../Vfs/FileProviderDictionary.cs | 28 +++++----- CUE4Parse/UE4/IO/IoStoreReader.cs | 1 + CUE4Parse/UE4/Pak/PakFileReader.cs | 3 +- .../VirtualFileSystem/AbstractVfsReader.cs | 54 ++++++++++++++++++- .../AesVfsReaderForProvider.cs | 2 +- CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs | 1 + 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/CUE4Parse/FileProvider/Vfs/FileProviderDictionary.cs b/CUE4Parse/FileProvider/Vfs/FileProviderDictionary.cs index 34acd42cd..9ab6b4dc4 100644 --- a/CUE4Parse/FileProvider/Vfs/FileProviderDictionary.cs +++ b/CUE4Parse/FileProvider/Vfs/FileProviderDictionary.cs @@ -16,7 +16,7 @@ public class FileProviderDictionary : IReadOnlyDictionary private readonly KeyEnumerable _keys; private readonly ValueEnumerable _values; - private readonly ConcurrentBag> _indicesBag = new (); + private readonly ConcurrentBag>> _indicesBag = new (); public readonly bool IsCaseInsensitive; public IEnumerable Keys => _keys; @@ -30,7 +30,7 @@ public FileProviderDictionary(bool isCaseInsensitive) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void AddFiles(IReadOnlyDictionary newFiles) + public void AddFiles(IReadOnlyDictionary newFiles, long readOrder = 0) { foreach (var file in newFiles.Values) { @@ -39,7 +39,7 @@ public void AddFiles(IReadOnlyDictionary newFiles) _byId[ioEntry.ChunkId.AsPackageId()] = file; } } - _indicesBag.Add(newFiles); + _indicesBag.Add(new KeyValuePair>(readOrder, newFiles)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -54,9 +54,9 @@ public bool ContainsKey(string key) { if (IsCaseInsensitive) key = key.ToLowerInvariant(); - foreach (var files in _indicesBag) + foreach (var files in _indicesBag.OrderByDescending(kvp => kvp.Key)) { - if (files.ContainsKey(key)) + if (files.Value.ContainsKey(key)) return true; } @@ -68,9 +68,9 @@ public bool TryGetValue(string key, out GameFile value) { if (IsCaseInsensitive) key = key.ToLowerInvariant(); - foreach (var files in _indicesBag) + foreach (var files in _indicesBag.OrderByDescending(kvp => kvp.Key)) { - if (files.TryGetValue(key, out value)) + if (files.Value.TryGetValue(key, out value)) return true; } @@ -96,9 +96,9 @@ public GameFile this[string path] [MethodImpl(MethodImplOptions.AggressiveInlining)] public IEnumerator> GetEnumerator() { - foreach (var index in _indicesBag) + foreach (var index in _indicesBag.OrderByDescending(kvp => kvp.Key)) { - foreach (var entry in index) + foreach (var entry in index.Value) { yield return entry; } @@ -108,7 +108,7 @@ public IEnumerator> GetEnumerator() [MethodImpl(MethodImplOptions.AggressiveInlining)] IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - public int Count => _indicesBag.Sum(it => it.Count); + public int Count => _indicesBag.Sum(it => it.Value.Count); private class KeyEnumerable : IEnumerable { @@ -121,9 +121,9 @@ internal KeyEnumerable(FileProviderDictionary orig) public IEnumerator GetEnumerator() { - foreach (var index in _orig._indicesBag) + foreach (var index in _orig._indicesBag.OrderByDescending(kvp => kvp.Key)) { - foreach (var key in index.Keys) + foreach (var key in index.Value.Keys) { yield return key; } @@ -144,9 +144,9 @@ internal ValueEnumerable(FileProviderDictionary orig) public IEnumerator GetEnumerator() { - foreach (var index in _orig._indicesBag) + foreach (var index in _orig._indicesBag.OrderByDescending(kvp => kvp.Key)) { - foreach (var key in index.Values) + foreach (var key in index.Value.Values) { yield return key; } diff --git a/CUE4Parse/UE4/IO/IoStoreReader.cs b/CUE4Parse/UE4/IO/IoStoreReader.cs index 9913d25de..0bf774651 100644 --- a/CUE4Parse/UE4/IO/IoStoreReader.cs +++ b/CUE4Parse/UE4/IO/IoStoreReader.cs @@ -298,6 +298,7 @@ public override IReadOnlyDictionary Mount(bool caseInsensitive sb.Append($" ({EncryptedFileCount} encrypted)"); if (MountPoint.Contains("/")) sb.Append($", mount point: \"{MountPoint}\""); + sb.Append($", order {ReadOrder}"); sb.Append($", version {(int) TocResource.Header.Version} in {elapsed}"); log.Information(sb.ToString()); } diff --git a/CUE4Parse/UE4/Pak/PakFileReader.cs b/CUE4Parse/UE4/Pak/PakFileReader.cs index 795f2a9fd..f74ed3cf5 100644 --- a/CUE4Parse/UE4/Pak/PakFileReader.cs +++ b/CUE4Parse/UE4/Pak/PakFileReader.cs @@ -142,6 +142,7 @@ public override IReadOnlyDictionary Mount(bool caseInsensitive sb.Append($" ({EncryptedFileCount} encrypted)"); if (MountPoint.Contains("/")) sb.Append($", mount point: \"{MountPoint}\""); + sb.Append($", order {ReadOrder}"); sb.Append($", version {(int) Info.Version} in {elapsed}"); log.Information(sb.ToString()); } @@ -368,4 +369,4 @@ public override void Dispose() Ar.Dispose(); } } -} \ No newline at end of file +} diff --git a/CUE4Parse/UE4/VirtualFileSystem/AbstractVfsReader.cs b/CUE4Parse/UE4/VirtualFileSystem/AbstractVfsReader.cs index e476ee23c..d3eddeb29 100644 --- a/CUE4Parse/UE4/VirtualFileSystem/AbstractVfsReader.cs +++ b/CUE4Parse/UE4/VirtualFileSystem/AbstractVfsReader.cs @@ -15,6 +15,8 @@ public abstract partial class AbstractVfsReader : IVfsReader public string Path { get; } public string Name { get; } + public long ReadOrder { get; private set; } + public IReadOnlyDictionary Files { get; protected set; } public virtual int FileCount => Files.Count; @@ -66,7 +68,57 @@ protected void ValidateMountPoint(ref string mountPoint) mountPoint = "/"; } - mountPoint = mountPoint.Substring(1); + mountPoint = mountPoint[1..]; + VerifyReadOrder(); + } + + private void VerifyReadOrder() + { + ReadOrder = GetPakOrderFromPakFilePath(); + if (!Name.EndsWith("_P.pak")) return; + + var chunkVersionNumber = 1u; + var versionEndIndex = Name.LastIndexOf('_'); + if (versionEndIndex != -1 && versionEndIndex > 0) + { + var versionStartIndex = Name.LastIndexOf('_', versionEndIndex - 1); + if (versionStartIndex != -1) + { + versionStartIndex++; + var versionString = Name.Substring(versionStartIndex, versionEndIndex - versionStartIndex); + if (int.TryParse(versionString, out var chunkVersionSigned) && chunkVersionSigned >= 1) + { + // Increment by one so that the first patch file still gets more priority than the base pak file + chunkVersionNumber = (uint)chunkVersionSigned + 1; + } + } + } + ReadOrder += 100 * chunkVersionNumber; + } + + private int GetPakOrderFromPakFilePath() + { + // if (Path.StartsWith($"{FPaths.ProjectContentDir()}Paks/{FApp.GetProjectName()}-")) + // { + // // ProjectName/Content/Paks/ProjectName- + // return 4; + // } + // if (Path.StartsWith(FPaths.ProjectContentDir())) + // { + // // ProjectName/Content/ + // return 3; + // } + // if (Path.StartsWith(FPaths.EngineContentDir())) + // { + // // Engine/Content/ + // return 2; + // } + // if (Path.StartsWith(FPaths.ProjectSavedDir())) + // { + // // %LocalAppData%/ProjectName/Saved/ + // return 1; + // } + return 3; } protected const int MAX_MOUNTPOINT_TEST_LENGTH = 128; diff --git a/CUE4Parse/UE4/VirtualFileSystem/AesVfsReaderForProvider.cs b/CUE4Parse/UE4/VirtualFileSystem/AesVfsReaderForProvider.cs index 65be1db0a..f2b2bd6b5 100644 --- a/CUE4Parse/UE4/VirtualFileSystem/AesVfsReaderForProvider.cs +++ b/CUE4Parse/UE4/VirtualFileSystem/AesVfsReaderForProvider.cs @@ -8,7 +8,7 @@ public abstract partial class AbstractVfsReader { public void MountTo(FileProviderDictionary files, bool caseInsensitive, EventHandler? vfsMounted = null) { - files.AddFiles(Mount(caseInsensitive)); + files.AddFiles(Mount(caseInsensitive), ReadOrder); vfsMounted?.Invoke(this, files.Count); } } diff --git a/CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs b/CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs index 9fb846ca4..2163cba34 100644 --- a/CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs +++ b/CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs @@ -10,6 +10,7 @@ public interface IVfsReader : IDisposable { public string Path { get; } public string Name { get; } + public long ReadOrder { get; } public IReadOnlyDictionary Files { get; } public int FileCount { get; }