Skip to content

Commit

Permalink
sketchy read order, also see #40
Browse files Browse the repository at this point in the history
  • Loading branch information
@4sval
4sval committed Jan 22, 2025
1 parent ff03587 commit 7f57432
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 17 deletions.
28 changes: 14 additions & 14 deletions CUE4Parse/FileProvider/Vfs/FileProviderDictionary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public class FileProviderDictionary : IReadOnlyDictionary<string, GameFile>

private readonly KeyEnumerable _keys;
private readonly ValueEnumerable _values;
private readonly ConcurrentBag<IReadOnlyDictionary<string, GameFile>> _indicesBag = new ();
private readonly ConcurrentBag<KeyValuePair<long, IReadOnlyDictionary<string, GameFile>>> _indicesBag = new ();

public readonly bool IsCaseInsensitive;
public IEnumerable<string> Keys => _keys;
Expand All @@ -30,7 +30,7 @@ public FileProviderDictionary(bool isCaseInsensitive)
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AddFiles(IReadOnlyDictionary<string, GameFile> newFiles)
public void AddFiles(IReadOnlyDictionary<string, GameFile> newFiles, long readOrder = 0)
{
foreach (var file in newFiles.Values)
{
Expand All @@ -39,7 +39,7 @@ public void AddFiles(IReadOnlyDictionary<string, GameFile> newFiles)
_byId[ioEntry.ChunkId.AsPackageId()] = file;
}
}
_indicesBag.Add(newFiles);
_indicesBag.Add(new KeyValuePair<long, IReadOnlyDictionary<string, GameFile>>(readOrder, newFiles));
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand All @@ -54,9 +54,9 @@ public bool ContainsKey(string key)
{
if (IsCaseInsensitive)
key = key.ToLowerInvariant();
foreach (var files in _indicesBag)
foreach (var files in _indicesBag.OrderByDescending(kvp => kvp.Key))
{
if (files.ContainsKey(key))
if (files.Value.ContainsKey(key))
return true;
}

Expand All @@ -68,9 +68,9 @@ public bool TryGetValue(string key, out GameFile value)
{
if (IsCaseInsensitive)
key = key.ToLowerInvariant();
foreach (var files in _indicesBag)
foreach (var files in _indicesBag.OrderByDescending(kvp => kvp.Key))
{
if (files.TryGetValue(key, out value))
if (files.Value.TryGetValue(key, out value))
return true;
}

Expand All @@ -96,9 +96,9 @@ public GameFile this[string path]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public IEnumerator<KeyValuePair<string, GameFile>> GetEnumerator()
{
foreach (var index in _indicesBag)
foreach (var index in _indicesBag.OrderByDescending(kvp => kvp.Key))
{
foreach (var entry in index)
foreach (var entry in index.Value)
{
yield return entry;
}
Expand All @@ -108,7 +108,7 @@ public IEnumerator<KeyValuePair<string, GameFile>> GetEnumerator()
[MethodImpl(MethodImplOptions.AggressiveInlining)]
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();

public int Count => _indicesBag.Sum(it => it.Count);
public int Count => _indicesBag.Sum(it => it.Value.Count);

private class KeyEnumerable : IEnumerable<string>
{
Expand All @@ -121,9 +121,9 @@ internal KeyEnumerable(FileProviderDictionary orig)

public IEnumerator<string> GetEnumerator()
{
foreach (var index in _orig._indicesBag)
foreach (var index in _orig._indicesBag.OrderByDescending(kvp => kvp.Key))
{
foreach (var key in index.Keys)
foreach (var key in index.Value.Keys)
{
yield return key;
}
Expand All @@ -144,9 +144,9 @@ internal ValueEnumerable(FileProviderDictionary orig)

public IEnumerator<GameFile> GetEnumerator()
{
foreach (var index in _orig._indicesBag)
foreach (var index in _orig._indicesBag.OrderByDescending(kvp => kvp.Key))
{
foreach (var key in index.Values)
foreach (var key in index.Value.Values)
{
yield return key;
}
Expand Down
1 change: 1 addition & 0 deletions CUE4Parse/UE4/IO/IoStoreReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ public override IReadOnlyDictionary<string, GameFile> Mount(bool caseInsensitive
sb.Append($" ({EncryptedFileCount} encrypted)");
if (MountPoint.Contains("/"))
sb.Append($", mount point: \"{MountPoint}\"");
sb.Append($", order {ReadOrder}");
sb.Append($", version {(int) TocResource.Header.Version} in {elapsed}");
log.Information(sb.ToString());
}
Expand Down
3 changes: 2 additions & 1 deletion CUE4Parse/UE4/Pak/PakFileReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ public override IReadOnlyDictionary<string, GameFile> Mount(bool caseInsensitive
sb.Append($" ({EncryptedFileCount} encrypted)");
if (MountPoint.Contains("/"))
sb.Append($", mount point: \"{MountPoint}\"");
sb.Append($", order {ReadOrder}");
sb.Append($", version {(int) Info.Version} in {elapsed}");
log.Information(sb.ToString());
}
Expand Down Expand Up @@ -368,4 +369,4 @@ public override void Dispose()
Ar.Dispose();
}
}
}
}
54 changes: 53 additions & 1 deletion CUE4Parse/UE4/VirtualFileSystem/AbstractVfsReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ public abstract partial class AbstractVfsReader : IVfsReader

public string Path { get; }
public string Name { get; }
public long ReadOrder { get; private set; }

public IReadOnlyDictionary<string, GameFile> Files { get; protected set; }
public virtual int FileCount => Files.Count;

Expand Down Expand Up @@ -66,7 +68,57 @@ protected void ValidateMountPoint(ref string mountPoint)
mountPoint = "/";
}

mountPoint = mountPoint.Substring(1);
mountPoint = mountPoint[1..];
VerifyReadOrder();
}

private void VerifyReadOrder()
{
ReadOrder = GetPakOrderFromPakFilePath();
if (!Name.EndsWith("_P.pak")) return;

var chunkVersionNumber = 1u;
var versionEndIndex = Name.LastIndexOf('_');
if (versionEndIndex != -1 && versionEndIndex > 0)
{
var versionStartIndex = Name.LastIndexOf('_', versionEndIndex - 1);
if (versionStartIndex != -1)
{
versionStartIndex++;
var versionString = Name.Substring(versionStartIndex, versionEndIndex - versionStartIndex);
if (int.TryParse(versionString, out var chunkVersionSigned) && chunkVersionSigned >= 1)
{
// Increment by one so that the first patch file still gets more priority than the base pak file
chunkVersionNumber = (uint)chunkVersionSigned + 1;
}
}
}
ReadOrder += 100 * chunkVersionNumber;
}

private int GetPakOrderFromPakFilePath()
{
// if (Path.StartsWith($"{FPaths.ProjectContentDir()}Paks/{FApp.GetProjectName()}-"))
// {
// // ProjectName/Content/Paks/ProjectName-
// return 4;
// }
// if (Path.StartsWith(FPaths.ProjectContentDir()))
// {
// // ProjectName/Content/
// return 3;
// }
// if (Path.StartsWith(FPaths.EngineContentDir()))
// {
// // Engine/Content/
// return 2;
// }
// if (Path.StartsWith(FPaths.ProjectSavedDir()))
// {
// // %LocalAppData%/ProjectName/Saved/
// return 1;
// }
return 3;
}

protected const int MAX_MOUNTPOINT_TEST_LENGTH = 128;
Expand Down
2 changes: 1 addition & 1 deletion CUE4Parse/UE4/VirtualFileSystem/AesVfsReaderForProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ public abstract partial class AbstractVfsReader
{
public void MountTo(FileProviderDictionary files, bool caseInsensitive, EventHandler<int>? vfsMounted = null)
{
files.AddFiles(Mount(caseInsensitive));
files.AddFiles(Mount(caseInsensitive), ReadOrder);
vfsMounted?.Invoke(this, files.Count);
}
}
Expand Down
1 change: 1 addition & 0 deletions CUE4Parse/UE4/VirtualFileSystem/IVfsReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public interface IVfsReader : IDisposable
{
public string Path { get; }
public string Name { get; }
public long ReadOrder { get; }

public IReadOnlyDictionary<string, GameFile> Files { get; }
public int FileCount { get; }
Expand Down

0 comments on commit 7f57432

Please sign in to comment.