Skip to content

Add AsyncPCK support (+ Web implementation) #114690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adamscott wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Add AsyncPCK support (+ Web implementation) #114690

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
678cfcb
Add AsyncPCK support
adamscott Nov 13, 2025
0e8f958
Remove old code/dev remnants
adamscott Jan 8, 2026
101dfd9
Apply AThousandShips suggestions
adamscott Jan 8, 2026
3d766b5
Remove fuzzy search changes remnant of development
adamscott Jan 8, 2026
c4d712d
Remove superfluous include (thanks to AThousandShips)
adamscott Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions core/config/project_settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,11 +582,17 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f
return false;
}

if (p_pack == "res://") {
String pack = p_pack.trim_suffix("/");

if (pack == "res://") {
// Loading the resource directory as a pack source is reserved for internal use only.
return false;
}

if (pack.ends_with(".asyncpck")) {
PackedData::get_singleton()->add_pack_source(memnew(PackedSourceAsyncPCK));
}

if (!p_main_pack && !using_datapack && !OS::get_singleton()->get_resource_dir().is_empty()) {
// Add the project's resource file system to PackedData so directory access keeps working when
// the game is running without a main pack, like in the editor or on Android.
Expand All @@ -596,7 +602,7 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f
using_datapack = true;
}

bool ok = PackedData::get_singleton()->add_pack(p_pack, p_replace_files, p_offset) == OK;
bool ok = PackedData::get_singleton()->add_pack(pack, p_replace_files, p_offset) == OK;
if (!ok) {
return false;
}
Expand Down Expand Up @@ -755,7 +761,7 @@ Error ProjectSettings::_setup(const String &p_path, const String &p_main_pack, b
}
}

#ifdef ANDROID_ENABLED
#if defined(ANDROID_ENABLED) || defined(WEB_ENABLED)
// Attempt to load sparse PCK assets.
_load_resource_pack("res://assets.sparsepck", false, 0, true);
#endif
Expand Down
21 changes: 21 additions & 0 deletions core/core_bind.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,22 @@ void OS::remove_script_loggers(const ScriptLanguage *p_script) {
}
}

bool OS::async_pck_is_supported() const {
return ::OS::get_singleton()->async_pck_is_supported();
}

bool OS::async_pck_is_file_installable(const String &p_path) const {
return ::OS::get_singleton()->async_pck_is_file_installable(p_path);
}

Error OS::async_pck_install_file(const String &p_path) const {
return ::OS::get_singleton()->async_pck_install_file(p_path);
}

Dictionary OS::async_pck_install_file_get_status(const String &p_path) const {
return ::OS::get_singleton()->async_pck_install_file_get_status(p_path);
}

void OS::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_entropy", "size"), &OS::get_entropy);
ClassDB::bind_method(D_METHOD("get_system_ca_certificates"), &OS::get_system_ca_certificates);
Expand Down Expand Up @@ -851,6 +867,11 @@ void OS::_bind_methods() {
ClassDB::bind_method(D_METHOD("add_logger", "logger"), &OS::add_logger);
ClassDB::bind_method(D_METHOD("remove_logger", "logger"), &OS::remove_logger);

ClassDB::bind_method(D_METHOD("async_pck_is_supported"), &OS::async_pck_is_supported);
ClassDB::bind_method(D_METHOD("async_pck_is_file_installable", "path"), &OS::async_pck_is_file_installable);
ClassDB::bind_method(D_METHOD("async_pck_install_file", "path"), &OS::async_pck_install_file);
ClassDB::bind_method(D_METHOD("async_pck_install_file_get_status", "path"), &OS::async_pck_install_file_get_status);

ADD_PROPERTY(PropertyInfo(Variant::BOOL, "low_processor_usage_mode"), "set_low_processor_usage_mode", "is_in_low_processor_usage_mode");
ADD_PROPERTY(PropertyInfo(Variant::INT, "low_processor_usage_mode_sleep_usec"), "set_low_processor_usage_mode_sleep_usec", "get_low_processor_usage_mode_sleep_usec");
ADD_PROPERTY(PropertyInfo(Variant::BOOL, "delta_smoothing"), "set_delta_smoothing", "is_delta_smoothing_enabled");
Expand Down
5 changes: 5 additions & 0 deletions core/core_bind.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,11 @@ class OS : public Object {
void remove_logger(const Ref<Logger> &p_logger);
void remove_script_loggers(const ScriptLanguage *p_script);

bool async_pck_is_supported() const;
bool async_pck_is_file_installable(const String &p_path) const;
Error async_pck_install_file(const String &p_path) const;
Dictionary async_pck_install_file_get_status(const String &p_path) const;

static OS *get_singleton() { return singleton; }

OS();
Expand Down
133 changes: 96 additions & 37 deletions core/io/file_access_pack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,54 +46,59 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
return ERR_FILE_UNRECOGNIZED;
}

void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle, bool p_delta) {
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, BitField<PackedFile::PackedFileProperty> p_properties) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());

bool exists = files.has(pmd5);

PackedFile pf;
pf.encrypted = p_encrypted;
pf.bundle = p_bundle;
pf.delta = p_delta;
pf.properties = p_properties;
pf.pack = p_pkg_path;
pf.offset = p_ofs;
pf.size = p_size;
for (int i = 0; i < 16; i++) {
pf.md5[i] = p_md5[i];
}
memcpy(pf.md5, p_md5, 16);
pf.src = p_src;

if (p_delta) {
if (pf.properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA)) {
delta_patches[pmd5].push_back(pf);
} else if (!exists || p_replace_files) {
files[pmd5] = pf;
delta_patches[pmd5].clear();
}

if (!exists) {
// Search for directory.
PackedDir *cd = root;
if (exists) {
return;
}

if (simplified_path.contains_char('/')) { // In a subdirectory.
Vector<String> ds = simplified_path.get_base_dir().split("/");
// Search for directory.
PackedDir *cd = root;

for (int j = 0; j < ds.size(); j++) {
if (!cd->subdirs.has(ds[j])) {
PackedDir *pd = memnew(PackedDir);
pd->name = ds[j];
pd->parent = cd;
cd->subdirs[pd->name] = pd;
cd = pd;
} else {
cd = cd->subdirs[ds[j]];
}
if (simplified_path.contains_char('/')) { // In a subdirectory.
Vector<String> ds = simplified_path.get_base_dir().split("/");

for (int j = 0; j < ds.size(); j++) {
if (!cd->subdirs.has(ds[j])) {
PackedDir *pd = memnew(PackedDir);
pd->name = ds[j];
pd->parent = cd;
cd->subdirs[pd->name] = pd;
cd = pd;
} else {
cd = cd->subdirs[ds[j]];
}
}
String filename = simplified_path.get_file();
// Don't add as a file if the path points to a directory.
if (!filename.is_empty()) {
cd->files.insert(filename);
}
String filename = simplified_path.get_file();
// Don't add as a file if the path points to a directory.
if (!filename.is_empty()) {
cd->files.insert(filename);
}

if (p_properties.has_flag(PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC)) {
Ref<FileAccess> file = FileAccess::create_for_path(p_path);
if (!file->file_exists(p_path)) {
async_files[pmd5] = pf;
}
}
}
Expand Down Expand Up @@ -164,6 +169,26 @@ bool PackedData::has_delta_patches(const String &p_path) const {
return !E->value.is_empty();
}

String PackedData::get_file_pack_path(const String &p_path) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, PackedFile, PathMD5>::Iterator file_iterator = files.find(pmd5);
if (!file_iterator) {
return "";
}
return file_iterator->value.pack;
}

String PackedData::get_file_async_pack_path(const String &p_path) {
String simplified_path = p_path.simplify_path().trim_prefix("res://");
PathMD5 pmd5(simplified_path.md5_buffer());
HashMap<PathMD5, PackedFile, PathMD5>::Iterator file_iterator = async_files.find(pmd5);
if (!file_iterator) {
return "";
}
return file_iterator->value.pack;
}

HashSet<String> PackedData::get_file_paths() const {
HashSet<String> file_paths;
_get_file_paths(root, root->name, file_paths);
Expand Down Expand Up @@ -298,6 +323,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
bool enc_directory = (pack_flags & PACK_DIR_ENCRYPTED);
bool rel_filebase = (pack_flags & PACK_REL_FILEBASE); // Note: Always enabled for V3.
bool sparse_bundle = (pack_flags & PACK_SPARSE_BUNDLE);
bool async = (pack_flags & PACK_ASYNC);

uint64_t file_base = f->get_64();
if ((version == PACK_FORMAT_VERSION_V3) || (version == PACK_FORMAT_VERSION_V2 && rel_filebase)) {
Expand All @@ -324,9 +350,7 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,

Vector<uint8_t> key;
key.resize(32);
for (int i = 0; i < key.size(); i++) {
key.write[i] = script_encryption_key[i];
}
memcpy(key.ptrw(), script_encryption_key, 32);

Error err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_READ, false);
ERR_FAIL_COND_V_MSG(err, false, "Can't open encrypted pack directory.");
Expand All @@ -350,7 +374,20 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
if (flags & PACK_FILE_REMOVAL) { // The file was removed.
PackedData::get_singleton()->remove_path(path);
} else {
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle, (flags & PACK_FILE_DELTA));
BitField<PackedData::PackedFile::PackedFileProperty> properties = PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_NONE;
if (flags & PACK_FILE_ENCRYPTED) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED);
}
if (sparse_bundle) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED);
}
if (flags & PACK_FILE_DELTA) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_DELTA);
}
if (async) {
properties.set_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ASYNC);
}
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, properties);
}
}

Expand All @@ -373,6 +410,27 @@ Ref<FileAccess> PackedSourcePCK::get_file(const String &p_path, PackedData::Pack

//////////////////////////////////////////////////////////////////

bool PackedSourceAsyncPCK::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset) {
String path = p_path.simplify_path();
if (path.ends_with("/")) {
path = path.trim_suffix("/");
}
if (!path.ends_with(".asyncpck")) {
return false;
}
path = path.path_join("assets").path_join("assets.sparsepck");
return PackedSourcePCK::try_open_pack(path, p_replace_files, p_offset);
}

Ref<FileAccess> PackedSourceAsyncPCK::get_file(const String &p_path, PackedData::PackedFile *p_file) {
String pack = p_file->pack.get_base_dir();
String path = p_path.simplify_path().trim_prefix("res://");
String file_path = pack.path_join(path);
return memnew(FileAccessPack(file_path, *p_file));
}

//////////////////////////////////////////////////////////////////

bool PackedSourceDirectory::try_open_pack(const String &p_path, bool p_replace_files, uint64_t p_offset) {
// Load with offset feature only supported for PCK files.
ERR_FAIL_COND_V_MSG(p_offset != 0, false, "Invalid PCK data. Note that loading files with a non-zero offset isn't supported with directories.");
Expand Down Expand Up @@ -400,7 +458,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
for (const String &file_name : da->get_files()) {
String file_path = p_path.path_join(file_name);
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false, false);
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files);
}

for (const String &sub_dir_name : da->get_directories()) {
Expand Down Expand Up @@ -510,28 +568,29 @@ void FileAccessPack::close() {
FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) {
path = p_path;
pf = p_file;
if (pf.bundle) {

if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_BUNDLED)) {
String simplified_path = p_path.simplify_path();
f = FileAccess::open(simplified_path, FileAccess::READ | FileAccess::SKIP_PACK);
ERR_FAIL_COND_MSG(f.is_null(), vformat(R"(Can't open pack-referenced file "%s" from sparse pack "%s".)", simplified_path, pf.pack));
off = 0; // For the sparse pack offset is always zero.
ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s' from '%s'.", simplified_path, String(pf.pack)));
} else {
f = FileAccess::open(pf.pack, FileAccess::READ);
ERR_FAIL_COND_MSG(f.is_null(), vformat(R"(Can't open pack-referenced file "%s" from pack "%s".)", p_path, pf.pack));
f->seek(pf.offset);
off = pf.offset;
ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack)));
}

if (pf.encrypted) {
if (pf.properties.has_flag(PackedData::PackedFile::PackedFileProperty::PACKED_FILE_PROPERTY_ENCRYPTED)) {
Ref<FileAccessEncrypted> fae;
fae.instantiate();
ERR_FAIL_COND_MSG(fae.is_null(), vformat(R"(Can't open encrypted pack-referenced file "%s" from pack "%s".)", p_path, pf.pack));

Vector<uint8_t> key;
key.resize(32);
for (int i = 0; i < key.size(); i++) {
key.write[i] = script_encryption_key[i];
}
memcpy(key.ptrw(), script_encryption_key, 32);

Error err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_READ, false);
ERR_FAIL_COND_MSG(err, vformat(R"(Can't open encrypted pack-referenced file "%s" from pack "%s".)", p_path, pf.pack));
Expand Down
Loading
Loading