Merge branch 'Tencent:master' into master

unity/v8cc/CMakeLists.txt

@@ -0,0 +1,71 @@
+# Tencent is pleased to support the open source community by making xLua available.
+# Copyright (C) 2016 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+# http://opensource.org/licenses/MIT
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+cmake_minimum_required(VERSION 3.15)
+cmake_policy(SET CMP0091 NEW)
+
+project(V8CC)
+
+if("${JS_ENGINE}" MATCHES "^v8_10.6.194")
+    set(CMAKE_CXX_STANDARD 17)
+else ()
+    set(CMAKE_CXX_STANDARD 14)
+endif ()
+
+set(BACKEND_ROOT ${PROJECT_SOURCE_DIR}/../native_src/.backends/${JS_ENGINE})
+
+
+string (REPLACE ";" "$<SEMICOLON>${BACKEND_ROOT}" BACKEND_INC_NAMES "${BACKEND_ROOT}${BACKEND_INC_NAMES}")
+string (REPLACE ";" "$<SEMICOLON>${BACKEND_ROOT}" BACKEND_LIB_NAMES "${BACKEND_ROOT}${BACKEND_LIB_NAMES}")
+string (REPLACE ";" "$<SEMICOLON>" BACKEND_DEFINITIONS "${BACKEND_DEFINITIONS}")
+
+include_directories(
+    ${PROJECT_SOURCE_DIR}
+    ${BACKEND_INC_NAMES}
+)
+
+add_executable(v8cc v8cc.cc)
+
+
+set(V8CC_COMPILE_DEFINITIONS)
+
+if ( WIN32 AND NOT CYGWIN )
+    list(APPEND V8CC_COMPILE_DEFINITIONS BUILDING_V8_SHARED)
+endif ()
+
+if ( MSYS OR WIN32 )
+    if ( WIN32 ) 
+        target_link_libraries(v8cc
+            winmm.lib
+            dbghelp.lib
+            shlwapi.lib
+        )
+    endif ()
+
+    # definition
+    list(APPEND V8CC_COMPILE_DEFINITIONS PLATFORM_WINDOWS)
+elseif ( APPLE )
+    #definition
+    list(APPEND V8CC_COMPILE_DEFINITIONS PLATFORM_MAC)
+elseif (UNIX)
+    # link
+    target_link_libraries(v8cc pthread)
+
+    # definition
+    list(APPEND V8CC_COMPILE_DEFINITIONS PLATFORM_LINUX)
+endif ()
+
+# link
+target_link_libraries(v8cc ${BACKEND_LIB_NAMES} )
+list(APPEND V8CC_COMPILE_DEFINITIONS ${BACKEND_DEFINITIONS})
+
+target_compile_definitions (v8cc PUBLIC ${V8CC_COMPILE_DEFINITIONS})
+
+
+if ( WIN32 AND NOT CYGWIN AND NOT ( CMAKE_SYSTEM_NAME STREQUAL "WindowsStore" ) AND NOT ANDROID AND NOT MSYS)
+	set_property(TARGET v8cc PROPERTY
+             MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+endif ()

unity/v8cc/v8cc.cc

@@ -0,0 +1,202 @@
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <string>
+
+#include "libplatform/libplatform.h"
+#include "v8.h"
+
+bool endsWith(const std::string &fullString, const std::string &ending) {
+    if (fullString.length() >= ending.length()) {
+        return (0 == fullString.compare (fullString.length() - ending.length(), ending.length(), ending));
+    } else {
+        return false;
+    }
+}
+
+template <class T>
+v8::MaybeLocal<T> Compile(v8::Local<v8::Context> context, v8::ScriptCompiler::Source* source,
+                      v8::ScriptCompiler::CompileOptions options) {}
+template <>
+v8::MaybeLocal<v8::Script> Compile(v8::Local<v8::Context> context,
+                           v8::ScriptCompiler::Source* source,
+                           v8::ScriptCompiler::CompileOptions options) {
+  return v8::ScriptCompiler::Compile(context, source, options);
+}
+
+template <>
+v8::MaybeLocal<v8::Module> Compile(v8::Local<v8::Context> context,
+                           v8::ScriptCompiler::Source* source,
+                           v8::ScriptCompiler::CompileOptions options) {
+  return v8::ScriptCompiler::CompileModule(context->GetIsolate(), source, options);
+}
+
+template <class T>
+v8::MaybeLocal<T> CompileString(v8::Local<v8::Context> context,
+                                   v8::Local<v8::String> source,
+                                   const v8::ScriptOrigin& origin) {
+  v8::ScriptCompiler::Source script_source(source, origin);
+  v8::MaybeLocal<T> result = Compile<T>(context, &script_source, v8::ScriptCompiler::kNoCompileOptions);
+  return result;
+}
+
+struct CodeCacheHeader {
+    uint32_t MagicNumber;
+    uint32_t VersionHash;
+    uint32_t SourceHash;
+    uint32_t FlagHash;
+    uint32_t PayloadLength;
+    uint32_t Checksum;
+};
+
+int main(int argc, char* argv[]) {
+    if (argc < 2) {
+        std::cerr << "Usage: " << argv[0] << " <filename> [--module] [--no-cjs-wrap] [--verbose] [v8_flag1] [v8_flag2] ..." << std::endl;
+        return 1;
+    }
+
+    std::string filename = argv[1];
+    bool is_module = endsWith(filename, ".mjs");
+    bool no_cjs_wrap = false;
+    bool verbose = false;
+    std::string flags = "--no-lazy --no-flush-bytecode --no-enable_lazy_source_positions";
+    if (argc > 2) {
+        for (int i = 2; i < argc; ++i) {
+            std::string arg = argv[i];
+            if (arg == "--module") {
+                is_module = true;
+                continue;
+            }
+            if (arg == "--no-cjs-wrap") {
+                no_cjs_wrap = true;
+                continue;
+            }
+            if (arg == "--verbose") {
+                verbose = true;
+                continue;
+            }
+            flags += (" " + arg);
+        }
+    }
+
+    std::ifstream file(filename);
+    if (!file.is_open()) {
+        std::cerr << "Error opening file: " << filename << std::endl;
+        return 1;
+    }
+
+    std::stringstream buffer;
+    buffer << file.rdbuf();
+
+    std::string fileContent = buffer.str();
+
+    v8::V8::SetFlagsFromString(flags.c_str(), flags.size());
+
+    file.close();
+
+    v8::ScriptCompiler::CachedData* cached_data = nullptr;
+    // --- begin get code cache ---
+    std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform();
+    v8::V8::InitializePlatform(platform.get());
+    v8::V8::Initialize();
+
+    // Create a new Isolate and make it the current one.
+    v8::Isolate::CreateParams create_params;
+    create_params.array_buffer_allocator =
+        v8::ArrayBuffer::Allocator::NewDefaultAllocator();
+    v8::Isolate* isolate = v8::Isolate::New(create_params);
+    int source_length = 0;
+    {
+        v8::Isolate::Scope isolate_scope(isolate);
+        v8::HandleScope handle_scope(isolate);
+        v8::Local<v8::Context> context = v8::Context::New(isolate);
+        v8::Context::Scope context_scope(context);
+        v8::TryCatch try_catch(isolate);
+        auto script_url = v8::String::NewFromUtf8(isolate, filename.c_str()).ToLocalChecked();
+        if (!is_module && !no_cjs_wrap) {
+            fileContent = "(function (exports, require, module, __filename, __dirname) { " + fileContent + "\n});";
+        }
+        v8::Local<v8::String> source =
+            v8::String::NewFromUtf8(isolate, fileContent.c_str()).ToLocalChecked();
+        source_length = source->Length();
+        if (is_module) {
+#if V8_MAJOR_VERSION > 8
+            v8::ScriptOrigin origin(isolate, script_url, 0, 0, true, -1, v8::Local<v8::Value>(), false, false, true);
+#else
+            v8::ScriptOrigin origin(script_url, v8::Integer::New(isolate, 0), v8::Integer::New(isolate, 0), v8::True(isolate),
+                v8::Local<v8::Integer>(), v8::Local<v8::Value>(), v8::False(isolate), v8::False(isolate), v8::True(isolate));
+#endif
+            auto module = CompileString<v8::Module>(context, source, origin);
+
+            if (!module.IsEmpty()) {
+                cached_data = v8::ScriptCompiler::CreateCodeCache(module.ToLocalChecked()->GetUnboundModuleScript());
+            }
+        } else {
+#if V8_MAJOR_VERSION > 8
+            v8::ScriptOrigin origin(isolate, script_url);
+#else
+            v8::ScriptOrigin origin(script_url);
+#endif
+
+            auto script = CompileString<v8::Script>(context, source, origin);
+            if (!script.IsEmpty()) {
+                cached_data = v8::ScriptCompiler::CreateCodeCache(script.ToLocalChecked()->GetUnboundScript());
+            }
+        }
+        if (try_catch.HasCaught()) {
+            v8::Local<v8::Value> stack_trace;
+            if (try_catch.StackTrace(context).ToLocal(&stack_trace))
+            {
+                v8::String::Utf8Value info(isolate, stack_trace);
+                std::cout << (*info) << std::endl;
+            }
+            return 1;
+        }
+    }
+
+    // Dispose the isolate and tear down V8.
+    isolate->Dispose();
+    v8::V8::Dispose();
+    v8::V8::ShutdownPlatform();
+    delete create_params.array_buffer_allocator;
+    // --- end get code cache ---
+
+    if (!cached_data) {
+        std::cout << "cached_data is nullptr!!!" << std::endl;
+    }
+
+    auto dot_pos = filename.find_last_of('.');
+    std::string output_filename = filename.substr(0, dot_pos == std::string::npos ? filename.size(): dot_pos) + (is_module ? ".mbc" : ".cbc");
+
+    std::ofstream output_file(output_filename, std::ios::binary);
+    if (!output_file.is_open()) {
+        std::cerr << "Error creating file: " << output_filename << std::endl;
+        return 1;
+    }
+
+    output_file.write((const char*)cached_data->data, cached_data->length);
+    output_file.close();
+
+    if (verbose) {
+        std::cout << "esm: " << is_module << std::endl;
+        std::cout << "cjs: " << (!is_module && !no_cjs_wrap) << std::endl;
+        std::cout << "v8 flags: " << flags << std::endl;
+
+        //std::cout << fileContent << std::endl;
+        std::cout << "input : " << filename << ", source length: " << source_length << std::endl;
+        std::cout << "output: " << output_filename << ", bytecode length: " << cached_data->length << std::endl;
+
+        const CodeCacheHeader *cch = (const CodeCacheHeader *)cached_data->data;
+        std::cout << "MagicNumber : " << cch->MagicNumber << std::endl;
+        std::cout << "VersionHash : " << cch->VersionHash << std::endl;
+        std::cout << "SourceHash : " << cch->SourceHash << std::endl;
+        std::cout << "FlagHash : " << cch->FlagHash << std::endl;
+        std::cout << "PayloadLength : " << cch->PayloadLength << std::endl;
+        std::cout << "Checksum : " << cch->Checksum << std::endl;
+    }
+
+    delete cached_data;
+
+    return 0;
+}

unreal/Puerts/Content/JavaScript/puerts/modular.js

@@ -53,7 +53,7 @@ var global = global || (function () { return this; }());
 
     let moduleCache = Object.create(null);
     let buildinModule = Object.create(null);
-    function executeModule(fullPath, script, debugPath, sid, isESM) {
+    function executeModule(fullPath, script, debugPath, sid, isESM, bytecode) {
         sid = (typeof sid == 'undefined') ? 0 : sid;
         let fullPathInJs = fullPath.replace(/\\/g, '\\\\');
         let fullDirInJs = (fullPath.indexOf('/') != -1) ? fullPath.substring(0, fullPath.lastIndexOf("/")) : fullPath.substring(0, fullPath.lastIndexOf("\\")).replace(/\\/g, '\\\\');
@@ -63,8 +63,8 @@ var global = global || (function () { return this; }());
         let wrapped = evalScript(
             // Wrap the script in the same way NodeJS does it. It is important since IDEs (VSCode) will use this wrapper pattern
             // to enable stepping through original source in-place.
-            isESM ? script: "(function (exports, require, module, __filename, __dirname) { " + script + "\n});", 
-            debugPath, isESM, fullPath
+            (isESM || bytecode) ? script: "(function (exports, require, module, __filename, __dirname) { " + script + "\n});", 
+            debugPath, isESM, fullPath, bytecode
         )
         if (isESM) return wrapped;
         wrapped(exports, puerts.genRequire(fullDirInJs), module, fullPathInJs, fullDirInJs)
@@ -76,6 +76,32 @@ var global = global || (function () { return this; }());
         return (packageConfigure && packageConfigure.type === "module") ? packageConfigure.main : undefined;
     }
 
+    function getSourceLengthFromBytecode(buf, isESM) {
+        let sourceHash = (new Uint32Array(buf))[2];
+        //console.log(`sourceHash:${sourceHash}`);
+        const kModuleFlagMask = (1 << 31);
+        const mask = isESM ? kModuleFlagMask : 0;
+
+        // Remove the mask to get the original length
+        const length = sourceHash & ~mask;
+
+        return length;
+    }
+
+    let baseString
+    function generateEmptyCode(length) {
+        if (baseString === undefined) {
+            baseString = " ".repeat(128*1024);
+        }
+        if (length <= baseString.length) {
+            return baseString.slice(0, length);
+        } else {
+            const fullString = baseString.repeat(Math.floor(length / baseString.length));
+            const remainingLength = length % baseString.length;
+            return fullString.concat(baseString.slice(0, remainingLength));
+        }
+    }
+
     function genRequire(requiringDir, outerIsESM) {
         let localModuleCache = Object.create(null);
         function require(moduleName) {
@@ -119,9 +145,14 @@ var global = global || (function () { return this; }());
             localModuleCache[moduleName] = m;
             moduleCache[key] = m;
             let sid = addModule(m);
-            let script = loadModule(fullPath);
-            let isESM = outerIsESM === true || fullPath.endsWith(".mjs")
-            if (fullPath.endsWith(".cjs")) isESM = false;
+            let isESM = outerIsESM === true || fullPath.endsWith(".mjs") || fullPath.endsWith(".mbc");
+            if (fullPath.endsWith(".cjs") || fullPath.endsWith(".cbc")) isESM = false;
+            let script = isESM ? undefined : loadModule(fullPath);
+            let bytecode = undefined;
+            if (fullPath.endsWith(".mbc") || fullPath.endsWith(".cbc")) {
+                bytecode = script;
+                script = generateEmptyCode(getSourceLengthFromBytecode(bytecode));
+            }
             try {
                 if (fullPath.endsWith(".json")) {
                     let packageConfigure = JSON.parse(script);
@@ -149,7 +180,7 @@ var global = global || (function () { return this; }());
                         m.exports = packageConfigure;
                     }
                 } else {
-                    let r = executeModule(fullPath, script, debugPath, sid, isESM);
+                    let r = executeModule(fullPath, script, debugPath, sid, isESM, bytecode);
                     if (isESM) {
                         m.exports = r;
                     }

unreal/Puerts/Source/JsEnv/JsEnv.Build.cs

@@ -46,6 +46,8 @@ enum SupportedV8Versions
     private bool SingleThreaded = false;
 
     public static bool WithSourceControl = false;
+
+    public bool WithByteCode = false;
 
     public JsEnv(ReadOnlyTargetRules Target) : base(Target)
     {
@@ -368,6 +370,11 @@ void ThirdParty(ReadOnlyTargetRules Target)
             PrivateDefinitions.Add("USING_SINGLE_THREAD_PLATFORM");
         }
 
+        if (WithByteCode)
+        {
+            PrivateDefinitions.Add("WITH_V8_BYTECODE");
+        }
+
         string v8LibSuffix = "";
 
         if (UseV8Version == SupportedV8Versions.V8_4_371_19)

unreal/Puerts/Source/JsEnv/Private/DefaultJSModuleLoader.cpp

@@ -74,6 +74,10 @@ bool DefaultJSModuleLoader::SearchModuleInDir(
     return SearchModuleWithExtInDir(Dir, RequiredModule + ".js", Path, AbsolutePath) ||
            SearchModuleWithExtInDir(Dir, RequiredModule + ".mjs", Path, AbsolutePath) ||
            SearchModuleWithExtInDir(Dir, RequiredModule + ".cjs", Path, AbsolutePath) ||
+#if defined(WITH_V8_BYTECODE)
+           SearchModuleWithExtInDir(Dir, RequiredModule + ".mbc", Path, AbsolutePath) ||
+           SearchModuleWithExtInDir(Dir, RequiredModule + ".cbc", Path, AbsolutePath) ||
+#endif
            SearchModuleWithExtInDir(Dir, RequiredModule / "package.json", Path, AbsolutePath) ||
            SearchModuleWithExtInDir(Dir, RequiredModule / "index.js", Path, AbsolutePath);
 }