Properly specify UTF8 in tests and when dealing with Closure (#25134)

juj · web-flow · commit 63db282a8bdd · 2025-09-02T21:58:17.000+03:00
This is from earlier PR #21478
diff --git a/test/test_browser.py b/test/test_browser.py
@@ -3409,10 +3409,12 @@ def test_modularize(self, args, code, opts):
     # this test is synchronous, so avoid async startup due to wasm features
     self.compile_btest('browser_test_hello_world.c', ['-sMODULARIZE', '-sSINGLE_FILE'] + args + opts)
     create_file('a.html', '''
+      <!DOCTYPE html><html lang="en"><head><meta charset="utf-8"></head><body>
       <script src="a.out.js"></script>
       <script>
         %s
       </script>
+      </body></html>
     ''' % code)
     self.run_browser('a.html', '/report_result?0')
 
@@ -4827,7 +4829,7 @@ def test_single_file_locate_file(self):
   # Tests that SINGLE_FILE works as intended in a Worker in JS output
   def test_single_file_worker_js(self):
     self.compile_btest('browser_test_hello_world.c', ['-o', 'test.js', '--proxy-to-worker', '-sSINGLE_FILE'])
-    create_file('test.html', '<script src="test.js"></script>')
+    create_file('test.html', '<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"></head><body><script src="test.js"></script></body></html>')
     self.run_browser('test.html', '/report_result?0')
     self.assertExists('test.js')
     self.assertNotExists('test.worker.js')
diff --git a/tools/building.py b/tools/building.py
@@ -609,6 +609,9 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None):
   # Tell closure never to inject the 'use strict' directive.
   args += ['--emit_use_strict=false']
   args += ['--assume_static_inheritance_is_not_used=false']
+  # Always output UTF-8 files, this helps generate UTF-8 code points instead of escaping code points with \uxxxx inside strings.
+  # Closure outputs ASCII by default, and must be adjusted to output UTF8 (https://github.com/google/closure-compiler/issues/4158)
+  args += ['--charset=UTF8']
 
   if settings.IGNORE_CLOSURE_COMPILER_ERRORS:
     args.append('--jscomp_off=*')
@@ -669,7 +672,8 @@ def move_to_safe_7bit_ascii_filename(filename):
   # 7-bit ASCII range. Therefore make sure the command line we pass does not contain any such
   # input files by passing all input filenames relative to the cwd. (user temp directory might
   # be in user's home directory, and user's profile name might contain unicode characters)
-  proc = run_process(cmd, stderr=PIPE, check=False, env=env, cwd=tempfiles.tmpdir)
+  # https://github.com/google/closure-compiler/issues/4159: Closure outputs stdout/stderr in iso-8859-1 on Windows.
+  proc = run_process(cmd, stderr=PIPE, check=False, env=env, cwd=tempfiles.tmpdir, encoding='iso-8859-1' if WINDOWS else 'utf-8')
 
   # XXX Closure bug: if Closure is invoked with --create_source_map, Closure should create a
   # outfile.map source map file (https://github.com/google/closure-compiler/wiki/Source-Maps)
