Fix for issue 1011: add 3rd input encoding parameter to json_encode modifier and let it default to \Smarty\Smarty::$_CHARSET

docs/designers/language-modifiers/language-modifier-json-encode.md

@@ -11,9 +11,10 @@ Depending on the value of `$user` this would return a string in JSON-format, e.g
 
 ## Parameters
 
-| Parameter | Type | Required | Description                                                                               |
-|-----------|------|----------|-------------------------------------------------------------------------------------------|
-| 1         | int  | No       | bitmask of flags, directly passed to [PHP's json_encode](https://www.php.net/json_encode) |
+| Parameter | Type   | Required | Description                                                                               |
+|-----------|--------|----------|-------------------------------------------------------------------------------------------|
+| 1         | int    | No       | bitmask of flags, directly passed to [PHP's json_encode](https://www.php.net/json_encode) |
+| 2         | string | No       | input encoding; defaults to \Smarty\Smarty::$_CHARSET which defaults to UTF-8             |
 
 
 ## Examples
@@ -24,4 +25,4 @@ Without it, an array `$myArray = ["a","b"]` would be formatted as a javascript a
 ```smarty
 {$myArray|json_encode} # renders: ["a","b"]
 {$myArray|json_encode:16} # renders: {"0":"a","1":"b"}
-```
+```

docs/designers/language-modifiers/language-modifier-round.md

@@ -23,13 +23,3 @@ If 'precision' is negative, the number is rounded to the nearest power of 10. Se
 The parameter 'mode' defines how the rounding is done. By default, 2.5 is rounded to 3, whereas 2.45 is rounded to 2.
 You usually don't need to change this. For more details on rounding modes, 
 see [PHP's documentation on round](https://www.php.net/manual/en/function.round).
-
-## Examples
-
-By passing `16` as the second parameter, you can force json_encode to always format the JSON-string as an object.
-Without it, an array `$myArray = ["a","b"]` would be formatted as a javascript array:
-
-```smarty
-{$myArray|json_encode} # renders: ["a","b"]
-{$myArray|json_encode:16} # renders: {"0":"a","1":"b"}
-```

src/Extension/DefaultExtension.php

@@ -31,7 +31,6 @@ public function getModifierCompiler(string $modifier): ?\Smarty\Compile\Modifier
 			case 'indent': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\IndentModifierCompiler(); break;
 			case 'is_array': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\IsArrayModifierCompiler(); break;
 			case 'isset': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\IssetModifierCompiler(); break;
-			case 'json_encode': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\JsonEncodeModifierCompiler(); break;
 			case 'lower': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\LowerModifierCompiler(); break;
 			case 'nl2br': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\Nl2brModifierCompiler(); break;
 			case 'noprint': $this->modifiers[$modifier] = new \Smarty\Compile\Modifier\NoPrintModifierCompiler(); break;
@@ -62,6 +61,7 @@ public function getModifierCallback(string $modifierName) {
 			case 'implode': return [$this, 'smarty_modifier_implode'];
 			case 'in_array': return [$this, 'smarty_modifier_in_array'];
 			case 'join': return [$this, 'smarty_modifier_join'];
+			case 'json_encode': return [$this, 'smarty_modifier_json_encode'];
 			case 'mb_wordwrap': return [$this, 'smarty_modifier_mb_wordwrap'];
 			case 'number_format': return [$this, 'smarty_modifier_number_format'];
 			case 'regex_replace': return [$this, 'smarty_modifier_regex_replace'];
@@ -605,6 +605,41 @@ public function smarty_modifier_join($values, $separator = '')
 		return implode((string) ($separator ?? ''), (array) $values);
 	}
 
+	/**
+	 * Smarty json_encode modifier plugin.
+	 * Type:     modifier
+	 * Name:     json_encode
+	 * Purpose:  Returns the JSON representation of the given value or false on error. The resulting string will be UTF-8 encoded.
+	 *
+	 * @param mixed $value
+	 * @param int   $flags
+	 * @param string $input_encoding of $value; defaults to \Smarty\Smarty::$_CHARSET
+	 *
+	 * @return string|false
+	 */
+	public function smarty_modifier_json_encode($value, $flags = 0, string $input_encoding = null)
+	{
+		if (!$input_encoding) {
+			$input_encoding = \Smarty\Smarty::$_CHARSET;
+		}
+
+		# json_encode() expects UTF-8 input, so recursively encode $value if necessary into UTF-8
+		if ($value && strcasecmp($input_encoding, 'UTF-8')) {
+			if (is_string($value)) {	# shortcut for the most common case
+				$value = mb_convert_encoding($value, 'UTF-8', $input_encoding);
+			}
+			elseif (DefaultExtension\RecursiveTranscoder::is_transcoding_candidate($value)) {
+				$value = DefaultExtension\RecursiveTranscoder::transcode($value, 'UTF-8', $input_encoding, ['ignore_JsonSerializable_objects' => true]);
+				if ($value === false) {
+					# If transcode() throws an exception on failure, then the interpreter will never arrive here
+					return false;	# failure
+				}
+			}
+		}
+
+		return \json_encode($value, $flags);	# string|false
+	}
+
 	/**
 	 * Smarty wordwrap modifier plugin
 	 * Type:     modifier

src/Extension/DefaultExtension/RecursiveTranscoder.php

@@ -0,0 +1,119 @@
+<?php declare(strict_types=1);
+
+namespace Smarty\Extension\DefaultExtension;
+
+use Smarty\Exception;
+
+class RecursiveTranscoder {
+
+	/**
+	 * Determines if the given value is a possible candidate for transcoding.
+	 *
+	 * @param mixed $value
+	 * @return bool
+	 */
+	public static function is_transcoding_candidate($value): bool {
+		if (empty($value)) {
+			return false;
+		}
+		return is_string($value) || is_array($value) || is_object($value);
+	}
+
+
+	/**
+	 * Similar to mb_convert_encoding(), but operates recursively on keys and values of arrays, and on objects too.
+	 * Objects implementing \JsonSerializable and unsupported types are returned unchanged.
+	 * The following boolean options/hints are supported (all default to false):
+	 *	- ignore_keys: do not transcode array keys
+	 *	- ignore_objects: leave objects alone, i.e. do not convert them into associative arrays and transcode them
+	 *	- ignore_JsonSerializable_objects: if transcoded result is meant as input for json_encode(), then set this to true.
+	 *
+	 * @param mixed $data
+	 * @param string $to_encoding
+	 * @param string $from_encoding of $data; defaults to \Smarty\Smarty::$_CHARSET
+	 * @param array $options
+	 * @return mixed
+	 */
+	public static function transcode($data, string $to_encoding, string $from_encoding = null, array $options = null) {
+		if (!static::is_transcoding_candidate($data)) {
+			return $data;
+		}
+		if (!$from_encoding) {
+			$from_encoding = \Smarty\Smarty::$_CHARSET;
+		}
+		if (strcasecmp($to_encoding, $from_encoding) == 0) {
+			return $data;
+		}
+
+		# most cases:
+		if (is_string($data)) {
+			return mb_convert_encoding($data, $to_encoding, $from_encoding);	# string|false
+		}
+
+		# convert object to array to be transcoded as array
+		if (is_object($data)) {
+			if (!empty($options['ignore_objects'])) {
+				return $data;
+			}
+			if (is_a($data, \JsonSerializable::class)) {
+				if (!empty($options['ignore_JsonSerializable_objects'])) {
+					return $data;	# \JsonSerializable objects should be trusted to serialize themselves into data that can be consumed by json_encode() no matter what the application's default encoding is.
+				}
+			}
+			$data = get_object_vars($data); # public properties as key => value pairs
+		}
+
+		if (!(is_array($data) && $data)) {
+			return $data;	# any empty array or non-array type as a possible result of object conversion above
+		}
+
+		# $data is a filled array
+		$must_transcode_keys = empty($options['ignore_keys']);
+		$result = $must_transcode_keys ? [] : null;	# replacement for $data if keys are transcoded too (i.e. $must_transcode_keys)
+		$this_func = __FUNCTION__;	# for recursion
+		foreach ($data as $k => &$v) {
+			if ($must_transcode_keys && is_string($k)) {
+				$converted_k = mb_convert_encoding($k, $to_encoding, $from_encoding);	# string|false
+				if ($converted_k === false) {	# this means mb_convert_encoding() failed which should've triggered a warning
+					# One of three things can be done here:
+					# 1. throw an Exception
+					# 2. return false, indicating to caller that mb_convert_encoding() failed
+					# 3. do nothing and use the original key
+					#return false;
+					throw Exception("Failed to encode array key \"$k\" from $from_encoding to $to_encoding");
+				}
+				else {
+					$k = $converted_k;
+				}
+			}
+			if (static::is_transcoding_candidate($v)) {
+				# recurse
+				$converted_v = static::$this_func($v, $to_encoding, $from_encoding, $options);
+				if ($converted_v === false) {	# this means that $v is a string and that mb_convert_encoding() failed, which should've triggered a warning
+					# One of four things can be done here:
+					# 1. throw an Exception
+					# 2. return false, indicating to caller that mb_convert_encoding() failed
+					# 3. do nothing and use the original value
+					# 4. replace the original value with false
+					#return false;
+					throw Exception('Failed to encode array value' . (is_string($v) ? " \"$k\"" : '')  . 'of type ' . gettype($v) . " from $from_encoding to $to_encoding");
+				}
+				else {
+					$v = $converted_v;
+					if ($must_transcode_keys) {
+						$result[$k] = $v;
+					}
+				}
+			}
+			else {
+				# $v may be false here, and in this case it is not an error (since no transcoding occurred since it's not a transcoding candidate)
+				if ($must_transcode_keys) {
+					$result[$k] = $v;
+				}
+			}
+			unset($v);
+		}
+		return $must_transcode_keys ? $result : $data;
+	}
+
+}

tests/UnitTests/TemplateSource/TagTests/PluginModifier/PluginModifierJsonEncodeCp1252Test.php

@@ -0,0 +1,88 @@
+<?php
+/**
+ * Smarty PHPunit tests of modifier.
+ * This file should be saved in UTF-8 encoding for comment legibility.
+ */
+
+namespace UnitTests\TemplateSource\TagTests\PluginModifier;
+use PHPUnit_Smarty;
+
+class PluginModifierJsonEncodeCp1252Test extends PHPUnit_Smarty
+{
+	public function setUp(): void
+	{
+		$this->setUpSmarty(__DIR__);
+		\Smarty\Smarty::$_CHARSET = 'cp1252';
+	}
+
+	public function tearDown(): void
+	{
+		\Smarty\Smarty::$_CHARSET = 'UTF-8';
+	}
+
+	/**
+	 * @dataProvider dataForDefault
+	 */
+	public function testDefault($value, $expected)
+	{
+		$tpl = $this->smarty->createTemplate('string:{$v|json_encode}');
+		$tpl->assign("v", $value);
+		$this->assertEquals($expected, $this->smarty->fetch($tpl));
+	}
+
+	/**
+	 * @dataProvider dataForDefault
+	 */
+	public function testDefaultAsFunction($value, $expected)
+	{
+		$tpl = $this->smarty->createTemplate('string:{json_encode($v)}');
+		$tpl->assign("v", $value);
+		$this->assertEquals($expected, $this->smarty->fetch($tpl));
+	}
+
+	public function dataForDefault() {
+		$json_serializable_object = new class() implements \JsonSerializable {
+			public function jsonSerialize(): mixed {
+				return ["Schl\xC3\xBCssel" => "Stra\xC3\x9Fe"];	# UTF-8 ready for json_encode(); to prove that transcoding doesn't attempt to transcode this again
+				#return ['Schlüssel' => 'Straße'];	# alternatively, this can be used, but then this file must always be saved in UTF-8 encoding or else the test will fail.
+			}
+		};
+		return [
+			["abc", '"abc"'],
+			[["abc"], '["abc"]'],
+			[["abc",["a"=>2]], '["abc",{"a":2}]'],
+			[["\x80uro",["Schl\xFCssel"=>"Stra\xDFe"]], '["\u20acuro",{"Schl\u00fcssel":"Stra\u00dfe"}]'],	# x80 = € = euro, xFC = ü = uuml, xDF = ß = szlig
+			[$json_serializable_object, '{"Schl\u00fcssel":"Stra\u00dfe"}'],
+		];
+	}
+
+	/**
+	 * @dataProvider dataForForceObject
+	 */
+	public function testForceObject($value, $expected)
+	{
+		$tpl = $this->smarty->createTemplate('string:{$v|json_encode:16}');
+		$tpl->assign("v", $value);
+		$this->assertEquals($expected, $this->smarty->fetch($tpl));
+	}
+
+	/**
+	 * @dataProvider dataForForceObject
+	 */
+	public function testForceObjectAsFunction($value, $expected)
+	{
+		$tpl = $this->smarty->createTemplate('string:{json_encode($v,16)}');
+		$tpl->assign("v", $value);
+		$this->assertEquals($expected, $this->smarty->fetch($tpl));
+	}
+
+	public function dataForForceObject() {
+		return [
+			["abc", '"abc"'],
+			[["abc"], '{"0":"abc"}'],
+			[["abc",["a"=>2]], '{"0":"abc","1":{"a":2}}'],
+			[["\x80uro"], '{"0":"\u20acuro"}'],
+		];
+	}
+
+}

tests/UnitTests/TemplateSource/TagTests/PluginModifier/PluginModifierJsonEncodeTest.php

@@ -1,6 +1,7 @@
 <?php
 /**
- * Smarty PHPunit tests of modifier
+ * Smarty PHPunit tests of modifier.
+ * This file must be saved in UTF-8 encoding!
  */
 
 namespace UnitTests\TemplateSource\TagTests\PluginModifier;
@@ -38,6 +39,7 @@ public function dataForDefault() {
 			["abc", '"abc"'],
 			[["abc"], '["abc"]'],
 			[["abc",["a"=>2]], '["abc",{"a":2}]'],
+			[["€uro",["Schlüssel"=>"Straße"]], '["\u20acuro",{"Schl\u00fcssel":"Stra\u00dfe"}]'],	# \u{20ac} = € = euro, \u{00fc} = ü = uuml, \u{00df} = ß = szlig
 		];
 	}
 
@@ -66,6 +68,7 @@ public function dataForForceObject() {
 			["abc", '"abc"'],
 			[["abc"], '{"0":"abc"}'],
 			[["abc",["a"=>2]], '{"0":"abc","1":{"a":2}}'],
+			[["€uro"], '{"0":"\u20acuro"}'],
 		];
 	}