diff --git a/src/core/context_optimizer.py b/src/core/context_optimizer.py index d047ec2a..62d45d1c 100644 --- a/src/core/context_optimizer.py +++ b/src/core/context_optimizer.py @@ -129,7 +129,8 @@ def estimate_tokens_with_margin( pass # Method 2: Character-based estimation with language factors - lang_lower = language.lower() + from src.utils.lang_normalize import normalize_lang_key + lang_lower = normalize_lang_key(language) ratio = CHAR_TO_TOKEN_RATIOS.get(lang_lower, 4.0) # Default to English base_tokens = prompt_length / ratio diff --git a/src/core/epub/lang_support.py b/src/core/epub/lang_support.py index b1ce9581..505e0d21 100644 --- a/src/core/epub/lang_support.py +++ b/src/core/epub/lang_support.py @@ -95,6 +95,14 @@ "tagalog": "tl", } +# Regional language variants that map to BCP47 tags with region subtags. +# Kept separate from LANGUAGE_NAME_TO_CODE so the BCP47 conformance test +# (which validates primary subtags only) is not affected. +_REGIONAL_NAME_TO_CODE: Dict[str, str] = { + "portuguese (brazil)": "pt-BR", + "portuguese (portugal)": "pt-PT", +} + # Merge RTL entries first so any conflict is resolved by LTR keys (none expected). LANGUAGE_NAME_TO_CODE: Dict[str, str] = {**_RTL_NAME_TO_CODE, **_LTR_NAME_TO_CODE} @@ -108,9 +116,15 @@ def get_language_code(language: Optional[str]) -> Optional[str]: """Resolve a language name or locale to an ISO 639-1 code. + For regional variants like "Portuguese (Brazil)" / "Portuguese (Portugal)", + returns the full BCP47 tag with region subtag ("pt-BR" / "pt-PT") so + e-readers apply the correct hyphenation and TTS. + Examples: get_language_code("English") -> "en" get_language_code("Spanish") -> "es" + get_language_code("Portuguese (Brazil)") -> "pt-BR" + get_language_code("Portuguese (Portugal)") -> "pt-PT" get_language_code("en-US") -> "en" get_language_code("fr") -> "fr" get_language_code("Klingon") -> None @@ -122,6 +136,10 @@ def get_language_code(language: Optional[str]) -> Optional[str]: if not lang_lower: return None + # Check regional variants first (e.g. "portuguese (brazil)" -> "pt-BR") + if lang_lower in _REGIONAL_NAME_TO_CODE: + return _REGIONAL_NAME_TO_CODE[lang_lower] + if lang_lower in LANGUAGE_NAME_TO_CODE: return LANGUAGE_NAME_TO_CODE[lang_lower] diff --git a/src/core/pricing/estimator.py b/src/core/pricing/estimator.py index 334386ad..ecf89c92 100644 --- a/src/core/pricing/estimator.py +++ b/src/core/pricing/estimator.py @@ -41,7 +41,8 @@ def get_output_ratio(src_lang: str, tgt_lang: str) -> tuple[float, float]: if not src_lang or not tgt_lang: return DEFAULT_RATIO - key = (src_lang.lower().strip(), tgt_lang.lower().strip()) + from src.utils.lang_normalize import normalize_lang_key + key = (normalize_lang_key(src_lang), normalize_lang_key(tgt_lang)) return LANGUAGE_RATIOS.get(key, DEFAULT_RATIO) diff --git a/src/prompts/examples/helpers.py b/src/prompts/examples/helpers.py index 63454419..dd0e2569 100644 --- a/src/prompts/examples/helpers.py +++ b/src/prompts/examples/helpers.py @@ -11,6 +11,7 @@ from .placeholder_examples import get_example_for_pair from .subtitle_examples import SUBTITLE_EXAMPLES from .output_examples import OUTPUT_FORMAT_EXAMPLES +from src.utils.lang_normalize import normalize_lang_key def get_placeholder_example( @@ -33,14 +34,14 @@ def get_placeholder_example( def get_subtitle_example(target_lang: str) -> str: """Get subtitle format example for a target language.""" return SUBTITLE_EXAMPLES.get( - target_lang.lower(), + normalize_lang_key(target_lang), "[1]First translated line\n[2]Second translated line" ) def get_output_format_example(target_lang: str, has_placeholders: bool = True) -> str: """Get output format example for a target language.""" - lang_key = target_lang.lower() + lang_key = normalize_lang_key(target_lang) mode_key = "standard" if has_placeholders else "plain" if lang_key in OUTPUT_FORMAT_EXAMPLES: diff --git a/src/prompts/examples/placeholder_examples.py b/src/prompts/examples/placeholder_examples.py index 2b942370..b7d8667a 100644 --- a/src/prompts/examples/placeholder_examples.py +++ b/src/prompts/examples/placeholder_examples.py @@ -252,8 +252,9 @@ def get_example_for_pair(source_lang: str, target_lang: str) -> Dict[str, str]: Returns: Dict with "source", "correct", "wrong" keys """ - source_key = source_lang.lower() - target_key = target_lang.lower() + from src.utils.lang_normalize import normalize_lang_key + source_key = normalize_lang_key(source_lang) + target_key = normalize_lang_key(target_lang) # Get source language text (fallback to English) source_data = TRANSLATIONS.get(source_key, TRANSLATIONS[DEFAULT_LANGUAGE]) diff --git a/src/prompts/prompts.py b/src/prompts/prompts.py index 5b12981e..5bf92c22 100644 --- a/src/prompts/prompts.py +++ b/src/prompts/prompts.py @@ -201,7 +201,8 @@ def generate_translation_prompt( } # Try to match target language to get appropriate example - target_lang_lower = target_language.lower() + from src.utils.lang_normalize import normalize_lang_key + target_lang_lower = normalize_lang_key(target_language) example_format_text = example_texts.get(target_lang_lower, "Your translated text here") # Build the output format section outside the f-string to avoid backslash issues in Python 3.11 @@ -432,7 +433,8 @@ def generate_refinement_prompt( "korean": "다듬어진 텍스트는 여기에", } - target_lang_lower = target_language.lower() + from src.utils.lang_normalize import normalize_lang_key + target_lang_lower = normalize_lang_key(target_language) example_format_text = example_texts.get(target_lang_lower, "Your refined text here") output_format_section = _get_output_format_section( diff --git a/src/tts/tts_config.py b/src/tts/tts_config.py index f95cc06f..d04160f5 100644 --- a/src/tts/tts_config.py +++ b/src/tts/tts_config.py @@ -82,6 +82,8 @@ # Portuguese "portuguese": "pt-BR-FranciscaNeural", + "portuguese (brazil)": "pt-BR-FranciscaNeural", + "portuguese (portugal)": "pt-PT-RaquelNeural", "pt": "pt-BR-FranciscaNeural", "pt-br": "pt-BR-FranciscaNeural", "pt-pt": "pt-PT-RaquelNeural", diff --git a/src/utils/lang_normalize.py b/src/utils/lang_normalize.py new file mode 100644 index 00000000..8790b267 --- /dev/null +++ b/src/utils/lang_normalize.py @@ -0,0 +1,39 @@ +""" +Language name normalization for dict lookups. + +The UI may pass regional variants like "Portuguese (Brazil)" or +"Portuguese (Portugal)". These normalize to the base language name +("portuguese") for example/pricing/context dict lookups, which are +shared across regional variants. + +The full name (with region) is still passed to the LLM prompt so the +model knows which variant to produce. +""" +import re + +_REGIONAL_SUFFIX = re.compile(r'\s*\([^)]*\)\s*') + + +def normalize_lang_key(language: str) -> str: + """Normalize a language name to a lowercase base key for dict lookups. + + Strips regional suffixes in parentheses so that + "Portuguese (Brazil)" and "Portuguese (Portugal)" both map to + "portuguese", matching the shared dict keys used by example texts, + pricing ratios, and context optimization. + + Examples: + >>> normalize_lang_key("Portuguese (Brazil)") + 'portuguese' + >>> normalize_lang_key("Portuguese (Portugal)") + 'portuguese' + >>> normalize_lang_key("Portuguese") + 'portuguese' + >>> normalize_lang_key("English") + 'english' + >>> normalize_lang_key("") + '' + """ + if not language: + return "" + return _REGIONAL_SUFFIX.sub("", language).strip().lower() diff --git a/src/web/static/js/index.js b/src/web/static/js/index.js index bc150981..25787a0d 100644 --- a/src/web/static/js/index.js +++ b/src/web/static/js/index.js @@ -650,7 +650,8 @@ async function showTTSModal(filename, filepath) { - + + @@ -775,7 +776,8 @@ async function showTTSModal(filename, filepath) { - + + diff --git a/src/web/static/js/ui/form-manager.js b/src/web/static/js/ui/form-manager.js index 8b8bd2a1..a63799dc 100644 --- a/src/web/static/js/ui/form-manager.js +++ b/src/web/static/js/ui/form-manager.js @@ -323,7 +323,8 @@ export const FormManager = { detectBrowserLanguage() { // Get browser language (e.g., "fr-FR", "en-US", "zh-CN") const browserLang = navigator.language || navigator.userLanguage || 'en'; - const langCode = browserLang.split('-')[0].toLowerCase(); + const fullLangCode = browserLang.toLowerCase(); + const langCode = fullLangCode.split('-')[0]; // Map language codes to full names used in the UI const languageMap = { @@ -334,7 +335,7 @@ export const FormManager = { 'de': 'German', 'ja': 'Japanese', 'ko': 'Korean', - 'pt': 'Portuguese', + 'pt': 'Portuguese (Brazil)', 'ru': 'Russian', 'ar': 'Arabic', 'it': 'Italian', @@ -375,6 +376,11 @@ export const FormManager = { 'am': 'Amharic' }; + // Regional variant overrides: check full locale before base code + if (fullLangCode === 'pt-pt' || fullLangCode === 'pt-pt-x') { + return 'Portuguese (Portugal)'; + } + return languageMap[langCode] || 'English'; // Default to English if not found }, diff --git a/src/web/templates/translation_interface.html b/src/web/templates/translation_interface.html index c4ce8320..77a3f65c 100644 --- a/src/web/templates/translation_interface.html +++ b/src/web/templates/translation_interface.html @@ -324,7 +324,8 @@

German - + + @@ -392,7 +393,8 @@

German - + + @@ -656,7 +658,8 @@

German - + + @@ -675,7 +678,8 @@

German - + + @@ -1369,7 +1373,8 @@

German - + + @@ -1437,7 +1442,8 @@

German - + + diff --git a/tests/test_epub_xhtml_lang_attributes.py b/tests/test_epub_xhtml_lang_attributes.py index 3283ea52..5b41b7f9 100644 --- a/tests/test_epub_xhtml_lang_attributes.py +++ b/tests/test_epub_xhtml_lang_attributes.py @@ -45,6 +45,10 @@ ("German", "de"), ("Italian", "it"), ("Portuguese", "pt"), + ("Portuguese (Brazil)", "pt-BR"), + ("Portuguese (Portugal)", "pt-PT"), + ("portuguese (brazil)", "pt-BR"), + ("portuguese (portugal)", "pt-PT"), ("Russian", "ru"), ("Chinese", "zh"), ("Japanese", "ja"),