Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/core/context_optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ def estimate_tokens_with_margin(
pass

# Method 2: Character-based estimation with language factors
lang_lower = language.lower()
from src.utils.lang_normalize import normalize_lang_key
lang_lower = normalize_lang_key(language)
ratio = CHAR_TO_TOKEN_RATIOS.get(lang_lower, 4.0) # Default to English

base_tokens = prompt_length / ratio
Expand Down
18 changes: 18 additions & 0 deletions src/core/epub/lang_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,14 @@
"tagalog": "tl",
}

# Regional language variants that map to BCP47 tags with region subtags.
# Kept separate from LANGUAGE_NAME_TO_CODE so the BCP47 conformance test
# (which validates primary subtags only) is not affected.
_REGIONAL_NAME_TO_CODE: Dict[str, str] = {
"portuguese (brazil)": "pt-BR",
"portuguese (portugal)": "pt-PT",
}

# Merge RTL entries first so any conflict is resolved by LTR keys (none expected).
LANGUAGE_NAME_TO_CODE: Dict[str, str] = {**_RTL_NAME_TO_CODE, **_LTR_NAME_TO_CODE}

Expand All @@ -108,9 +116,15 @@
def get_language_code(language: Optional[str]) -> Optional[str]:
"""Resolve a language name or locale to an ISO 639-1 code.

For regional variants like "Portuguese (Brazil)" / "Portuguese (Portugal)",
returns the full BCP47 tag with region subtag ("pt-BR" / "pt-PT") so
e-readers apply the correct hyphenation and TTS.

Examples:
get_language_code("English") -> "en"
get_language_code("Spanish") -> "es"
get_language_code("Portuguese (Brazil)") -> "pt-BR"
get_language_code("Portuguese (Portugal)") -> "pt-PT"
get_language_code("en-US") -> "en"
get_language_code("fr") -> "fr"
get_language_code("Klingon") -> None
Expand All @@ -122,6 +136,10 @@ def get_language_code(language: Optional[str]) -> Optional[str]:
if not lang_lower:
return None

# Check regional variants first (e.g. "portuguese (brazil)" -> "pt-BR")
if lang_lower in _REGIONAL_NAME_TO_CODE:
return _REGIONAL_NAME_TO_CODE[lang_lower]

if lang_lower in LANGUAGE_NAME_TO_CODE:
return LANGUAGE_NAME_TO_CODE[lang_lower]

Expand Down
3 changes: 2 additions & 1 deletion src/core/pricing/estimator.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
def get_output_ratio(src_lang: str, tgt_lang: str) -> tuple[float, float]:
if not src_lang or not tgt_lang:
return DEFAULT_RATIO
key = (src_lang.lower().strip(), tgt_lang.lower().strip())
from src.utils.lang_normalize import normalize_lang_key
key = (normalize_lang_key(src_lang), normalize_lang_key(tgt_lang))
return LANGUAGE_RATIOS.get(key, DEFAULT_RATIO)


Expand Down
5 changes: 3 additions & 2 deletions src/prompts/examples/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from .placeholder_examples import get_example_for_pair
from .subtitle_examples import SUBTITLE_EXAMPLES
from .output_examples import OUTPUT_FORMAT_EXAMPLES
from src.utils.lang_normalize import normalize_lang_key


def get_placeholder_example(
Expand All @@ -33,14 +34,14 @@ def get_placeholder_example(
def get_subtitle_example(target_lang: str) -> str:
"""Get subtitle format example for a target language."""
return SUBTITLE_EXAMPLES.get(
target_lang.lower(),
normalize_lang_key(target_lang),
"[1]First translated line\n[2]Second translated line"
)


def get_output_format_example(target_lang: str, has_placeholders: bool = True) -> str:
"""Get output format example for a target language."""
lang_key = target_lang.lower()
lang_key = normalize_lang_key(target_lang)
mode_key = "standard" if has_placeholders else "plain"

if lang_key in OUTPUT_FORMAT_EXAMPLES:
Expand Down
5 changes: 3 additions & 2 deletions src/prompts/examples/placeholder_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,8 +252,9 @@ def get_example_for_pair(source_lang: str, target_lang: str) -> Dict[str, str]:
Returns:
Dict with "source", "correct", "wrong" keys
"""
source_key = source_lang.lower()
target_key = target_lang.lower()
from src.utils.lang_normalize import normalize_lang_key
source_key = normalize_lang_key(source_lang)
target_key = normalize_lang_key(target_lang)

# Get source language text (fallback to English)
source_data = TRANSLATIONS.get(source_key, TRANSLATIONS[DEFAULT_LANGUAGE])
Expand Down
6 changes: 4 additions & 2 deletions src/prompts/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,8 @@ def generate_translation_prompt(
}

# Try to match target language to get appropriate example
target_lang_lower = target_language.lower()
from src.utils.lang_normalize import normalize_lang_key
target_lang_lower = normalize_lang_key(target_language)
example_format_text = example_texts.get(target_lang_lower, "Your translated text here")

# Build the output format section outside the f-string to avoid backslash issues in Python 3.11
Expand Down Expand Up @@ -432,7 +433,8 @@ def generate_refinement_prompt(
"korean": "다듬어진 텍스트는 여기에",
}

target_lang_lower = target_language.lower()
from src.utils.lang_normalize import normalize_lang_key
target_lang_lower = normalize_lang_key(target_language)
example_format_text = example_texts.get(target_lang_lower, "Your refined text here")

output_format_section = _get_output_format_section(
Expand Down
2 changes: 2 additions & 0 deletions src/tts/tts_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@

# Portuguese
"portuguese": "pt-BR-FranciscaNeural",
"portuguese (brazil)": "pt-BR-FranciscaNeural",
"portuguese (portugal)": "pt-PT-RaquelNeural",
"pt": "pt-BR-FranciscaNeural",
"pt-br": "pt-BR-FranciscaNeural",
"pt-pt": "pt-PT-RaquelNeural",
Expand Down
39 changes: 39 additions & 0 deletions src/utils/lang_normalize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""
Language name normalization for dict lookups.

The UI may pass regional variants like "Portuguese (Brazil)" or
"Portuguese (Portugal)". These normalize to the base language name
("portuguese") for example/pricing/context dict lookups, which are
shared across regional variants.

The full name (with region) is still passed to the LLM prompt so the
model knows which variant to produce.
"""
import re

_REGIONAL_SUFFIX = re.compile(r'\s*\([^)]*\)\s*')


def normalize_lang_key(language: str) -> str:
"""Normalize a language name to a lowercase base key for dict lookups.

Strips regional suffixes in parentheses so that
"Portuguese (Brazil)" and "Portuguese (Portugal)" both map to
"portuguese", matching the shared dict keys used by example texts,
pricing ratios, and context optimization.

Examples:
>>> normalize_lang_key("Portuguese (Brazil)")
'portuguese'
>>> normalize_lang_key("Portuguese (Portugal)")
'portuguese'
>>> normalize_lang_key("Portuguese")
'portuguese'
>>> normalize_lang_key("English")
'english'
>>> normalize_lang_key("")
''
"""
if not language:
return ""
return _REGIONAL_SUFFIX.sub("", language).strip().lower()
6 changes: 4 additions & 2 deletions src/web/static/js/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,8 @@ async function showTTSModal(filename, filepath) {
<option value="German">German (Deutsch)</option>
<option value="Japanese">Japanese (日本語)</option>
<option value="Korean">Korean (한국어)</option>
<option value="Portuguese">Portuguese (Português)</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil) (Português)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal) (Português)</option>
<option value="Russian">Russian (Русский)</option>
<option value="Arabic">Arabic (العربية)</option>
<!-- European -->
Expand Down Expand Up @@ -775,7 +776,8 @@ async function showTTSModal(filename, filepath) {
<option value="it">Italian (Italiano)</option>
<option value="ja">Japanese (日本語)</option>
<option value="ko">Korean (한국어)</option>
<option value="pt">Portuguese (Português)</option>
<option value="pt-br">Portuguese (Brazil) (Português)</option>
<option value="pt-pt">Portuguese (Portugal) (Português)</option>
<option value="ru">Russian (Русский)</option>
<option value="ar">Arabic (العربية)</option>
<!-- European -->
Expand Down
10 changes: 8 additions & 2 deletions src/web/static/js/ui/form-manager.js
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,8 @@ export const FormManager = {
detectBrowserLanguage() {
// Get browser language (e.g., "fr-FR", "en-US", "zh-CN")
const browserLang = navigator.language || navigator.userLanguage || 'en';
const langCode = browserLang.split('-')[0].toLowerCase();
const fullLangCode = browserLang.toLowerCase();
const langCode = fullLangCode.split('-')[0];

// Map language codes to full names used in the UI
const languageMap = {
Expand All @@ -334,7 +335,7 @@ export const FormManager = {
'de': 'German',
'ja': 'Japanese',
'ko': 'Korean',
'pt': 'Portuguese',
'pt': 'Portuguese (Brazil)',
'ru': 'Russian',
'ar': 'Arabic',
'it': 'Italian',
Expand Down Expand Up @@ -375,6 +376,11 @@ export const FormManager = {
'am': 'Amharic'
};

// Regional variant overrides: check full locale before base code
if (fullLangCode === 'pt-pt' || fullLangCode === 'pt-pt-x') {
return 'Portuguese (Portugal)';
}

return languageMap[langCode] || 'English'; // Default to English if not found
},

Expand Down
18 changes: 12 additions & 6 deletions src/web/templates/translation_interface.html
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,8 @@ <h3 id="updateOverlayTitle" class="update-overlay__title" data-i18n="common:upda
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
</optgroup>
Expand Down Expand Up @@ -392,7 +393,8 @@ <h3 id="updateOverlayTitle" class="update-overlay__title" data-i18n="common:upda
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
</optgroup>
Expand Down Expand Up @@ -656,7 +658,8 @@ <h2 style="display: flex; align-items: center; gap: 0.5rem; margin-bottom: 0.5re
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
<option value="Italian">Italian</option>
Expand All @@ -675,7 +678,8 @@ <h2 style="display: flex; align-items: center; gap: 0.5rem; margin-bottom: 0.5re
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
<option value="Italian">Italian</option>
Expand Down Expand Up @@ -1369,7 +1373,8 @@ <h2 id="glossaryEditorTitle" data-i18n="glossary:editor_title" style="margin: 0;
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
</optgroup>
Expand Down Expand Up @@ -1437,7 +1442,8 @@ <h2 id="glossaryEditorTitle" data-i18n="glossary:editor_title" style="margin: 0;
<option value="German">German</option>
<option value="Japanese">Japanese</option>
<option value="Korean">Korean</option>
<option value="Portuguese">Portuguese</option>
<option value="Portuguese (Brazil)">Portuguese (Brazil)</option>
<option value="Portuguese (Portugal)">Portuguese (Portugal)</option>
<option value="Russian">Russian</option>
<option value="Arabic">Arabic</option>
</optgroup>
Expand Down
4 changes: 4 additions & 0 deletions tests/test_epub_xhtml_lang_attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@
("German", "de"),
("Italian", "it"),
("Portuguese", "pt"),
("Portuguese (Brazil)", "pt-BR"),
("Portuguese (Portugal)", "pt-PT"),
("portuguese (brazil)", "pt-BR"),
("portuguese (portugal)", "pt-PT"),
("Russian", "ru"),
("Chinese", "zh"),
("Japanese", "ja"),
Expand Down