From 8adc8d4a5d6f5a33474f85093d8bb43c24c2b32e Mon Sep 17 00:00:00 2001 From: whcao <41630003+HIT-cwh@users.noreply.github.com> Date: Tue, 9 Jul 2024 08:42:51 +0800 Subject: [PATCH] [Feature] support output without loss in openai_map_fn (#816) support output without loss in openai_map_fn --- xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py index 64ed642f6..c1798dc45 100644 --- a/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py +++ b/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py @@ -32,10 +32,13 @@ def openai_map_fn(example): elif msg['role'] == 'user': input += msg['content'] elif msg['role'] == 'assistant': + output_with_loss = msg.get('loss', True) + output_with_loss = output_with_loss.lower() == 'true' conversation.append({ 'system': system, 'input': input, - 'output': msg['content'] + 'output': msg['content'], + 'output_with_loss': output_with_loss }) system = '' input = ''