TODEval/file_processor.py at main · emorynlp/TODEval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
import os
import json
from util import get_description

def get_states(type):
    processed_states = {}
    for f in os.listdir("data/multiwoz/"+type):
        if f.endswith(".json"):
            data = json.load(open("data/multiwoz/"+type+"/"+f, "r", encoding="utf-8"))
            for dialog in data:
                dialog_id = dialog['dialogue_id']
                utterances = [turn['utterance'] for turn in dialog['turns']]
                # slot_values = {}
                # requested_slots = []

                # last_user_turn = dialog['turns'][-2]
                # if last_user_turn['speaker'] != 'USER':
                #     print(f"Unexpected speaker in last user turn for dialog {dialog_id}")


                # information_types = ""
                # user_information = {}
                # agent_information = {}
                # for frame in last_user_turn['frames']:
                #     state = frame['state']
                #     if len(state['slot_values']) > 0:
                #         slot_values.update(state['slot_values'])
                #         information_types += f"## {frame['service']}:\n"
                #         for slot, value in state['slot_values'].items():
                #             information_types += f"* {slot}: {get_description(slot)}\n"
                #     if len(state['requested_slots']) > 0:
                #         requested_slots.extend(state['requested_slots'])

                user_slot = {}
                user_request = []
                agent_slot = {}
                for turn in dialog['turns']:
                    if turn['speaker'] == 'USER':
                        for frame in turn['frames']:
                            state = frame['state']
                            if len(state['slot_values']) > 0:
                                user_slot.update(state['slot_values'])
                            if len(state['requested_slots']) > 0:
                                user_request.extend(state['requested_slots'])
                    else:
                        for frame in turn['frames']:
                            slots = frame['slots']
                            for slot in slots:
                                agent_slot[slot['slot']] = slot['value']

                # if not all(user_slot.get(k) == v for k, v in slot_values.items()):
                #     print(f"Slot values do not match for dialog {dialog_id}")
                #     print(f"Expected: {slot_values}")
                #     print(f"Got: {user_slot}")
                #     raise ValueError("Slot values do not match")
                # if not set(requested_slots).issubset(set(user_request)):
                #     print(f"Requested slots do not match for dialog {dialog_id}")
                #     print(f"Expected: {requested_slots}")
                #     print(f"Got: {user_request}")
                #     raise ValueError("Requested slots do not match")

                processed_states[dialog_id] = {
                    'utterances': utterances,
                    'user_information': {
                        "slots": user_slot,
                        "requests": user_request
                    },
                    "agent_information": agent_slot
                }
    with open(f"data/multiwoz_{type}.json", "w", encoding="utf-8") as f:
        json.dump(processed_states, f, indent=4, ensure_ascii=False)

if __name__ == "__main__":
    get_states("train")
    get_states("dev")
    get_states("test")
    # res = {}
    # with open("data/multiwoz_manual_10.json", "r", encoding="utf-8") as f:
    #     dialogues = json.load(f)
    # with open("data/multiwoz_test.json", "r", encoding="utf-8") as f:
    #     test_dialogues = json.load(f)
    # for dial_id, dialogue in dialogues.items():
    #     res[dial_id] = {}
    #     res[dial_id]["utterances"] = dialogue
    #     res[dial_id]["slot_values"] = test_dialogues[dial_id]["slot_values"]
    #     res[dial_id]["requested_slots"] = test_dialogues[dial_id]["requested_slots"]
    # with open("data/multiwoz_manual_10_processed.json", "w", encoding="utf-8") as f:
    #     json.dump(res, f, indent=4, ensure_ascii=False)