Hugging Face's logo

Spaces:
aether-raid
/
atc-tts-mos
Sleeping

App Files Community
aether-raider commited on
Commit
5df8bb5
·
1 Parent(s): c084fda

made minor changes

Browse files
Files changed (2) hide show
  1. backend/session_manager.py +51 -37
  2. session_exports/55fd24f5-62bd-47e3-b50e-031888e261f9.json +516 -0
backend/session_manager.py CHANGED
@@ -61,51 +61,65 @@ class SessionManager:
61
  key = (clip.exercise, clip.exercise_id, clip.transcript)
62
  content_groups.setdefault(key, []).append(clip)
63
 
64
- # Model vs model pairs (ensure same gender)
 
65
  ab_model_pairs = []
66
- for _, group in content_groups.items():
67
- # Group clips by model and speaker (gender)
 
 
 
 
 
 
 
 
 
 
 
68
  model_speaker_map: Dict[str, Dict[str, List[Clip]]] = {}
69
  for clip in group:
70
  model_speaker_map.setdefault(clip.model, {}).setdefault(clip.speaker, []).append(clip)
71
-
72
  model_names = list(model_speaker_map.keys())
73
- if len(model_names) >= 2:
74
- # Try each possible gender to find matching clips
75
- for speaker in ["male", "female"]:
76
- # Find models that have clips for this speaker
77
- available_models = [
78
- model for model in model_names
79
- if speaker in model_speaker_map[model] and model_speaker_map[model][speaker]
80
- ]
81
-
82
- if len(available_models) >= 2:
83
- model_a, model_b = rng.sample(available_models, 2)
84
- clip_a = rng.choice(model_speaker_map[model_a][speaker])
85
- clip_b = rng.choice(model_speaker_map[model_b][speaker])
86
- ab_model_pairs.append((clip_a, clip_b))
87
- break # Found a valid pair for this content group
88
-
89
- if len(ab_model_pairs) >= 6:
90
- break
91
-
92
- # Gender vs gender pairs
93
  ab_gender_pairs = []
94
- for _, group in content_groups.items():
 
 
 
 
 
 
 
95
  model_gender_groups: Dict[str, Dict[str, List[Clip]]] = {}
96
  for clip in group:
97
- model_gender_groups.setdefault(clip.model, {}).setdefault(
98
- clip.speaker, []
99
- ).append(clip)
100
-
101
- for model, gender_groups in model_gender_groups.items():
102
- if "male" in gender_groups and "female" in gender_groups:
103
- clip_male = rng.choice(gender_groups["male"])
104
- clip_female = rng.choice(gender_groups["female"])
105
- ab_gender_pairs.append((clip_male, clip_female))
106
- if len(ab_gender_pairs) >= 6:
107
- break
108
- if len(ab_gender_pairs) >= 6:
109
  break
110
 
111
  session_data: Dict[str, Any] = {
 
61
  key = (clip.exercise, clip.exercise_id, clip.transcript)
62
  content_groups.setdefault(key, []).append(clip)
63
 
64
+
65
+ # --- Model vs Model (same gender, same exercise) ---
66
  ab_model_pairs = []
67
+ # Get all unique exercises
68
+ all_exercises = list({key[1] for key in content_groups.keys()})
69
+ rng.shuffle(all_exercises)
70
+ max_pairs = 6
71
+ for exercise_id in all_exercises:
72
+ # Find all content groups for this exercise
73
+ matching_keys = [k for k in content_groups if k[1] == exercise_id]
74
+ if not matching_keys:
75
+ continue
76
+ # Pick a random content group for this exercise
77
+ key = rng.choice(matching_keys)
78
+ group = content_groups[key]
79
+ # Group by model and speaker
80
  model_speaker_map: Dict[str, Dict[str, List[Clip]]] = {}
81
  for clip in group:
82
  model_speaker_map.setdefault(clip.model, {}).setdefault(clip.speaker, []).append(clip)
 
83
  model_names = list(model_speaker_map.keys())
84
+ if len(model_names) < 2:
85
+ continue
86
+ # Try to find a random valid gender for this exercise
87
+ valid_genders = [s for s in ["male", "female"] if sum(1 for m in model_names if s in model_speaker_map[m] and model_speaker_map[m][s]) >= 2]
88
+ if not valid_genders:
89
+ continue
90
+ speaker = rng.choice(valid_genders)
91
+ available_models = [model for model in model_names if speaker in model_speaker_map[model] and model_speaker_map[model][speaker]]
92
+ if len(available_models) < 2:
93
+ continue
94
+ model_a, model_b = rng.sample(available_models, 2)
95
+ clip_a = rng.choice(model_speaker_map[model_a][speaker])
96
+ clip_b = rng.choice(model_speaker_map[model_b][speaker])
97
+ ab_model_pairs.append((clip_a, clip_b))
98
+ if len(ab_model_pairs) >= max_pairs:
99
+ break
100
+
101
+ # --- Gender vs Gender (same model, same exercise) ---
 
 
102
  ab_gender_pairs = []
103
+ rng.shuffle(all_exercises)
104
+ for exercise_id in all_exercises:
105
+ matching_keys = [k for k in content_groups if k[1] == exercise_id]
106
+ if not matching_keys:
107
+ continue
108
+ key = rng.choice(matching_keys)
109
+ group = content_groups[key]
110
+ # Group by model and gender
111
  model_gender_groups: Dict[str, Dict[str, List[Clip]]] = {}
112
  for clip in group:
113
+ model_gender_groups.setdefault(clip.model, {}).setdefault(clip.speaker, []).append(clip)
114
+ valid_models = [m for m, genders in model_gender_groups.items() if "male" in genders and "female" in genders and genders["male"] and genders["female"]]
115
+ if not valid_models:
116
+ continue
117
+ model = rng.choice(valid_models)
118
+ gender_groups = model_gender_groups[model]
119
+ clip_male = rng.choice(gender_groups["male"])
120
+ clip_female = rng.choice(gender_groups["female"])
121
+ ab_gender_pairs.append((clip_male, clip_female))
122
+ if len(ab_gender_pairs) >= max_pairs:
 
 
123
  break
124
 
125
  session_data: Dict[str, Any] = {
session_exports/55fd24f5-62bd-47e3-b50e-031888e261f9.json ADDED
@@ -0,0 +1,516 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "session_metadata": {
3
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
4
+ "created_at": 1763346320.8270998,
5
+ "completed": true,
6
+ "exported_at": 1763346445.4934862,
7
+ "total_mos_ratings": 9,
8
+ "total_ab_comparisons": 12,
9
+ "created_at_readable": "2025-11-17T10:25:20.827100",
10
+ "exported_at_readable": "2025-11-17T10:27:25.493486"
11
+ },
12
+ "mos_ratings": [
13
+ {
14
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
15
+ "response_timestamp": 1763346379.0099323,
16
+ "clip_id": "csm_female_Exercise_5_225",
17
+ "exercise": "Exercise 5",
18
+ "exercise_id": "Exercise_5_225",
19
+ "transcript": "ZEBRA 2",
20
+ "model": "csm",
21
+ "display_model": "Model B",
22
+ "speaker": "female",
23
+ "clarity": null,
24
+ "pronunciation": null,
25
+ "prosody": null,
26
+ "naturalness": null,
27
+ "overall": null,
28
+ "comment": "",
29
+ "gender_mismatch": true,
30
+ "evaluation_type": "mos_rating",
31
+ "response_timestamp_readable": "2025-11-17T10:26:19.009932"
32
+ },
33
+ {
34
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
35
+ "response_timestamp": 1763346379.009947,
36
+ "clip_id": "csm_male_Exercise_5_239",
37
+ "exercise": "Exercise 5",
38
+ "exercise_id": "Exercise_5_239",
39
+ "transcript": "VIOLET restrict Sultan Shoal ops 500ft and below due flight of 2 F16 Sinjon Crossing eastbound, report established",
40
+ "model": "csm",
41
+ "display_model": "Model B",
42
+ "speaker": "male",
43
+ "clarity": null,
44
+ "pronunciation": null,
45
+ "prosody": null,
46
+ "naturalness": null,
47
+ "overall": null,
48
+ "comment": "",
49
+ "gender_mismatch": true,
50
+ "evaluation_type": "mos_rating",
51
+ "response_timestamp_readable": "2025-11-17T10:26:19.009947"
52
+ },
53
+ {
54
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
55
+ "response_timestamp": 1763346379.0099554,
56
+ "clip_id": "csm_female_Exercise_2_65",
57
+ "exercise": "Exercise 2",
58
+ "exercise_id": "Exercise_2_65",
59
+ "transcript": "TAIPAN radar identified, contact Tengah Approach 127.7.",
60
+ "model": "csm",
61
+ "display_model": "Model B",
62
+ "speaker": "female",
63
+ "clarity": null,
64
+ "pronunciation": null,
65
+ "prosody": null,
66
+ "naturalness": null,
67
+ "overall": null,
68
+ "comment": "",
69
+ "gender_mismatch": true,
70
+ "evaluation_type": "mos_rating",
71
+ "response_timestamp_readable": "2025-11-17T10:26:19.009955"
72
+ },
73
+ {
74
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
75
+ "response_timestamp": 1763346379.009962,
76
+ "clip_id": "xtts_female_Exercise_5_222",
77
+ "exercise": "Exercise 5",
78
+ "exercise_id": "Exercise_5_222",
79
+ "transcript": "Down the line, ZEBRA",
80
+ "model": "xtts",
81
+ "display_model": "Model A",
82
+ "speaker": "female",
83
+ "clarity": null,
84
+ "pronunciation": null,
85
+ "prosody": null,
86
+ "naturalness": null,
87
+ "overall": null,
88
+ "comment": "",
89
+ "gender_mismatch": true,
90
+ "evaluation_type": "mos_rating",
91
+ "response_timestamp_readable": "2025-11-17T10:26:19.009962"
92
+ },
93
+ {
94
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
95
+ "response_timestamp": 1763346379.009969,
96
+ "clip_id": "xtts_female_Exercise_5_254",
97
+ "exercise": "Exercise 5",
98
+ "exercise_id": "Exercise_5_254",
99
+ "transcript": "PLAZA 2 visual with Tengah.",
100
+ "model": "xtts",
101
+ "display_model": "Model A",
102
+ "speaker": "female",
103
+ "clarity": null,
104
+ "pronunciation": null,
105
+ "prosody": null,
106
+ "naturalness": null,
107
+ "overall": null,
108
+ "comment": "",
109
+ "gender_mismatch": true,
110
+ "evaluation_type": "mos_rating",
111
+ "response_timestamp_readable": "2025-11-17T10:26:19.009969"
112
+ },
113
+ {
114
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
115
+ "response_timestamp": 1763346379.0099764,
116
+ "clip_id": "xtts_male_Exercise_7_314",
117
+ "exercise": "Exercise 7",
118
+ "exercise_id": "Exercise_7_314",
119
+ "transcript": "BASKING copy BASKING 1 with 1 hung bomb on port side, request runway 18 hung ordnance recovery, descend 1000ft within Delta 4",
120
+ "model": "xtts",
121
+ "display_model": "Model A",
122
+ "speaker": "male",
123
+ "clarity": null,
124
+ "pronunciation": null,
125
+ "prosody": null,
126
+ "naturalness": null,
127
+ "overall": null,
128
+ "comment": "",
129
+ "gender_mismatch": true,
130
+ "evaluation_type": "mos_rating",
131
+ "response_timestamp_readable": "2025-11-17T10:26:19.009976"
132
+ },
133
+ {
134
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
135
+ "response_timestamp": 1763346379.0099828,
136
+ "clip_id": "orpheus_male_Exercise_4_167",
137
+ "exercise": "Exercise 4",
138
+ "exercise_id": "Exercise_4_167",
139
+ "transcript": "FIBER, request intentions for ILS, FIBER 1",
140
+ "model": "orpheus",
141
+ "display_model": "Model C",
142
+ "speaker": "male",
143
+ "clarity": null,
144
+ "pronunciation": null,
145
+ "prosody": null,
146
+ "naturalness": null,
147
+ "overall": null,
148
+ "comment": "",
149
+ "gender_mismatch": true,
150
+ "evaluation_type": "mos_rating",
151
+ "response_timestamp_readable": "2025-11-17T10:26:19.009983"
152
+ },
153
+ {
154
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
155
+ "response_timestamp": 1763346379.0099943,
156
+ "clip_id": "orpheus_female_Exercise_4_172",
157
+ "exercise": "Exercise 4",
158
+ "exercise_id": "Exercise_4_172",
159
+ "transcript": "FIBER 1 established localizer.",
160
+ "model": "orpheus",
161
+ "display_model": "Model C",
162
+ "speaker": "female",
163
+ "clarity": null,
164
+ "pronunciation": null,
165
+ "prosody": null,
166
+ "naturalness": null,
167
+ "overall": null,
168
+ "comment": "",
169
+ "gender_mismatch": true,
170
+ "evaluation_type": "mos_rating",
171
+ "response_timestamp_readable": "2025-11-17T10:26:19.009994"
172
+ },
173
+ {
174
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
175
+ "response_timestamp": 1763346379.0100038,
176
+ "clip_id": "orpheus_female_Exercise_5_225",
177
+ "exercise": "Exercise 5",
178
+ "exercise_id": "Exercise_5_225",
179
+ "transcript": "ZEBRA 2",
180
+ "model": "orpheus",
181
+ "display_model": "Model C",
182
+ "speaker": "female",
183
+ "clarity": null,
184
+ "pronunciation": null,
185
+ "prosody": null,
186
+ "naturalness": null,
187
+ "overall": null,
188
+ "comment": "",
189
+ "gender_mismatch": true,
190
+ "evaluation_type": "mos_rating",
191
+ "response_timestamp_readable": "2025-11-17T10:26:19.010004"
192
+ }
193
+ ],
194
+ "ab_comparisons": [
195
+ {
196
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
197
+ "response_timestamp": 1763346410.8734763,
198
+ "comparison_type": "model_vs_model",
199
+ "choice": null,
200
+ "comment": "",
201
+ "clip_a_id": "orpheus_female_Exercise_2_72",
202
+ "clip_a_exercise": "Exercise 2",
203
+ "clip_a_exercise_id": "Exercise_2_72",
204
+ "clip_a_transcript": "NINJA clear operate Sultan Shoal 1000ft and below, report ops normal every 15mins",
205
+ "clip_a_model": "orpheus",
206
+ "clip_a_display_model": "Model C",
207
+ "clip_a_speaker": "female",
208
+ "clip_b_id": "xtts_female_Exercise_2_72",
209
+ "clip_b_exercise": "Exercise 2",
210
+ "clip_b_exercise_id": "Exercise_2_72",
211
+ "clip_b_transcript": "NINJA clear operate Sultan Shoal 1000ft and below, report ops normal every 15mins",
212
+ "clip_b_model": "xtts",
213
+ "clip_b_display_model": "Model A",
214
+ "clip_b_speaker": "female",
215
+ "gender_mismatch_a": true,
216
+ "gender_mismatch_b": true,
217
+ "evaluation_type": "ab_comparison",
218
+ "response_timestamp_readable": "2025-11-17T10:26:50.873476"
219
+ },
220
+ {
221
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
222
+ "response_timestamp": 1763346410.873491,
223
+ "comparison_type": "model_vs_model",
224
+ "choice": null,
225
+ "comment": "",
226
+ "clip_a_id": "orpheus_male_Exercise_5_235",
227
+ "clip_a_exercise": "Exercise 5",
228
+ "clip_a_exercise_id": "Exercise_5_235",
229
+ "clip_a_transcript": "FIBER depart western coast 2000ft contact Tengah Approach 127.7.",
230
+ "clip_a_model": "orpheus",
231
+ "clip_a_display_model": "Model C",
232
+ "clip_a_speaker": "male",
233
+ "clip_b_id": "csm_male_Exercise_5_235",
234
+ "clip_b_exercise": "Exercise 5",
235
+ "clip_b_exercise_id": "Exercise_5_235",
236
+ "clip_b_transcript": "FIBER depart western coast 2000ft contact Tengah Approach 127.7.",
237
+ "clip_b_model": "csm",
238
+ "clip_b_display_model": "Model B",
239
+ "clip_b_speaker": "male",
240
+ "gender_mismatch_a": true,
241
+ "gender_mismatch_b": true,
242
+ "evaluation_type": "ab_comparison",
243
+ "response_timestamp_readable": "2025-11-17T10:26:50.873491"
244
+ },
245
+ {
246
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
247
+ "response_timestamp": 1763346410.873502,
248
+ "comparison_type": "model_vs_model",
249
+ "choice": null,
250
+ "comment": "",
251
+ "clip_a_id": "csm_female_Exercise_6_290",
252
+ "clip_a_exercise": "Exercise 6",
253
+ "clip_a_exercise_id": "Exercise_6_290",
254
+ "clip_a_transcript": "HONDA, established 1000ft and below",
255
+ "clip_a_model": "csm",
256
+ "clip_a_display_model": "Model B",
257
+ "clip_a_speaker": "female",
258
+ "clip_b_id": "xtts_female_Exercise_6_290",
259
+ "clip_b_exercise": "Exercise 6",
260
+ "clip_b_exercise_id": "Exercise_6_290",
261
+ "clip_b_transcript": "HONDA, established 1000ft and below",
262
+ "clip_b_model": "xtts",
263
+ "clip_b_display_model": "Model A",
264
+ "clip_b_speaker": "female",
265
+ "gender_mismatch_a": true,
266
+ "gender_mismatch_b": true,
267
+ "evaluation_type": "ab_comparison",
268
+ "response_timestamp_readable": "2025-11-17T10:26:50.873502"
269
+ },
270
+ {
271
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
272
+ "response_timestamp": 1763346410.8735106,
273
+ "comparison_type": "model_vs_model",
274
+ "choice": null,
275
+ "comment": "",
276
+ "clip_a_id": "orpheus_female_Exercise_6_297",
277
+ "clip_a_exercise": "Exercise 6",
278
+ "clip_a_exercise_id": "Exercise_6_297",
279
+ "clip_a_transcript": "CACTUS radar identified, contact Paya Lebar Approach 298.0.",
280
+ "clip_a_model": "orpheus",
281
+ "clip_a_display_model": "Model C",
282
+ "clip_a_speaker": "female",
283
+ "clip_b_id": "xtts_female_Exercise_6_297",
284
+ "clip_b_exercise": "Exercise 6",
285
+ "clip_b_exercise_id": "Exercise_6_297",
286
+ "clip_b_transcript": "CACTUS radar identified, contact Paya Lebar Approach 298.0.",
287
+ "clip_b_model": "xtts",
288
+ "clip_b_display_model": "Model A",
289
+ "clip_b_speaker": "female",
290
+ "gender_mismatch_a": true,
291
+ "gender_mismatch_b": true,
292
+ "evaluation_type": "ab_comparison",
293
+ "response_timestamp_readable": "2025-11-17T10:26:50.873511"
294
+ },
295
+ {
296
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
297
+ "response_timestamp": 1763346410.87352,
298
+ "comparison_type": "model_vs_model",
299
+ "choice": "B",
300
+ "comment": "aedea",
301
+ "clip_a_id": "xtts_female_Exercise_8_366",
302
+ "clip_a_exercise": "Exercise 8",
303
+ "clip_a_exercise_id": "Exercise_8_366",
304
+ "clip_a_transcript": "CAMEL ceasing service contact Tower 127.7",
305
+ "clip_a_model": "xtts",
306
+ "clip_a_display_model": "Model A",
307
+ "clip_a_speaker": "female",
308
+ "clip_b_id": "orpheus_female_Exercise_8_366",
309
+ "clip_b_exercise": "Exercise 8",
310
+ "clip_b_exercise_id": "Exercise_8_366",
311
+ "clip_b_transcript": "CAMEL ceasing service contact Tower 127.7",
312
+ "clip_b_model": "orpheus",
313
+ "clip_b_display_model": "Model C",
314
+ "clip_b_speaker": "female",
315
+ "gender_mismatch_a": false,
316
+ "gender_mismatch_b": false,
317
+ "evaluation_type": "ab_comparison",
318
+ "response_timestamp_readable": "2025-11-17T10:26:50.873520"
319
+ },
320
+ {
321
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
322
+ "response_timestamp": 1763346410.8735301,
323
+ "comparison_type": "model_vs_model",
324
+ "choice": "tie",
325
+ "comment": "",
326
+ "clip_a_id": "csm_female_Exercise_3_121",
327
+ "clip_a_exercise": "Exercise 3",
328
+ "clip_a_exercise_id": "Exercise_3_121",
329
+ "clip_a_transcript": "Tengah Approach, HILTON with you for Western Coast departure",
330
+ "clip_a_model": "csm",
331
+ "clip_a_display_model": "Model B",
332
+ "clip_a_speaker": "female",
333
+ "clip_b_id": "xtts_female_Exercise_3_121",
334
+ "clip_b_exercise": "Exercise 3",
335
+ "clip_b_exercise_id": "Exercise_3_121",
336
+ "clip_b_transcript": "Tengah Approach, HILTON with you for Western Coast departure",
337
+ "clip_b_model": "xtts",
338
+ "clip_b_display_model": "Model A",
339
+ "clip_b_speaker": "female",
340
+ "gender_mismatch_a": false,
341
+ "gender_mismatch_b": true,
342
+ "evaluation_type": "ab_comparison",
343
+ "response_timestamp_readable": "2025-11-17T10:26:50.873530"
344
+ },
345
+ {
346
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
347
+ "response_timestamp": 1763346442.3536737,
348
+ "comparison_type": "gender_vs_gender",
349
+ "choice": "tie",
350
+ "comment": "",
351
+ "clip_a_id": "xtts_male_Exercise_1_2",
352
+ "clip_a_exercise": "Exercise 1",
353
+ "clip_a_exercise_id": "Exercise_1_2",
354
+ "clip_a_transcript": "Request clearance, CAMEL 1xF16 for Western Coast Departure for Visual Straight-in.",
355
+ "clip_a_model": "xtts",
356
+ "clip_a_display_model": "Model A",
357
+ "clip_a_speaker": "male",
358
+ "clip_b_id": "xtts_female_Exercise_1_2",
359
+ "clip_b_exercise": "Exercise 1",
360
+ "clip_b_exercise_id": "Exercise_1_2",
361
+ "clip_b_transcript": "Request clearance, CAMEL 1xF16 for Western Coast Departure for Visual Straight-in.",
362
+ "clip_b_model": "xtts",
363
+ "clip_b_display_model": "Model A",
364
+ "clip_b_speaker": "female",
365
+ "gender_mismatch_a": false,
366
+ "gender_mismatch_b": false,
367
+ "evaluation_type": "ab_comparison",
368
+ "response_timestamp_readable": "2025-11-17T10:27:22.353674"
369
+ },
370
+ {
371
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
372
+ "response_timestamp": 1763346442.353687,
373
+ "comparison_type": "gender_vs_gender",
374
+ "choice": null,
375
+ "comment": "",
376
+ "clip_a_id": "xtts_male_Exercise_6_275",
377
+ "clip_a_exercise": "Exercise 6",
378
+ "clip_a_exercise_id": "Exercise_6_275",
379
+ "clip_a_transcript": "Tower controller requests clearance from Approach via landline, MANTIS for Delta 4 via Ordnance Route departure.",
380
+ "clip_a_model": "xtts",
381
+ "clip_a_display_model": "Model A",
382
+ "clip_a_speaker": "male",
383
+ "clip_b_id": "xtts_female_Exercise_6_275",
384
+ "clip_b_exercise": "Exercise 6",
385
+ "clip_b_exercise_id": "Exercise_6_275",
386
+ "clip_b_transcript": "Tower controller requests clearance from Approach via landline, MANTIS for Delta 4 via Ordnance Route departure.",
387
+ "clip_b_model": "xtts",
388
+ "clip_b_display_model": "Model A",
389
+ "clip_b_speaker": "female",
390
+ "gender_mismatch_a": true,
391
+ "gender_mismatch_b": false,
392
+ "evaluation_type": "ab_comparison",
393
+ "response_timestamp_readable": "2025-11-17T10:27:22.353687"
394
+ },
395
+ {
396
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
397
+ "response_timestamp": 1763346442.353696,
398
+ "comparison_type": "gender_vs_gender",
399
+ "choice": "tie",
400
+ "comment": "",
401
+ "clip_a_id": "orpheus_male_Exercise_3_109",
402
+ "clip_a_exercise": "Exercise 3",
403
+ "clip_a_exercise_id": "Exercise_3_109",
404
+ "clip_a_transcript": "RAFFLES IMF.",
405
+ "clip_a_model": "orpheus",
406
+ "clip_a_display_model": "Model C",
407
+ "clip_a_speaker": "male",
408
+ "clip_b_id": "orpheus_female_Exercise_3_109",
409
+ "clip_b_exercise": "Exercise 3",
410
+ "clip_b_exercise_id": "Exercise_3_109",
411
+ "clip_b_transcript": "RAFFLES IMF.",
412
+ "clip_b_model": "orpheus",
413
+ "clip_b_display_model": "Model C",
414
+ "clip_b_speaker": "female",
415
+ "gender_mismatch_a": false,
416
+ "gender_mismatch_b": false,
417
+ "evaluation_type": "ab_comparison",
418
+ "response_timestamp_readable": "2025-11-17T10:27:22.353696"
419
+ },
420
+ {
421
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
422
+ "response_timestamp": 1763346442.3537056,
423
+ "comparison_type": "gender_vs_gender",
424
+ "choice": "female",
425
+ "comment": "",
426
+ "clip_a_id": "orpheus_male_Exercise_2_70",
427
+ "clip_a_exercise": "Exercise 2",
428
+ "clip_a_exercise_id": "Exercise_2_70",
429
+ "clip_a_transcript": "NINJA copy, report established Sultan Shoal",
430
+ "clip_a_model": "orpheus",
431
+ "clip_a_display_model": "Model C",
432
+ "clip_a_speaker": "male",
433
+ "clip_b_id": "orpheus_female_Exercise_2_70",
434
+ "clip_b_exercise": "Exercise 2",
435
+ "clip_b_exercise_id": "Exercise_2_70",
436
+ "clip_b_transcript": "NINJA copy, report established Sultan Shoal",
437
+ "clip_b_model": "orpheus",
438
+ "clip_b_display_model": "Model C",
439
+ "clip_b_speaker": "female",
440
+ "gender_mismatch_a": false,
441
+ "gender_mismatch_b": false,
442
+ "evaluation_type": "ab_comparison",
443
+ "response_timestamp_readable": "2025-11-17T10:27:22.353706"
444
+ },
445
+ {
446
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
447
+ "response_timestamp": 1763346442.3537147,
448
+ "comparison_type": "gender_vs_gender",
449
+ "choice": "female",
450
+ "comment": "",
451
+ "clip_a_id": "orpheus_male_Exercise_5_231",
452
+ "clip_a_exercise": "Exercise 5",
453
+ "clip_a_exercise_id": "Exercise_5_231",
454
+ "clip_a_transcript": "ZEBRA 2 established localizer.",
455
+ "clip_a_model": "orpheus",
456
+ "clip_a_display_model": "Model C",
457
+ "clip_a_speaker": "male",
458
+ "clip_b_id": "orpheus_female_Exercise_5_231",
459
+ "clip_b_exercise": "Exercise 5",
460
+ "clip_b_exercise_id": "Exercise_5_231",
461
+ "clip_b_transcript": "ZEBRA 2 established localizer.",
462
+ "clip_b_model": "orpheus",
463
+ "clip_b_display_model": "Model C",
464
+ "clip_b_speaker": "female",
465
+ "gender_mismatch_a": false,
466
+ "gender_mismatch_b": false,
467
+ "evaluation_type": "ab_comparison",
468
+ "response_timestamp_readable": "2025-11-17T10:27:22.353715"
469
+ },
470
+ {
471
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
472
+ "response_timestamp": 1763346442.3537233,
473
+ "comparison_type": "gender_vs_gender",
474
+ "choice": null,
475
+ "comment": "",
476
+ "clip_a_id": "xtts_male_Exercise_1_13",
477
+ "clip_a_exercise": "Exercise 1",
478
+ "clip_a_exercise_id": "Exercise_1_13",
479
+ "clip_a_transcript": "TAIPAN visual with Tengah.",
480
+ "clip_a_model": "xtts",
481
+ "clip_a_display_model": "Model A",
482
+ "clip_a_speaker": "male",
483
+ "clip_b_id": "xtts_female_Exercise_1_13",
484
+ "clip_b_exercise": "Exercise 1",
485
+ "clip_b_exercise_id": "Exercise_1_13",
486
+ "clip_b_transcript": "TAIPAN visual with Tengah.",
487
+ "clip_b_model": "xtts",
488
+ "clip_b_display_model": "Model A",
489
+ "clip_b_speaker": "female",
490
+ "gender_mismatch_a": true,
491
+ "gender_mismatch_b": false,
492
+ "evaluation_type": "ab_comparison",
493
+ "response_timestamp_readable": "2025-11-17T10:27:22.353723"
494
+ }
495
+ ],
496
+ "overall_feedback": [
497
+ {
498
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
499
+ "overall_preference": "All models were similar quality",
500
+ "final_comments": "",
501
+ "timestamp": 1763346445.4933136
502
+ }
503
+ ],
504
+ "summary": {
505
+ "session_id": "55fd24f5-62bd-47e3-b50e-031888e261f9",
506
+ "evaluation_completed": true,
507
+ "total_mos_ratings": 9,
508
+ "total_ab_comparisons": 12,
509
+ "models_evaluated": [
510
+ "csm",
511
+ "xtts",
512
+ "orpheus"
513
+ ],
514
+ "evaluation_date": "2025-11-17T10:25:20.827100"
515
+ }
516
+ }