How to export multimodal chat evaluation annotations and sample export formats.
Export JSON annotations
# Set the export params to include/exclude certain fields.
export_params= {
"attachments": True,
"metadata_fields": True,
"data_row_details": True,
"project_details": True,
"label_details": True,
"performance_details": True,
"interpolated_frames": True
}
# Note: Filters follow AND logic, so typically using one filter is sufficient.
filters= {
"last_activity_at": ["2000-01-01 00:00:00", "2050-01-01 00:00:00"],
"workflow_status": "<wkf-status>"
}
export_task = project.export(params=export_params, filters=filters)
export_task.wait_till_done()
# Return JSON output strings from export task results/errors, one by one:
# Callback used for JSON Converter
def json_stream_handler(output: lb.BufferedJsonConverterOutput):
print(output.json)
if export_task.has_errors():
export_task.get_buffered_stream(
stream_type=lb.StreamType.ERRORS
).start(stream_handler=lambda error: print(error))
if export_task.has_result():
export_json = export_task.get_buffered_stream(
stream_type=lb.StreamType.RESULT
).start(stream_handler=json_stream_handler)
print("file size: ", export_task.get_total_file_size(stream_type=lb.StreamType.RESULT))
print("line count: ", export_task.get_total_lines(stream_type=lb.StreamType.RESULT))
# Set the export params to include/exclude certain fields. Make sure each of these fields are correctly grabbed
export_params= {
"attachments": True,
"metadata_fields": True,
"data_row_details": True,
"project_details": True,
"label_details": True,
"performance_details": True
}
# You can set the range for last_activity_at and label_created_at.
# For context, last_activity_at captures the creation and modification of labels, metadata, status, comments and reviews.
# Note: This is an AND logic between the filters, so usually using one filter is sufficient.
filters= {
"last_activity_at": ["2000-01-01 00:00:00", "2050-01-01 00:00:00"],
"workflow_status": "Done"
}
export_task = project.export_v2(params=export_params, filters=filters)
export_task.wait_till_done()
if export_task.errors:
print(export_task.errors)
export_json = export_task.result
print("results: ", export_json)
Annotation export formats
Classification - Radio
{
"feature_id": "clws3nkhh00043b6p7qktjffs",
"feature_schema_id": "clws3n1yu055907wlg1uh1i8g",
"name": "Context subject",
"value": "context_subject",
"message_id": "clws2arr200073b6p7ehwkoh4",
"conversational_radio_answer": {
"feature_id": "clws3nkhh00033b6ps99i1d56",
"feature_schema_id": "clws3n1yu055a07wl2axp39nq",
"name": "Car",
"value": "car",
"classifications": []
}
}
Classification - Free-form text
{
"feature_id": "clws3or5q00093b6p00oxtkve",
"feature_schema_id": "clws3n1yv055v07wlbnd57c9h",
"name": "Additional comment",
"value": "additional_comment",
"text_answer": {
"content": "Color: white"
}
}
Classification - Checklist
{
"feature_id": "clpiyav9m00053b6mpe2qk0im",
"feature_schema_id": "clpiya8h507dx072efo9kb97j",
"name": "Evaluations done",
"checklist_answers": [
{
"feature_id": "clpiyav9m00043b6mvhur1p19",
"name": "Usefulness",
"classifications": []
},
{
"feature_id": "clpiyax4d00073b6mqyyt0504",
"name": "Toxicity",
"classifications": []
}
]
}
Message Ranking
{
"feature_id": "clwrt25ar00093b6po9g7fiu3",
"feature_schema_id": "clwkfbcnn01kl07z68er64ffi",
"name": "Rank given prompt",
"annotation_kind": "MessageRanking",
"classifications": [],
"ranked_messages": {
"parent_message_id": "clwrt16zz00033b6p6ud7w233",
"ranked_messages": [
{
"message_id": "clwrt1fw700063b6pe8zpo1xp",
"order": 1,
"model_config_id": "e606593e-def3-4662-9737-3fc4ac412dd3"
},
{
"message_id": "clwrt1fw700073b6p8k3vkh1x",
"order": 2,
"model_config_id": "8fcda464-a0c7-4300-a9fa-c9a5cf82cab2"
}
]
}
}
Message Multi Selection
{
"feature_id": "clwrt2dru000b3b6pkj3i5zxk",
"feature_schema_id": "clwknckmp030v07xp6kow09xd",
"name": "Subject relevance",
"annotation_kind": "MessageMultiSelection",
"classifications": [],
"selected_messages": {
"parent_message_id": "clwrt16zz00033b6p6ud7w233",
"messages": [
{
"message_id": "clwrt1fw700063b6pe8zpo1xp",
"model_config_id": "e606593e-def3-4662-9737-3fc4ac412dd3"
}
]
}
}
Sample project export
{
"data_row": {
"id": "clwrm93488cpd0734f740r9lr",
"global_key": "clwrm92sb0grg07wncr6dhihw",
"row_data": "{\"type\":\"application/vnd.labelbox.conversational.model-chat-evaluation\",\"version\":2,\"actors\":{\"clwrt16zz00023b6pg6hlhtub\":{\"role\":\"human\",\"metadata\":{\"name\":\"User\"}},\"clwrt17a900043b6py2tepemd\":{\"role\":\"model\",\"metadata\":{\"modelConfigId\":\"e606593e-def3-4662-9737-3fc4ac412dd3\"}},\"clwrt17a900053b6p0xtyv1im\":{\"role\":\"model\",\"metadata\":{\"modelConfigId\":\"8fcda464-a0c7-4300-a9fa-c9a5cf82cab2\"}}},\"messages\":{\"clwrt16zz00033b6p6ud7w233\":{\"actorId\":\"clwrt16zz00023b6pg6hlhtub\",\"content\":[{\"type\":\"text\",\"content\":\"Is there a car in this image?\"},{\"type\":\"fileData\",\"fileUri\":\"https://c7.staticflickr.com/3/2447/3799589580_4b73283f92_o.jpg\",\"mimeType\":\"image/jpeg\"}],\"childMessageIds\":[\"clwrt1fw700063b6pe8zpo1xp\",\"clwrt1fw700073b6p8k3vkh1x\"]},\"clwrt1fw700063b6pe8zpo1xp\":{\"actorId\":\"clwrt17a900043b6py2tepemd\",\"content\":[{\"type\":\"text\",\"content\":\"Yes, there is a car in this image. It appears to be a white hatchback parked in a parking lot.\"}],\"childMessageIds\":[]},\"clwrt1fw700073b6p8k3vkh1x\":{\"actorId\":\"clwrt17a900053b6p0xtyv1im\",\"content\":[{\"type\":\"text\",\"content\":\" Yes, there is a white Volkswagen Golf in the image.\"}],\"childMessageIds\":[]}},\"rootMessageIds\":[\"clwrt16zz00033b6p6ud7w233\"]}",
"details": {
"dataset_id": "clwkf82ul001l0755mhckcmuy",
"dataset_name": "Model evaluation 101",
"created_at": "2024-05-29T09:20:43.342+00:00",
"updated_at": "2024-05-29T12:31:52.027+00:00",
"last_activity_at": "2024-05-29T12:33:21.203+00:00",
"created_by": "[email protected]"
}
},
"media_attributes": {
"asset_type": "conversational",
"mime_type": "application/vnd.labelbox.conversational.model-chat-evaluation",
"labelable_ids": [
"clwrt16zz00033b6p6ud7w233",
"clwrt1fw700063b6pe8zpo1xp",
"clwrt1fw700073b6p8k3vkh1x"
],
"message_count": 3
},
"attachments": [],
"metadata_fields": [],
"projects": {
"clwrm92jc0grc07wncq654q5s": {
"name": "Multi modal chat evalution",
"labels": [
{
"label_kind": "Default",
"version": "1.0.0",
"id": "clwrsyrdp02mr07x5378j3ysr",
"label_details": {
"created_at": "2024-05-29T12:33:21.000+00:00",
"updated_at": "2024-05-29T12:33:21.000+00:00",
"created_by": "[email protected]",
"content_last_updated_at": "2024-05-29T12:33:20.787+00:00",
"reviews": []
},
"performance_details": {
"seconds_to_create": 250,
"seconds_to_review": 0,
"skipped": false
},
"annotations": {
"objects": [
{
"feature_id": "clwrt25ar00093b6po9g7fiu3",
"feature_schema_id": "clwkfbcnn01kl07z68er64ffi",
"name": "Rank given prompt",
"annotation_kind": "MessageRanking",
"classifications": [],
"ranked_messages": {
"parent_message_id": "clwrt16zz00033b6p6ud7w233",
"ranked_messages": [
{
"message_id": "clwrt1fw700063b6pe8zpo1xp",
"order": 1,
"model_config_id": "e606593e-def3-4662-9737-3fc4ac412dd3"
},
{
"message_id": "clwrt1fw700073b6p8k3vkh1x",
"order": 2,
"model_config_id": "8fcda464-a0c7-4300-a9fa-c9a5cf82cab2"
}
]
}
},
{
"feature_id": "clwrt2dru000b3b6pkj3i5zxk",
"feature_schema_id": "clwknckmp030v07xp6kow09xd",
"name": "Subject relevance",
"annotation_kind": "MessageMultiSelection",
"classifications": [],
"selected_messages": {
"parent_message_id": "clwrt16zz00033b6p6ud7w233",
"messages": [
{
"message_id": "clwrt1fw700063b6pe8zpo1xp",
"model_config_id": "e606593e-def3-4662-9737-3fc4ac412dd3"
}
]
}
}
],
"classifications": [],
"relationships": []
}
}
],
"project_details": {
"ontology_id": "clwkn6leb034b07waeolfb7tl",
"task_id": "0c392f64-bc65-44e6-8848-ae5a12000328",
"task_name": "Initial review task",
"batch_id": "c2267a60-1d9c-11ef-9edf-ed6f8eede535",
"batch_name": "batch_clwrm92jc0grc07wncq654q5s_clwrm9eph006d0750b3s5b2cr",
"workflow_status": "IN_REVIEW",
"priority": 5,
"consensus_expected_label_count": 2,
"workflow_history": [
{
"action": "Move",
"created_at": "2024-05-29T12:33:21.160+00:00",
"created_by": "[email protected]",
"previous_task_name": "Initial labeling task",
"previous_task_id": "ebf41e2b-e231-0e43-adee-92252c1858dd",
"next_task_name": "Initial review task",
"next_task_id": "0c392f64-bc65-44e6-8848-ae5a12000328"
},
{
"action": "Move",
"created_at": "2024-05-29T12:33:21.151+00:00",
"created_by": "[email protected]",
"next_task_name": "Initial labeling task",
"next_task_id": "ebf41e2b-e231-0e43-adee-92252c1858dd"
}
]
},
"project_model_configs": {
"models": {
"5d9647e7-aafe-4275-944d-c2e9b122105a": {
"name": "Google Gemini Pro Vision",
"created_at": "2023-12-13T21:51:42.657+00:00"
},
"036aeb03-b0b4-442e-a6e1-161382bc1228": {
"name": "OpenAI GPT-4o",
"created_at": "2024-05-23T23:56:03.078+00:00"
}
},
"model_configs": {
"e606593e-def3-4662-9737-3fc4ac412dd3": {
"name": "GPT_4o_multi_modal",
"model_id": "036aeb03-b0b4-442e-a6e1-161382bc1228",
"inference_params": {
"top_p": 1,
"detail": "high",
"temperature": 0,
"use_text_only": false,
"max_new_tokens": 1024,
"use_image_only": false,
"use_attachments": false
}
},
"8fcda464-a0c7-4300-a9fa-c9a5cf82cab2": {
"name": "Gemini_pro_vison_multi_modal",
"model_id": "5d9647e7-aafe-4275-944d-c2e9b122105a",
"inference_params": {
"use_attachments": false
}
}
},
"project_model_configs": [
{
"id": "clwrsxr1y009i07v93w2h45qm",
"model_id": "036aeb03-b0b4-442e-a6e1-161382bc1228",
"model_config_id": "e606593e-def3-4662-9737-3fc4ac412dd3"
},
{
"id": "clwrsyext00ar07v5czpo7w2a",
"model_id": "5d9647e7-aafe-4275-944d-c2e9b122105a",
"model_config_id": "8fcda464-a0c7-4300-a9fa-c9a5cf82cab2"
}
]
},
"project_tags": []
}
}