Upload image predictions

How to upload predictions on image data in a model run and sample upload formats.

Overview

To upload predictions in Labelbox, you need to create a prediction payload. In this section, we provide this payload for every supported annotation type.

Predictions Payload Types

Labelbox supports two formats for the annotations payload:

  • Python annotation types (recommended)
    • Provides a seamless transition between third-party platforms, machine learning pipelines, and Labelbox.
    • Allows you to build annotations locally with local file paths, numpy arrays, or URLs
    • Easily convert Python Annotation Type format to NDJSON format to quickly import annotations to Labelbox
    • Supports one-level nested classification (radio, checklist, or free-form text) under a tool or classification annotation.
  • JSON
    • Skips formatting annotation payload in the Labelbox Python annotation type
    • Supports any levels of nested classification (radio, checklist, or free-form text) under a tool or classification annotation.

Confidence Score

You can include confidence scores and custom metrics when you upload your model predictions to a model run. However, given the predictions and annotations in a model run, Labelbox will automatically calculate some auto-generated metrics upon upload.

📘

Uploading confidence scores is optional

If you do not specify a confidence score, the prediction will be treated as if it had a confidence score of 1.

Supported Prediction

The following predictions are supported for an image data row:

  • Radio
  • Checklist
  • Free-form text
  • Bounding box
  • Point
  • Polyline
  • Polygon
  • Segmentation masks

Classification

Radio (single-choice)

radio_prediction = lb_types.ClassificationAnnotation( name="radio_question", value=lb_types.Radio(answer = lb_types.ClassificationAnswer(name = "second_radio_answer", confidence=0.5)) )
radio_prediction_ndjson = { "name": "radio_question", "answer": {"name": "second_radio_answer", "confidence": 0.5} }

Classification: Nested radio

nested_radio_prediction = lb_types.ClassificationAnnotation( name="nested_radio_question", value=lb_types.Radio( answer=lb_types.ClassificationAnswer( name="first_radio_answer", confidence=0.5, classifications=[ lb_types.ClassificationAnnotation( name="sub_radio_question", value=lb_types.Radio( answer=lb_types.ClassificationAnswer( name="first_sub_radio_answer", confidence=0.5 ) ) ) ] ) ) )
nested_radio_prediction_ndjson = { "name": "nested_radio_question", "answer": { "name": "first_radio_answer", "confidence": 0.5, "classifications": [{ "name": "sub_radio_question", "answer": { "name": "first_sub_radio_answer", "confidence": 0.5 } }] } }

Classification: Nested checklist

nested_checklist_prediction = lb_types.ClassificationAnnotation( name="nested_checklist_question", value=lb_types.Checklist( answer=[lb_types.ClassificationAnswer( name="first_checklist_answer", confidence=0.5, classifications=[ lb_types.ClassificationAnnotation( name="sub_checklist_question", value=lb_types.Checklist( answer=[lb_types.ClassificationAnswer( name="first_sub_checklist_answer", confidence=0.5 )] )) ] )] ) )
nested_checklist_prediction_ndjson = { "name": "nested_checklist_question", "answer": [{ "name": "first_checklist_answer", "confidence": 0.5, "classifications": [{ "name": "sub_checklist_question", "answer": { "name": "first_sub_checklist_answer", "confidence": 0.5 } }] }] }

Checklist (multiple choice)

checklist_prediction = lb_types.ClassificationAnnotation( name="checklist_question", value=lb_types.Checklist( answer = [ lb_types.ClassificationAnswer( name = "first_checklist_answer", confidence=0.5 ), lb_types.ClassificationAnswer( name = "second_checklist_answer", confidence=0.5 ) ] ) )
checklist_prediction_ndjson = { "name": "checklist_question", "answer": [ {"name": "first_checklist_answer" , "confidence": 0.5}, {"name": "second_checklist_answer", "confidence": 0.5} ] }

Bounding box

bbox_prediction = lb_types.ObjectAnnotation( name = "bounding_box", confidence=0.5, value=lb_types.Rectangle( start=lb_types.Point(x=1690, y=977), # x = left, y = top end=lb_types.Point(x=1915, y=1307), # x= left + width , y = top + height ) )
bbox_prediction_ndjson = { "name": "bounding_box", "confidence": 0.5, "bbox": { "top": 977, "left": 1690, "height": 330, "width": 225 } }

Bounding box with nested classification

bbox_with_radio_subclass_prediction = lb_types.ObjectAnnotation( name="bbox_with_radio_subclass", confidence=0.5, value=lb_types.Rectangle( start=lb_types.Point(x=541, y=933), # x = left, y = top end=lb_types.Point(x=871, y=1124), # x= left + width , y = top + height ), classifications=[ lb_types.ClassificationAnnotation( name="sub_radio_question", value=lb_types.Radio(answer=lb_types.ClassificationAnswer(name="first_sub_radio_answer", confidence=0.5)) ) ] )
bbox_with_radio_subclass_prediction_ndjson = { "name": "bbox_with_radio_subclass", "confidence": 0.5 "classifications": [{ "name": "sub_radio_question", "answer": { "name":"first_sub_radio_answer", "confidence": 0.5} }], "bbox": { "top": 933, "left": 541, "height": 191, "width": 330 } }

Polygon

polygon_prediction = lb_types.ObjectAnnotation( name = "polygon", confidence = 0.5, value=lb_types.Polygon( points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419), lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519), lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579), lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)] ), )
polygon_prediction_ndjson = { "name": "polygon", "confidence": 0.5, "polygon": [ {"x": 1489.581, "y": 183.934}, {"x": 2278.306, "y": 256.885}, {"x": 2428.197, "y": 200.437}, {"x": 2560.0, "y": 335.419}, {"x": 2557.386, "y": 503.165}, {"x": 2320.596, "y": 503.103}, {"x": 2156.083, "y": 628.943}, {"x": 2161.111, "y": 785.519}, {"x": 2002.115, "y": 894.647}, {"x": 1838.456, "y": 877.874}, {"x": 1436.53, "y": 874.636}, {"x": 1411.403, "y": 758.579}, {"x": 1353.853, "y": 751.74}, {"x": 1345.264, "y": 453.461}, {"x": 1426.011, "y": 421.129}, {"x": 1489.581, "y": 183.934} ] }

Classification: free-form text

text_annotation = lb_types.ClassificationAnnotation( name="free_text", value=lb_types.Text(answer="sample text", confidence=0.5) )
text_annotation_ndjson = { "name": "free_text", "confidence": 0.5, "answer": "sample text", }

Segmentation mask

# We recommend to use byte array url = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png" response = requests.get(url) mask_data = lb.types.MaskData(im_bytes=response.content) # You can also use "url" instead of img_bytes to pass the PNG mask url. mask_prediction = lb_types.ObjectAnnotation( name="mask", confidence=0.5, value=lb_types.Mask( mask=mask_data, color=(255, 255, 255)) )
# using instanceURI, bytes array is not fully supported. mask_prediction_ndjson = { "name": "mask", "confidence": 0.5, "classifications": [], "mask": {"instanceURI": url, "colorRGB": (255, 255, 255)} }

Segmentation mask with nested classification

url_2 = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png" response_2 = requests.get(url_2) mask_data_2 = lb_types.MaskData(im_bytes=response_2.content) # Python annotation mask_with_text_subclass_prediction = lb_types.ObjectAnnotation( name = "mask_with_text_subclass", # must match your ontology feature"s name value=lb_types.Mask( mask=mask_data_2, confidence=0.5, color=(255, 255, 255)), classifications=[ lb_types.ClassificationAnnotation( name="sub_free_text", value=lb_types.Text(answer="free text answer") )] )
mask_with_text_subclass_prediction_ndjson = { "name": "mask_with_text_subclass", "mask": {"instanceURI": url_2, "colorRGB": (255, 255, 255)}, "classifications":[{ "name": "sub_free_text", "answer": "free text answer" }] }

Point

point_prediction = lb_types.ObjectAnnotation( name = "point", confidence=0.5, value = lb_types.Point(x=1166.606, y=1441.768), )
point_prediction_ndjson = { "name": "point", "confidence": 0.5, "classifications": [], "point": {"x": 1166.606, "y": 1441.768} }

Polyline

polyline_prediction = lb_types.ObjectAnnotation( name = "polyline", confidence=0.5, value=lb_types.Line( points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716), lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552), lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007), lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251), lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)] ), )
polyline_prediction_ndjson = { "name": "polyline", "confidence":0.5, "classifications": [], "line": [ {"x": 2534.353, "y": 249.471}, {"x": 2429.492, "y": 182.092}, {"x": 2294.322, "y": 221.962}, {"x": 2224.491, "y": 180.463}, {"x": 2136.123, "y": 204.716}, {"x": 1712.247, "y": 173.949}, {"x": 1703.838, "y": 84.438}, {"x": 1579.772, "y": 82.61}, {"x": 1583.442, "y": 167.552}, {"x": 1478.869, "y": 164.903}, {"x": 1418.941, "y": 318.149}, {"x": 1243.128, "y": 400.815}, {"x": 1022.067, "y": 319.007}, {"x": 892.367, "y": 379.216}, {"x": 670.273, "y": 364.408}, {"x": 613.114, "y": 288.16}, {"x": 377.559, "y": 238.251}, {"x": 368.087, "y": 185.064}, {"x": 246.557, "y": 167.286}, {"x": 236.648, "y": 285.61}, {"x": 90.929, "y": 326.412} ] }

Example: Upload predictions to model run

To upload predictions to a model run:

Before you start

These examples require the following libraries:

import labelbox as lb import labelbox.types as lb_types import uuid import requests

Replace the value of API_KEY with a valid API key to connect to the Labelbox client.

API_KEY = None client = lb.Client(API_KEY)

Step 1: Import data rows into Catalog

# send a sample image as batch to the project global_key = "2560px-Kitano_Street_Kobe01s5s4110.jpeg" test_img_url = { "row_data": "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg", "global_key": global_key } dataset = client.create_dataset(name="image_prediction_demo") task = dataset.create_data_rows([test_img_url]) print("Errors:",task.errors) print("Failed data rows:", task.failed_data_rows)

Step 2: Set up ontology for predictions

Your model run ontology should support all tools and classifications used in your predictions.

This example shows how to create an ontology with all supported prediction types .

ontology_builder = lb.OntologyBuilder( classifications=[ # List of Classification objects lb.Classification( class_type=lb.Classification.Type.RADIO, name="radio_question", options=[ lb.Option(value="first_radio_answer"), lb.Option(value="second_radio_answer") ] ), lb.Classification( class_type=lb.Classification.Type.CHECKLIST, name="checklist_question", options=[ lb.Option(value="first_checklist_answer"), lb.Option(value="second_checklist_answer") ] ), lb.Classification( class_type=lb.Classification.Type.TEXT, name="free_text" ), lb.Classification( class_type=lb.Classification.Type.RADIO, name="nested_radio_question", options=[ lb.Option("first_radio_answer", options=[ lb.Classification( class_type=lb.Classification.Type.RADIO, name="sub_radio_question", options=[lb.Option("first_sub_radio_answer")] ) ] ) ] ), lb.Classification( class_type=lb.Classification.Type.CHECKLIST, name="nested_checklist_question", options=[ lb.Option("first_checklist_answer", options=[ lb.Classification( class_type=lb.Classification.Type.CHECKLIST, name="sub_checklist_question", options=[lb.Option("first_sub_checklist_answer")] ) ] ) ] ), ], tools=[ # List of tools lb.Tool( tool=lb.Tool.Type.BBOX, name="bounding_box"), lb.Tool( tool=lb.Tool.Type.BBOX, name="bbox_with_radio_subclass", classifications=[ lb.Classification( class_type=lb.Classification.Type.RADIO, name="sub_radio_question", options=[ lb.Option(value="first_sub_radio_answer") ] ), ] ), lb.Tool( tool=lb.Tool.Type.POLYGON, name="polygon"), lb.Tool( tool=lb.Tool.Type.RASTER_SEGMENTATION, name="mask"), lb.Tool( tool=lb.Tool.Type.RASTER_SEGMENTATION, name="mask_with_text_subclass", classifications=[ lb.Classification( class_type=lb.Classification.Type.TEXT, name="sub_free_text") ] ), lb.Tool( tool=lb.Tool.Type.POINT, name="point"), lb.Tool( tool=lb.Tool.Type.LINE, name="polyline")] ) ontology = client.create_ontology("Image Prediction Import Demo", ontology_builder.asdict(), media_type=lb.MediaType.Image )

Step 3: Create model and model run

Create a Model using the ontology and a model run.

# create Model model = client.create_model(name="image_model_run_" + str(uuid.uuid4()), ontology_id=ontology.uid) # create Model Run model_run = model.create_model_run("iteration 1")

Step 4: Send data rows to model run

model_run.upsert_data_rows(global_keys=[global_key])

Step 5: Create prediction payloads

For help creating prediction payloads, see supported predictions. You can declare payloads as Python annotation types (preferred) or as NDJSON objects. This example demonstrates each format and shows how to compose annotations into labels attached to the data rows.

The resulting label_prediction_ndjson and label_prediction payloads should have exactly the same prediction content (except for the value of generated uuid string values).

label_prediction = [] label_prediction.append(lb_types.Label( data={"global_key" : global_key }, annotations = [ radio_prediction, nested_radio_prediction, checklist_prediction, nested_checklist_prediction, bbox_prediction, bbox_with_radio_subclass_prediction, polyline_prediction, polygon_prediction, mask_prediction, mask_with_text_subclass_prediction, point_prediction, text_annotation ] ) )
label_prediction_ndjson = [] for annot in [ radio_prediction_ndjson, checklist_prediction_ndjson, bbox_prediction_ndjson, bbox_with_radio_subclass_prediction_ndjson, polygon_prediction_ndjson, mask_prediction_ndjson, mask_with_text_subclass_prediction_ndjson, point_prediction_ndjson, polyline_prediction_ndjson, text_annotation_ndjson, nested_radio_prediction_ndjson, nested_checklist_prediction_ndjson ]: annot.update({ "dataRow": {"globalKey": global_key} }) label_prediction_ndjson.append(annot)

Step 6: Upload prediction payloads to model run

# Upload the prediction label to the Model Run upload_job_prediction = model_run.add_predictions( name="prediction_upload_job"+str(uuid.uuid4()), predictions=label_prediction) # Errors will appear for prediction uploads that failed. print("Errors:", upload_job_prediction.errors) print("Status of uploads: ", upload_job_prediction.statuses)

Step 7: Send annotations to a model run

This step is optional.

This example creates a project with ground truth annotations to visualize both annotations and predictions in the model run.

To send annotations to a model run:

  1. Import them into a project
  2. Create a label payload
  3. Send them to the model run.
# 7.1 Create a labelbox project project = client.create_project(name="Image Prediction Demo", media_type=lb.MediaType.Image) project.connect_ontology(ontology) # 7.2 Create a batch project.create_batch( "batch_predictions_demo", # Each batch in a project must have a unique name global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys priority=5 # priority between 1(Highest) - 5(lowest) ) # 7.3 Create your annotation payload as explained in: # https://docs.labelbox.com/reference/import-image-annotations#supported-annotations radio_annotation ... nested_radio_annotation ... nested_checklist_annotation ... checklist_annotation ... bbox_annotation ... bbox_with_radio_subclass_annotation ... polygon_annotation ... text_annotation ... mask_annotation ... mask_with_text_subclass_annotation ... point_annotation ... polyline_annotation ... # 7.4 Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations labels = [] annotations = [ radio_annotation, nested_radio_annotation, checklist_annotation, nested_checklist_annotation, text_annotation, bbox_annotation, bbox_with_radio_subclass_annotation, polygon_annotation, mask_annotation, mask_with_text_subclass_annotation, point_annotation, polyline_annotation ] labels.append( lb_types.Label( data={"global_key" : global_key }, annotations=annotations)) # 7.5 Upload annotations to the project using Label import upload_job_annotation = lb.LabelImport.create_from_objects( client = client, project_id = project.uid, name="annotation_import_" + str(uuid.uuid4()), labels=labels) upload_job_annotation.wait_until_done() # Errors will appear for annotation uploads that failed. print("Errors:", upload_job_annotation.errors) print("Status of uploads: ", upload_job_annotation.statuses) # 7.6 Send the annotations to the model run model_run.upsert_labels(project_id=project.uid)